| /* |
| * Copyright (C) 2004, 2006, 2007, 2008 Apple Inc. All rights reserved. |
| * Copyright (C) 2006 Alexey Proskuryakov <ap@nypop.com> |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY |
| * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY |
| * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include "config.h" |
| #include "TextCodecICU.h" |
| |
| #include "CharacterNames.h" |
| #include "CString.h" |
| #include "PlatformString.h" |
| #include "ThreadGlobalData.h" |
| #include <unicode/ucnv.h> |
| #include <unicode/ucnv_cb.h> |
| #include <wtf/Assertions.h> |
| #include <wtf/PassOwnPtr.h> |
| #include <wtf/StringExtras.h> |
| #include <wtf/Threading.h> |
| |
| using std::min; |
| |
| namespace WebCore { |
| |
| const size_t ConversionBufferSize = 16384; |
| |
| ICUConverterWrapper::~ICUConverterWrapper() |
| { |
| if (converter) |
| ucnv_close(converter); |
| } |
| |
| static UConverter*& cachedConverterICU() |
| { |
| return threadGlobalData().cachedConverterICU().converter; |
| } |
| |
| static PassOwnPtr<TextCodec> newTextCodecICU(const TextEncoding& encoding, const void*) |
| { |
| return new TextCodecICU(encoding); |
| } |
| |
| void TextCodecICU::registerBaseEncodingNames(EncodingNameRegistrar registrar) |
| { |
| registrar("UTF-8", "UTF-8"); |
| } |
| |
| void TextCodecICU::registerBaseCodecs(TextCodecRegistrar registrar) |
| { |
| registrar("UTF-8", newTextCodecICU, 0); |
| } |
| |
| // FIXME: Registering all the encodings we get from ucnv_getAvailableName |
| // includes encodings we don't want or need. For example, all |
| // the encodings with commas and version numbers. |
| |
| void TextCodecICU::registerExtendedEncodingNames(EncodingNameRegistrar registrar) |
| { |
| // We register Hebrew with logical ordering using a separate name. |
| // Otherwise, this would share the same canonical name as the |
| // visual ordering case, and then TextEncoding could not tell them |
| // apart; ICU treats these names as synonyms. |
| registrar("ISO-8859-8-I", "ISO-8859-8-I"); |
| |
| int32_t numEncodings = ucnv_countAvailable(); |
| for (int32_t i = 0; i < numEncodings; ++i) { |
| const char* name = ucnv_getAvailableName(i); |
| UErrorCode error = U_ZERO_ERROR; |
| // Try MIME before trying IANA to pick up commonly used names like |
| // 'EUC-JP' instead of horrendously long names like |
| // 'Extended_UNIX_Code_Packed_Format_for_Japanese'. |
| const char* standardName = ucnv_getStandardName(name, "MIME", &error); |
| if (!U_SUCCESS(error) || !standardName) { |
| error = U_ZERO_ERROR; |
| // Try IANA to pick up 'windows-12xx' and other names |
| // which are not preferred MIME names but are widely used. |
| standardName = ucnv_getStandardName(name, "IANA", &error); |
| if (!U_SUCCESS(error) || !standardName) |
| continue; |
| } |
| |
| // 1. Treat GB2312 encoding as GBK (its more modern superset), to match other browsers. |
| // 2. On the Web, GB2312 is encoded as EUC-CN or HZ, while ICU provides a native encoding |
| // for encoding GB_2312-80 and several others. So, we need to override this behavior, too. |
| if (strcmp(standardName, "GB2312") == 0 || strcmp(standardName, "GB_2312-80") == 0) |
| standardName = "GBK"; |
| // Similarly, EUC-KR encodings all map to an extended version. |
| else if (strcmp(standardName, "KSC_5601") == 0 || strcmp(standardName, "EUC-KR") == 0 || strcmp(standardName, "cp1363") == 0) |
| standardName = "windows-949"; |
| // And so on. |
| else if (strcasecmp(standardName, "iso-8859-9") == 0) // This name is returned in different case by ICU 3.2 and 3.6. |
| standardName = "windows-1254"; |
| else if (strcmp(standardName, "TIS-620") == 0) |
| standardName = "windows-874"; |
| |
| registrar(standardName, standardName); |
| |
| uint16_t numAliases = ucnv_countAliases(name, &error); |
| ASSERT(U_SUCCESS(error)); |
| if (U_SUCCESS(error)) |
| for (uint16_t j = 0; j < numAliases; ++j) { |
| error = U_ZERO_ERROR; |
| const char* alias = ucnv_getAlias(name, j, &error); |
| ASSERT(U_SUCCESS(error)); |
| if (U_SUCCESS(error) && alias != standardName) |
| registrar(alias, standardName); |
| } |
| } |
| |
| // Additional aliases. |
| // These are present in modern versions of ICU, but not in ICU 3.2 (shipped with Mac OS X 10.4). |
| registrar("macroman", "macintosh"); |
| registrar("maccyrillic", "x-mac-cyrillic"); |
| |
| // Additional aliases that historically were present in the encoding |
| // table in WebKit on Macintosh that don't seem to be present in ICU. |
| // Perhaps we can prove these are not used on the web and remove them. |
| // Or perhaps we can get them added to ICU. |
| registrar("xmacroman", "macintosh"); |
| registrar("xmacukrainian", "x-mac-cyrillic"); |
| registrar("cnbig5", "Big5"); |
| registrar("xxbig5", "Big5"); |
| registrar("cngb", "GBK"); |
| registrar("csgb231280", "GBK"); |
| registrar("xeuccn", "GBK"); |
| registrar("xgbk", "GBK"); |
| registrar("csISO88598I", "ISO_8859-8-I"); |
| registrar("koi", "KOI8-R"); |
| registrar("logical", "ISO-8859-8-I"); |
| registrar("unicode11utf8", "UTF-8"); |
| registrar("unicode20utf8", "UTF-8"); |
| registrar("xunicode20utf8", "UTF-8"); |
| registrar("visual", "ISO-8859-8"); |
| registrar("winarabic", "windows-1256"); |
| registrar("winbaltic", "windows-1257"); |
| registrar("wincyrillic", "windows-1251"); |
| registrar("iso885911", "windows-874"); |
| registrar("dos874", "windows-874"); |
| registrar("wingreek", "windows-1253"); |
| registrar("winhebrew", "windows-1255"); |
| registrar("winlatin2", "windows-1250"); |
| registrar("winturkish", "windows-1254"); |
| registrar("winvietnamese", "windows-1258"); |
| registrar("xcp1250", "windows-1250"); |
| registrar("xcp1251", "windows-1251"); |
| registrar("xeuc", "EUC-JP"); |
| registrar("xwindows949", "windows-949"); |
| registrar("xuhc", "windows-949"); |
| |
| // These aliases are present in modern versions of ICU, but use different codecs, and have no standard names. |
| // They are not present in ICU 3.2. |
| registrar("dos720", "cp864"); |
| registrar("jis7", "ISO-2022-JP"); |
| } |
| |
| void TextCodecICU::registerExtendedCodecs(TextCodecRegistrar registrar) |
| { |
| // See comment above in registerEncodingNames. |
| registrar("ISO-8859-8-I", newTextCodecICU, 0); |
| |
| int32_t numEncodings = ucnv_countAvailable(); |
| for (int32_t i = 0; i < numEncodings; ++i) { |
| const char* name = ucnv_getAvailableName(i); |
| UErrorCode error = U_ZERO_ERROR; |
| const char* standardName = ucnv_getStandardName(name, "MIME", &error); |
| if (!U_SUCCESS(error) || !standardName) { |
| error = U_ZERO_ERROR; |
| standardName = ucnv_getStandardName(name, "IANA", &error); |
| if (!U_SUCCESS(error) || !standardName) |
| continue; |
| } |
| registrar(standardName, newTextCodecICU, 0); |
| } |
| } |
| |
| TextCodecICU::TextCodecICU(const TextEncoding& encoding) |
| : m_encoding(encoding) |
| , m_numBufferedBytes(0) |
| , m_converterICU(0) |
| , m_needsGBKFallbacks(false) |
| { |
| } |
| |
| TextCodecICU::~TextCodecICU() |
| { |
| releaseICUConverter(); |
| } |
| |
| void TextCodecICU::releaseICUConverter() const |
| { |
| if (m_converterICU) { |
| UConverter*& cachedConverter = cachedConverterICU(); |
| if (cachedConverter) |
| ucnv_close(cachedConverter); |
| cachedConverter = m_converterICU; |
| m_converterICU = 0; |
| } |
| } |
| |
| void TextCodecICU::createICUConverter() const |
| { |
| ASSERT(!m_converterICU); |
| |
| const char* name = m_encoding.name(); |
| m_needsGBKFallbacks = name[0] == 'G' && name[1] == 'B' && name[2] == 'K' && !name[3]; |
| |
| UErrorCode err; |
| |
| UConverter*& cachedConverter = cachedConverterICU(); |
| if (cachedConverter) { |
| err = U_ZERO_ERROR; |
| const char* cachedName = ucnv_getName(cachedConverter, &err); |
| if (U_SUCCESS(err) && m_encoding == cachedName) { |
| m_converterICU = cachedConverter; |
| cachedConverter = 0; |
| return; |
| } |
| } |
| |
| err = U_ZERO_ERROR; |
| m_converterICU = ucnv_open(m_encoding.name(), &err); |
| #if !LOG_DISABLED |
| if (err == U_AMBIGUOUS_ALIAS_WARNING) |
| LOG_ERROR("ICU ambiguous alias warning for encoding: %s", m_encoding.name()); |
| #endif |
| if (m_converterICU) |
| ucnv_setFallback(m_converterICU, TRUE); |
| } |
| |
| int TextCodecICU::decodeToBuffer(UChar* target, UChar* targetLimit, const char*& source, const char* sourceLimit, int32_t* offsets, bool flush, UErrorCode& err) |
| { |
| UChar* targetStart = target; |
| err = U_ZERO_ERROR; |
| ucnv_toUnicode(m_converterICU, &target, targetLimit, &source, sourceLimit, offsets, flush, &err); |
| return target - targetStart; |
| } |
| |
| class ErrorCallbackSetter { |
| public: |
| ErrorCallbackSetter(UConverter* converter, bool stopOnError) |
| : m_converter(converter) |
| , m_shouldStopOnEncodingErrors(stopOnError) |
| { |
| if (m_shouldStopOnEncodingErrors) { |
| UErrorCode err = U_ZERO_ERROR; |
| ucnv_setToUCallBack(m_converter, UCNV_TO_U_CALLBACK_SUBSTITUTE, |
| UCNV_SUB_STOP_ON_ILLEGAL, &m_savedAction, |
| &m_savedContext, &err); |
| ASSERT(err == U_ZERO_ERROR); |
| } |
| } |
| ~ErrorCallbackSetter() |
| { |
| if (m_shouldStopOnEncodingErrors) { |
| UErrorCode err = U_ZERO_ERROR; |
| const void* oldContext; |
| UConverterToUCallback oldAction; |
| ucnv_setToUCallBack(m_converter, m_savedAction, |
| m_savedContext, &oldAction, |
| &oldContext, &err); |
| ASSERT(oldAction == UCNV_TO_U_CALLBACK_SUBSTITUTE); |
| ASSERT(!strcmp(static_cast<const char*>(oldContext), UCNV_SUB_STOP_ON_ILLEGAL)); |
| ASSERT(err == U_ZERO_ERROR); |
| } |
| } |
| private: |
| UConverter* m_converter; |
| bool m_shouldStopOnEncodingErrors; |
| const void* m_savedContext; |
| UConverterToUCallback m_savedAction; |
| }; |
| |
| String TextCodecICU::decode(const char* bytes, size_t length, bool flush, bool stopOnError, bool& sawError) |
| { |
| // Get a converter for the passed-in encoding. |
| if (!m_converterICU) { |
| createICUConverter(); |
| ASSERT(m_converterICU); |
| if (!m_converterICU) { |
| LOG_ERROR("error creating ICU encoder even though encoding was in table"); |
| return String(); |
| } |
| } |
| |
| ErrorCallbackSetter callbackSetter(m_converterICU, stopOnError); |
| |
| Vector<UChar> result; |
| |
| UChar buffer[ConversionBufferSize]; |
| UChar* bufferLimit = buffer + ConversionBufferSize; |
| const char* source = reinterpret_cast<const char*>(bytes); |
| const char* sourceLimit = source + length; |
| int32_t* offsets = NULL; |
| UErrorCode err = U_ZERO_ERROR; |
| |
| do { |
| int ucharsDecoded = decodeToBuffer(buffer, bufferLimit, source, sourceLimit, offsets, flush, err); |
| result.append(buffer, ucharsDecoded); |
| } while (err == U_BUFFER_OVERFLOW_ERROR); |
| |
| if (U_FAILURE(err)) { |
| // flush the converter so it can be reused, and not be bothered by this error. |
| do { |
| decodeToBuffer(buffer, bufferLimit, source, sourceLimit, offsets, true, err); |
| } while (source < sourceLimit); |
| sawError = true; |
| } |
| |
| String resultString = String::adopt(result); |
| |
| // <http://bugs.webkit.org/show_bug.cgi?id=17014> |
| // Simplified Chinese pages use the code A3A0 to mean "full-width space", but ICU decodes it as U+E5E5. |
| if (strcmp(m_encoding.name(), "GBK") == 0 || strcasecmp(m_encoding.name(), "gb18030") == 0) |
| resultString.replace(0xE5E5, ideographicSpace); |
| |
| return resultString; |
| } |
| |
| // We need to apply these fallbacks ourselves as they are not currently supported by ICU and |
| // they were provided by the old TEC encoding path |
| // Needed to fix <rdar://problem/4708689> |
| static UChar getGbkEscape(UChar32 codePoint) |
| { |
| switch (codePoint) { |
| case 0x01F9: |
| return 0xE7C8; |
| case 0x1E3F: |
| return 0xE7C7; |
| case 0x22EF: |
| return 0x2026; |
| case 0x301C: |
| return 0xFF5E; |
| default: |
| return 0; |
| } |
| } |
| |
| // Invalid character handler when writing escaped entities for unrepresentable |
| // characters. See the declaration of TextCodec::encode for more. |
| static void urlEscapedEntityCallback(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length, |
| UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) |
| { |
| if (reason == UCNV_UNASSIGNED) { |
| *err = U_ZERO_ERROR; |
| |
| UnencodableReplacementArray entity; |
| int entityLen = TextCodec::getUnencodableReplacement(codePoint, URLEncodedEntitiesForUnencodables, entity); |
| ucnv_cbFromUWriteBytes(fromUArgs, entity, entityLen, 0, err); |
| } else |
| UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint, reason, err); |
| } |
| |
| // Substitutes special GBK characters, escaping all other unassigned entities. |
| static void gbkCallbackEscape(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length, |
| UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) |
| { |
| UChar outChar; |
| if (reason == UCNV_UNASSIGNED && (outChar = getGbkEscape(codePoint))) { |
| const UChar* source = &outChar; |
| *err = U_ZERO_ERROR; |
| ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); |
| return; |
| } |
| UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint, reason, err); |
| } |
| |
| // Combines both gbkUrlEscapedEntityCallback and GBK character substitution. |
| static void gbkUrlEscapedEntityCallack(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length, |
| UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) |
| { |
| if (reason == UCNV_UNASSIGNED) { |
| if (UChar outChar = getGbkEscape(codePoint)) { |
| const UChar* source = &outChar; |
| *err = U_ZERO_ERROR; |
| ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); |
| return; |
| } |
| urlEscapedEntityCallback(context, fromUArgs, codeUnits, length, codePoint, reason, err); |
| return; |
| } |
| UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint, reason, err); |
| } |
| |
| static void gbkCallbackSubstitute(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length, |
| UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err) |
| { |
| UChar outChar; |
| if (reason == UCNV_UNASSIGNED && (outChar = getGbkEscape(codePoint))) { |
| const UChar* source = &outChar; |
| *err = U_ZERO_ERROR; |
| ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err); |
| return; |
| } |
| UCNV_FROM_U_CALLBACK_SUBSTITUTE(context, fromUArgs, codeUnits, length, codePoint, reason, err); |
| } |
| |
| CString TextCodecICU::encode(const UChar* characters, size_t length, UnencodableHandling handling) |
| { |
| if (!length) |
| return ""; |
| |
| if (!m_converterICU) |
| createICUConverter(); |
| if (!m_converterICU) |
| return CString(); |
| |
| // FIXME: We should see if there is "force ASCII range" mode in ICU; |
| // until then, we change the backslash into a yen sign. |
| // Encoding will change the yen sign back into a backslash. |
| String copy(characters, length); |
| copy = m_encoding.displayString(copy.impl()); |
| |
| const UChar* source = copy.characters(); |
| const UChar* sourceLimit = source + copy.length(); |
| |
| UErrorCode err = U_ZERO_ERROR; |
| |
| switch (handling) { |
| case QuestionMarksForUnencodables: |
| ucnv_setSubstChars(m_converterICU, "?", 1, &err); |
| ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackSubstitute : UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0, 0, 0, &err); |
| break; |
| case EntitiesForUnencodables: |
| ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackEscape : UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC, 0, 0, &err); |
| break; |
| case URLEncodedEntitiesForUnencodables: |
| ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkUrlEscapedEntityCallack : urlEscapedEntityCallback, 0, 0, 0, &err); |
| break; |
| } |
| |
| ASSERT(U_SUCCESS(err)); |
| if (U_FAILURE(err)) |
| return CString(); |
| |
| Vector<char> result; |
| size_t size = 0; |
| do { |
| char buffer[ConversionBufferSize]; |
| char* target = buffer; |
| char* targetLimit = target + ConversionBufferSize; |
| err = U_ZERO_ERROR; |
| ucnv_fromUnicode(m_converterICU, &target, targetLimit, &source, sourceLimit, 0, true, &err); |
| size_t count = target - buffer; |
| result.grow(size + count); |
| memcpy(result.data() + size, buffer, count); |
| size += count; |
| } while (err == U_BUFFER_OVERFLOW_ERROR); |
| |
| return CString(result.data(), size); |
| } |
| |
| |
| } // namespace WebCore |
