runtime/utf.cc - platform/art - Git at Google

 /*
  * Copyright (C) 2011 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include "utf.h"

 #include "base/logging.h"
 #include "mirror/array.h"
 #include "mirror/object-inl.h"
 #include "utf-inl.h"

 namespace art {

 // This is used only from debugger and test code.
 size_t CountModifiedUtf8Chars(const char* utf8) {
   return CountModifiedUtf8Chars(utf8, strlen(utf8));
 }

 /*
  * This does not validate UTF8 rules (nor did older code). But it gets the right answer
  * for valid UTF-8 and that's fine because it's used only to size a buffer for later
  * conversion.
  *
  * Modified UTF-8 consists of a series of bytes up to 21 bit Unicode code points as follows:
  * U+0001  - U+007F   0xxxxxxx
  * U+0080  - U+07FF   110xxxxx 10xxxxxx
  * U+0800  - U+FFFF   1110xxxx 10xxxxxx 10xxxxxx
  * U+10000 - U+1FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
  *
  * U+0000 is encoded using the 2nd form to avoid nulls inside strings (this differs from
  * standard UTF-8).
  * The four byte encoding converts to two utf16 characters.
  */
 size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) {
   DCHECK_LE(byte_count, strlen(utf8));
   size_t len = 0;
   const char* end = utf8 + byte_count;
   for (; utf8 < end; ++utf8) {
     int ic = *utf8;
     len++;
     if (LIKELY((ic & 0x80) == 0)) {
       // One-byte encoding.
       continue;
     }
     // Two- or three-byte encoding.
     utf8++;
     if ((ic & 0x20) == 0) {
       // Two-byte encoding.
       continue;
     }
     utf8++;
     if ((ic & 0x10) == 0) {
       // Three-byte encoding.
       continue;
     }

     // Four-byte encoding: needs to be converted into a surrogate
     // pair.
     utf8++;
     len++;
   }
   return len;
 }

 // This is used only from debugger and test code.
 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, const char* utf8_data_in) {
   while (*utf8_data_in != '\0') {
     const uint32_t ch = GetUtf16FromUtf8(&utf8_data_in);
     const uint16_t leading = GetLeadingUtf16Char(ch);
     const uint16_t trailing = GetTrailingUtf16Char(ch);

     *utf16_data_out++ = leading;
     if (trailing != 0) {
       *utf16_data_out++ = trailing;
     }
   }
 }

 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, size_t out_chars,
                                 const char* utf8_data_in, size_t in_bytes) {
   const char *in_start = utf8_data_in;
   const char *in_end = utf8_data_in + in_bytes;
   uint16_t *out_p = utf16_data_out;

   if (LIKELY(out_chars == in_bytes)) {
     // Common case where all characters are ASCII.
     for (const char *p = in_start; p < in_end;) {
       // Safe even if char is signed because ASCII characters always have
       // the high bit cleared.
       *out_p++ = dchecked_integral_cast<uint16_t>(*p++);
     }
     return;
   }

   // String contains non-ASCII characters.
   for (const char *p = in_start; p < in_end;) {
     const uint32_t ch = GetUtf16FromUtf8(&p);
     const uint16_t leading = GetLeadingUtf16Char(ch);
     const uint16_t trailing = GetTrailingUtf16Char(ch);

     *out_p++ = leading;
     if (trailing != 0) {
       *out_p++ = trailing;
     }
   }
 }

 void ConvertUtf16ToModifiedUtf8(char* utf8_out, size_t byte_count,
                                 const uint16_t* utf16_in, size_t char_count) {
   if (LIKELY(byte_count == char_count)) {
     // Common case where all characters are ASCII.
     const uint16_t *utf16_end = utf16_in + char_count;
     for (const uint16_t *p = utf16_in; p < utf16_end;) {
       *utf8_out++ = dchecked_integral_cast<char>(*p++);
     }
     return;
   }

   // String contains non-ASCII characters.
   while (char_count--) {
     const uint16_t ch = *utf16_in++;
     if (ch > 0 && ch <= 0x7f) {
       *utf8_out++ = ch;
     } else {
       // Char_count == 0 here implies we've encountered an unpaired
       // surrogate and we have no choice but to encode it as 3-byte UTF
       // sequence. Note that unpaired surrogates can occur as a part of
       // "normal" operation.
       if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) {
         const uint16_t ch2 = *utf16_in;

         // Check if the other half of the pair is within the expected
         // range. If it isn't, we will have to emit both "halves" as
         // separate 3 byte sequences.
         if (ch2 >= 0xdc00 && ch2 <= 0xdfff) {
           utf16_in++;
           char_count--;
           const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00;
           *utf8_out++ = (code_point >> 18) | 0xf0;
           *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80;
           *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80;
           *utf8_out++ = (code_point & 0x3f) | 0x80;
           continue;
         }
       }

       if (ch > 0x07ff) {
         // Three byte encoding.
         *utf8_out++ = (ch >> 12) | 0xe0;
         *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80;
         *utf8_out++ = (ch & 0x3f) | 0x80;
       } else /*(ch > 0x7f || ch == 0)*/ {
         // Two byte encoding.
         *utf8_out++ = (ch >> 6) | 0xc0;
         *utf8_out++ = (ch & 0x3f) | 0x80;
       }
     }
   }
 }

 int32_t ComputeUtf16Hash(const uint16_t* chars, size_t char_count) {
   uint32_t hash = 0;
   while (char_count--) {
     hash = hash * 31 + *chars++;
   }
   return static_cast<int32_t>(hash);
 }

 uint32_t ComputeModifiedUtf8Hash(const char* chars) {
   uint32_t hash = 0;
   while (*chars != '\0') {
     hash = hash * 31 + *chars++;
   }
   return static_cast<int32_t>(hash);
 }

 int CompareModifiedUtf8ToUtf16AsCodePointValues(const char* utf8, const uint16_t* utf16,
                                                 size_t utf16_length) {
   for (;;) {
     if (*utf8 == '\0') {
       return (utf16_length == 0) ? 0 : -1;
     } else if (utf16_length == 0) {
       return 1;
     }

     const uint32_t pair = GetUtf16FromUtf8(&utf8);

     // First compare the leading utf16 char.
     const uint16_t lhs = GetLeadingUtf16Char(pair);
     const uint16_t rhs = *utf16++;
     --utf16_length;
     if (lhs != rhs) {
       return lhs > rhs ? 1 : -1;
     }

     // Then compare the trailing utf16 char. First check if there
     // are any characters left to consume.
     const uint16_t lhs2 = GetTrailingUtf16Char(pair);
     if (lhs2 != 0) {
       if (utf16_length == 0) {
         return 1;
       }

       const uint16_t rhs2 = *utf16++;
       --utf16_length;
       if (lhs2 != rhs2) {
         return lhs2 > rhs2 ? 1 : -1;
       }
     }
   }
 }

 size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) {
   size_t result = 0;
   const uint16_t *end = chars + char_count;
   while (chars < end) {
     const uint16_t ch = *chars++;
     if (LIKELY(ch != 0 && ch < 0x80)) {
       result++;
       continue;
     }
     if (ch < 0x800) {
       result += 2;
       continue;
     }
     if (ch >= 0xd800 && ch < 0xdc00) {
       if (chars < end) {
         const uint16_t ch2 = *chars;
         // If we find a properly paired surrogate, we emit it as a 4 byte
         // UTF sequence. If we find an unpaired leading or trailing surrogate,
         // we emit it as a 3 byte sequence like would have done earlier.
         if (ch2 >= 0xdc00 && ch2 < 0xe000) {
           chars++;
           result += 4;
           continue;
         }
       }
     }
     result += 3;
   }
   return result;
 }

 }  // namespace art
	/*
	* Copyright (C) 2011 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include "utf.h"

	#include "base/logging.h"
	#include "mirror/array.h"
	#include "mirror/object-inl.h"
	#include "utf-inl.h"

	namespace art {

	// This is used only from debugger and test code.
	size_t CountModifiedUtf8Chars(const char* utf8) {
	return CountModifiedUtf8Chars(utf8, strlen(utf8));
	}

	/*
	* This does not validate UTF8 rules (nor did older code). But it gets the right answer
	* for valid UTF-8 and that's fine because it's used only to size a buffer for later
	* conversion.
	*
	* Modified UTF-8 consists of a series of bytes up to 21 bit Unicode code points as follows:
	* U+0001 - U+007F 0xxxxxxx
	* U+0080 - U+07FF 110xxxxx 10xxxxxx
	* U+0800 - U+FFFF 1110xxxx 10xxxxxx 10xxxxxx
	* U+10000 - U+1FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
	*
	* U+0000 is encoded using the 2nd form to avoid nulls inside strings (this differs from
	* standard UTF-8).
	* The four byte encoding converts to two utf16 characters.
	*/
	size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) {
	DCHECK_LE(byte_count, strlen(utf8));
	size_t len = 0;
	const char* end = utf8 + byte_count;
	for (; utf8 < end; ++utf8) {
	int ic = *utf8;
	len++;
	if (LIKELY((ic & 0x80) == 0)) {
	// One-byte encoding.
	continue;
	}
	// Two- or three-byte encoding.
	utf8++;
	if ((ic & 0x20) == 0) {
	// Two-byte encoding.
	continue;
	}
	utf8++;
	if ((ic & 0x10) == 0) {
	// Three-byte encoding.
	continue;
	}

	// Four-byte encoding: needs to be converted into a surrogate
	// pair.
	utf8++;
	len++;
	}
	return len;
	}

	// This is used only from debugger and test code.
	void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, const char* utf8_data_in) {
	while (*utf8_data_in != '\0') {
	const uint32_t ch = GetUtf16FromUtf8(&utf8_data_in);
	const uint16_t leading = GetLeadingUtf16Char(ch);
	const uint16_t trailing = GetTrailingUtf16Char(ch);

	*utf16_data_out++ = leading;
	if (trailing != 0) {
	*utf16_data_out++ = trailing;
	}
	}
	}

	void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, size_t out_chars,
	const char* utf8_data_in, size_t in_bytes) {
	const char *in_start = utf8_data_in;
	const char *in_end = utf8_data_in + in_bytes;
	uint16_t *out_p = utf16_data_out;

	if (LIKELY(out_chars == in_bytes)) {
	// Common case where all characters are ASCII.
	for (const char *p = in_start; p < in_end;) {
	// Safe even if char is signed because ASCII characters always have
	// the high bit cleared.
	out_p++ = dchecked_integral_cast<uint16_t>(p++);
	}
	return;
	}

	// String contains non-ASCII characters.
	for (const char *p = in_start; p < in_end;) {
	const uint32_t ch = GetUtf16FromUtf8(&p);
	const uint16_t leading = GetLeadingUtf16Char(ch);
	const uint16_t trailing = GetTrailingUtf16Char(ch);

	*out_p++ = leading;
	if (trailing != 0) {
	*out_p++ = trailing;
	}
	}
	}

	void ConvertUtf16ToModifiedUtf8(char* utf8_out, size_t byte_count,
	const uint16_t* utf16_in, size_t char_count) {
	if (LIKELY(byte_count == char_count)) {
	// Common case where all characters are ASCII.
	const uint16_t *utf16_end = utf16_in + char_count;
	for (const uint16_t *p = utf16_in; p < utf16_end;) {
	utf8_out++ = dchecked_integral_cast<char>(p++);
	}
	return;
	}

	// String contains non-ASCII characters.
	while (char_count--) {
	const uint16_t ch = *utf16_in++;
	if (ch > 0 && ch <= 0x7f) {
	*utf8_out++ = ch;
	} else {
	// Char_count == 0 here implies we've encountered an unpaired
	// surrogate and we have no choice but to encode it as 3-byte UTF
	// sequence. Note that unpaired surrogates can occur as a part of
	// "normal" operation.
	if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) {
	const uint16_t ch2 = *utf16_in;

	// Check if the other half of the pair is within the expected
	// range. If it isn't, we will have to emit both "halves" as
	// separate 3 byte sequences.
	if (ch2 >= 0xdc00 && ch2 <= 0xdfff) {
	utf16_in++;
	char_count--;
	const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00;
	*utf8_out++ = (code_point >> 18) \| 0xf0;
	*utf8_out++ = ((code_point >> 12) & 0x3f) \| 0x80;
	*utf8_out++ = ((code_point >> 6) & 0x3f) \| 0x80;
	*utf8_out++ = (code_point & 0x3f) \| 0x80;
	continue;
	}
	}

	if (ch > 0x07ff) {
	// Three byte encoding.
	*utf8_out++ = (ch >> 12) \| 0xe0;
	*utf8_out++ = ((ch >> 6) & 0x3f) \| 0x80;
	*utf8_out++ = (ch & 0x3f) \| 0x80;
	} else /(ch > 0x7f \|\| ch == 0)/ {
	// Two byte encoding.
	*utf8_out++ = (ch >> 6) \| 0xc0;
	*utf8_out++ = (ch & 0x3f) \| 0x80;
	}
	}
	}
	}

	int32_t ComputeUtf16Hash(const uint16_t* chars, size_t char_count) {
	uint32_t hash = 0;
	while (char_count--) {
	hash = hash * 31 + *chars++;
	}
	return static_cast<int32_t>(hash);
	}

	uint32_t ComputeModifiedUtf8Hash(const char* chars) {
	uint32_t hash = 0;
	while (*chars != '\0') {
	hash = hash * 31 + *chars++;
	}
	return static_cast<int32_t>(hash);
	}

	int CompareModifiedUtf8ToUtf16AsCodePointValues(const char* utf8, const uint16_t* utf16,
	size_t utf16_length) {
	for (;;) {
	if (*utf8 == '\0') {
	return (utf16_length == 0) ? 0 : -1;
	} else if (utf16_length == 0) {
	return 1;
	}

	const uint32_t pair = GetUtf16FromUtf8(&utf8);

	// First compare the leading utf16 char.
	const uint16_t lhs = GetLeadingUtf16Char(pair);
	const uint16_t rhs = *utf16++;
	--utf16_length;
	if (lhs != rhs) {
	return lhs > rhs ? 1 : -1;
	}

	// Then compare the trailing utf16 char. First check if there
	// are any characters left to consume.
	const uint16_t lhs2 = GetTrailingUtf16Char(pair);
	if (lhs2 != 0) {
	if (utf16_length == 0) {
	return 1;
	}

	const uint16_t rhs2 = *utf16++;
	--utf16_length;
	if (lhs2 != rhs2) {
	return lhs2 > rhs2 ? 1 : -1;
	}
	}
	}
	}

	size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) {
	size_t result = 0;
	const uint16_t *end = chars + char_count;
	while (chars < end) {
	const uint16_t ch = *chars++;
	if (LIKELY(ch != 0 && ch < 0x80)) {
	result++;
	continue;
	}
	if (ch < 0x800) {
	result += 2;
	continue;
	}
	if (ch >= 0xd800 && ch < 0xdc00) {
	if (chars < end) {
	const uint16_t ch2 = *chars;
	// If we find a properly paired surrogate, we emit it as a 4 byte
	// UTF sequence. If we find an unpaired leading or trailing surrogate,
	// we emit it as a 3 byte sequence like would have done earlier.
	if (ch2 >= 0xdc00 && ch2 < 0xe000) {
	chars++;
	result += 4;
	continue;
	}
	}
	}
	result += 3;
	}
	return result;
	}

	} // namespace art