using System; using System.Diagnostics; using UnityEngine.Assertions; using Unity.Collections.LowLevel.Unsafe; namespace Unity.Collections { /// /// Kinds of format errors. /// public enum FormatError { /// /// No error. /// None, /// /// The target storage does not have sufficient capacity. /// Overflow, } /// /// Kinds of parse errors. /// public enum ParseError { /// /// No parse error. /// None, /// /// The text parsed does not form a number. /// Syntax, /// /// The number exceeds the range of the target type. /// Overflow, /// /// The number exceeds the precision of the target type. /// Underflow, } /// /// Kinds of copy errors. /// public enum CopyError { /// /// No copy error. /// None, /// /// The target storage does not have sufficient capacity. /// Truncation, } /// /// Kinds of conversion errors. /// public enum ConversionError { /// /// No conversion error. /// None, /// /// The target storage does not have sufficient capacity. /// Overflow, /// /// The bytes do not form a valid character. /// Encoding, /// /// The rune is not a valid code point. /// CodePoint, } /// /// Provides utility methods for UTF-8, UTF-16, UCS-4 (a.k.a. UTF-32), and WTF-8. /// [BurstCompatible] public unsafe struct Unicode { /// /// Representation of a Unicode character as a code point. /// [BurstCompatible] public struct Rune { /// /// The code point. /// /// The code point. public int value; /// /// Initializes and returns an instance of Rune. /// /// You are responsible for the code point being valid. /// The code point. public Rune(int codepoint) { value = codepoint; } /// /// Returns a rune. /// /// Because a char is 16-bit, it can only represent the first 2^16 code points, not all 1.1 million. /// A code point. /// A rune. public static explicit operator Rune(char codepoint) => new Rune { value = codepoint }; /// /// Returns true if a rune is a numerical digit character. /// /// The rune. /// True if the rune is a numerical digit character. public static bool IsDigit(Rune r) { return r.value >= '0' && r.value <= '9'; } /// /// Returns the number of bytes required to encode this rune as UTF-8. /// /// The number of bytes required to encode this rune as UTF-8. If the rune's codepoint /// is invalid, returns 4 (the maximum possible encoding length). public int LengthInUtf8Bytes() { if (value < 0) return 4; // invalid codepoint if (value <= 0x7F) return 1; if (value <= 0x7FF) return 2; if (value <= 0xFFFF) return 3; if (value <= 0x1FFFFF) return 4; // invalid codepoint, max size. return 4; } } /// The maximum value of a valid UNICODE code point public const int kMaximumValidCodePoint = 0x10FFFF; /// /// Returns true if a code point is valid. /// /// A code point. /// True if a code point is valid. public static bool IsValidCodePoint(int codepoint) { if (codepoint > kMaximumValidCodePoint) // maximum valid code point return false; // if (codepoint >= 0xD800 && codepoint <= 0xDFFF) // surrogate pair // return false; if (codepoint < 0) // negative? return false; return true; } /// /// Returns true if the byte is not the last byte of a UTF-8 character. /// /// The byte. /// True if the byte is not the last byte of a UTF-8 character. public static bool NotTrailer(byte b) { return (b & 0xC0) != 0x80; } /// /// The Unicode character �. /// /// This character is used to stand-in for characters that can't be rendered. /// The Unicode character �. public static Rune ReplacementCharacter => new Rune { value = 0xFFFD }; /// /// The null rune value. /// /// In this package, the "bad rune" is used as a null character. It represents no valid code point. /// The null rune value. public static Rune BadRune => new Rune { value = 0 }; /// /// Reads a UTF-8 encoded character from a buffer. /// /// Outputs the character read. If the read fails, outputs . /// The buffer of bytes to read. /// Reference to a byte index into the buffer. If the read succeeds, index is incremented by the /// size in bytes of the character read. If the read fails, index is incremented by 1. /// The size in bytes of the buffer. Used to check that the read is in bounds. /// if the read succeeds. Otherwise, returns or . public static ConversionError Utf8ToUcs(out Rune rune, byte* buffer, ref int index, int capacity) { int code = 0; rune = ReplacementCharacter; if (index + 1 > capacity) { return ConversionError.Overflow; } if ((buffer[index] & 0b10000000) == 0b00000000) // if high bit is 0, 1 byte { rune.value = buffer[index + 0]; index += 1; return ConversionError.None; } if ((buffer[index] & 0b11100000) == 0b11000000) // if high 3 bits are 110, 2 bytes { if (index + 2 > capacity) { index += 1; return ConversionError.Overflow; } code = (buffer[index + 0] & 0b00011111); code = (code << 6) | (buffer[index + 1] & 0b00111111); if (code < (1 << 7) || NotTrailer(buffer[index + 1])) { index += 1; return ConversionError.Encoding; } rune.value = code; index += 2; return ConversionError.None; } if ((buffer[index] & 0b11110000) == 0b11100000) // if high 4 bits are 1110, 3 bytes { if (index + 3 > capacity) { index += 1; return ConversionError.Overflow; } code = (buffer[index + 0] & 0b00001111); code = (code << 6) | (buffer[index + 1] & 0b00111111); code = (code << 6) | (buffer[index + 2] & 0b00111111); if (code < (1 << 11) || !IsValidCodePoint(code) || NotTrailer(buffer[index + 1]) || NotTrailer(buffer[index + 2])) { index += 1; return ConversionError.Encoding; } rune.value = code; index += 3; return ConversionError.None; } if ((buffer[index] & 0b11111000) == 0b11110000) // if high 5 bits are 11110, 4 bytes { if (index + 4 > capacity) { index += 1; return ConversionError.Overflow; } code = (buffer[index + 0] & 0b00000111); code = (code << 6) | (buffer[index + 1] & 0b00111111); code = (code << 6) | (buffer[index + 2] & 0b00111111); code = (code << 6) | (buffer[index + 3] & 0b00111111); if (code < (1 << 16) || !IsValidCodePoint(code) || NotTrailer(buffer[index + 1]) || NotTrailer(buffer[index + 2]) || NotTrailer(buffer[index + 3])) { index += 1; return ConversionError.Encoding; } rune.value = code; index += 4; return ConversionError.None; } index += 1; return ConversionError.Encoding; } /// /// Returns true if a char is a Unicode leading surrogate. /// /// The char. /// True if the char is a Unicode leading surrogate. static bool IsLeadingSurrogate(char c) { return c >= 0xD800 && c <= 0xDBFF; } /// /// Returns true if a char is a Unicode trailing surrogate. /// /// The char. /// True if the char is a Unicode trailing surrogate. static bool IsTrailingSurrogate(char c) { return c >= 0xDC00 && c <= 0xDFFF; } /// /// Reads a UTF-16 encoded character from a buffer. /// /// Outputs the character read. If the read fails, rune is not set. /// The buffer of chars to read. /// Reference to a char index into the buffer. If the read succeeds, index is incremented by the /// size in chars of the character read. If the read fails, index is not incremented. /// The size in chars of the buffer. Used to check that the read is in bounds. /// if the read succeeds. Otherwise, returns . public static ConversionError Utf16ToUcs(out Rune rune, char* buffer, ref int index, int capacity) { int code = 0; rune = ReplacementCharacter; if (index + 1 > capacity) return ConversionError.Overflow; if (!IsLeadingSurrogate(buffer[index]) || (index + 2 > capacity)) { rune.value = buffer[index]; index += 1; return ConversionError.None; } code = (buffer[index + 0] & 0x03FF); char next = buffer[index + 1]; if (!IsTrailingSurrogate(next)) { rune.value = buffer[index]; index += 1; return ConversionError.None; } code = (code << 10) | (buffer[index + 1] & 0x03FF); code += 0x10000; rune.value = code; index += 2; return ConversionError.None; } /// /// Writes a rune to a buffer as a UTF-8 encoded character. /// /// The rune to encode. /// The buffer to write to. /// Reference to a byte index into the buffer. If the write succeeds, index is incremented by the /// size in bytes of the character written. If the write fails, index is not incremented. /// The size in bytes of the buffer. Used to check that the write is in bounds. /// if the write succeeds. Otherwise, returns , , or . public static ConversionError UcsToUtf8(byte* buffer, ref int index, int capacity, Rune rune) { if (!IsValidCodePoint(rune.value)) { return ConversionError.CodePoint; } if (index + 1 > capacity) { return ConversionError.Overflow; } if (rune.value <= 0x7F) { buffer[index++] = (byte)rune.value; return ConversionError.None; } if (rune.value <= 0x7FF) { if (index + 2 > capacity) { return ConversionError.Overflow; } buffer[index++] = (byte)(0xC0 | (rune.value >> 6)); buffer[index++] = (byte)(0x80 | ((rune.value >> 0) & 0x3F)); return ConversionError.None; } if (rune.value <= 0xFFFF) { if (index + 3 > capacity) { return ConversionError.Overflow; } buffer[index++] = (byte)(0xE0 | (rune.value >> 12)); buffer[index++] = (byte)(0x80 | ((rune.value >> 6) & 0x3F)); buffer[index++] = (byte)(0x80 | ((rune.value >> 0) & 0x3F)); return ConversionError.None; } if (rune.value <= 0x1FFFFF) { if (index + 4 > capacity) { return ConversionError.Overflow; } buffer[index++] = (byte)(0xF0 | (rune.value >> 18)); buffer[index++] = (byte)(0x80 | ((rune.value >> 12) & 0x3F)); buffer[index++] = (byte)(0x80 | ((rune.value >> 6) & 0x3F)); buffer[index++] = (byte)(0x80 | ((rune.value >> 0) & 0x3F)); return ConversionError.None; } return ConversionError.Encoding; } /// /// Writes a rune to a buffer as a UTF-16 encoded character. /// /// The rune to encode. /// The buffer of chars to write to. /// Reference to a char index into the buffer. If the write succeeds, index is incremented by the /// size in chars of the character written. If the write fails, index is not incremented. /// The size in chars of the buffer. Used to check that the write is in bounds. /// if the write succeeds. Otherwise, returns , , or . public static ConversionError UcsToUtf16(char* buffer, ref int index, int capacity, Rune rune) { if (!IsValidCodePoint(rune.value)) { return ConversionError.CodePoint; } if (index + 1 > capacity) { return ConversionError.Overflow; } if (rune.value >= 0x10000) { if (index + 2 > capacity) { return ConversionError.Overflow; } int code = rune.value - 0x10000; if (code >= (1 << 20)) { return ConversionError.Encoding; } buffer[index++] = (char)(0xD800 | (code >> 10)); buffer[index++] = (char)(0xDC00 | (code & 0x3FF)); return ConversionError.None; } buffer[index++] = (char)rune.value; return ConversionError.None; } /// /// Copies UTF-16 characters from one buffer to another buffer as UTF-8. /// /// Assumes the source data is valid UTF-16. /// The source buffer. /// The number of chars to read from the source. /// The destination buffer. /// Outputs the number of bytes written to the destination. /// The size in bytes of the destination buffer. /// if the copy fully completes. Otherwise, returns . public static ConversionError Utf16ToUtf8(char* utf16Buffer, int utf16Length, byte* utf8Buffer, out int utf8Length, int utf8Capacity) { utf8Length = 0; for (var utf16Offset = 0; utf16Offset < utf16Length;) { Utf16ToUcs(out var ucs, utf16Buffer, ref utf16Offset, utf16Length); if (UcsToUtf8(utf8Buffer, ref utf8Length, utf8Capacity, ucs) == ConversionError.Overflow) return ConversionError.Overflow; } return ConversionError.None; } /// /// Copies UTF-8 characters from one buffer to another. /// /// Assumes the source data is valid UTF-8. /// The source buffer. /// The number of bytes to read from the source. /// The destination buffer. /// Outputs the number of bytes written to the destination. /// The size in bytes of the destination buffer. /// if the copy fully completes. Otherwise, returns . public static ConversionError Utf8ToUtf8(byte* srcBuffer, int srcLength, byte* destBuffer, out int destLength, int destCapacity) { if (destCapacity >= srcLength) { UnsafeUtility.MemCpy(destBuffer, srcBuffer, srcLength); destLength = srcLength; return ConversionError.None; } // TODO even in this case, it's possible to MemCpy all but the last 3 bytes that fit, and then by looking at only // TODO the high bits of the last 3 bytes that fit, decide how many of the 3 to append. but that requires a // TODO little UNICODE presence of mind that nobody has today. destLength = 0; for (var srcOffset = 0; srcOffset < srcLength;) { Utf8ToUcs(out var ucs, srcBuffer, ref srcOffset, srcLength); if (UcsToUtf8(destBuffer, ref destLength, destCapacity, ucs) == ConversionError.Overflow) return ConversionError.Overflow; } return ConversionError.None; } /// /// Copies UTF-8 characters from one buffer to another as UTF-16. /// /// Assumes the source data is valid UTF-8. /// The source buffer. /// The number of bytes to read from the source. /// The destination buffer. /// Outputs the number of chars written to the destination. /// The size in chars of the destination buffer. /// if the copy fully completes. Otherwise, . public static ConversionError Utf8ToUtf16(byte* utf8Buffer, int utf8Length, char* utf16Buffer, out int utf16Length, int utf16Capacity) { utf16Length = 0; for (var utf8Offset = 0; utf8Offset < utf8Length;) { Utf8ToUcs(out var ucs, utf8Buffer, ref utf8Offset, utf8Length); if (UcsToUtf16(utf16Buffer, ref utf16Length, utf16Capacity, ucs) == ConversionError.Overflow) return ConversionError.Overflow; } return ConversionError.None; } } }