| // Protocol Buffers - Google's data interchange format |
| // Copyright 2008 Google Inc. All rights reserved. |
| // https://developers.google.com/protocol-buffers/ |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are |
| // met: |
| // |
| // * Redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer. |
| // * Redistributions in binary form must reproduce the above |
| // copyright notice, this list of conditions and the following disclaimer |
| // in the documentation and/or other materials provided with the |
| // distribution. |
| // * Neither the name of Google Inc. nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| package com.google.protobuf; |
| |
| import static com.google.protobuf.UnsafeUtil.addressOffset; |
| import static com.google.protobuf.UnsafeUtil.hasUnsafeArrayOperations; |
| import static com.google.protobuf.UnsafeUtil.hasUnsafeByteBufferOperations; |
| import static java.lang.Character.MAX_SURROGATE; |
| import static java.lang.Character.MIN_HIGH_SURROGATE; |
| import static java.lang.Character.MIN_LOW_SURROGATE; |
| import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT; |
| import static java.lang.Character.MIN_SURROGATE; |
| import static java.lang.Character.isSurrogatePair; |
| import static java.lang.Character.toCodePoint; |
| |
| import java.nio.ByteBuffer; |
| |
| /** |
| * A set of low-level, high-performance static utility methods related to the UTF-8 character |
| * encoding. This class has no dependencies outside of the core JDK libraries. |
| * |
| * <p>There are several variants of UTF-8. The one implemented by this class is the restricted |
| * definition of UTF-8 introduced in Unicode 3.1, which mandates the rejection of "overlong" byte |
| * sequences as well as rejection of 3-byte surrogate codepoint byte sequences. Note that the UTF-8 |
| * decoder included in Oracle's JDK has been modified to also reject "overlong" byte sequences, but |
| * (as of 2011) still accepts 3-byte surrogate codepoint byte sequences. |
| * |
| * <p>The byte sequences considered valid by this class are exactly those that can be roundtrip |
| * converted to Strings and back to bytes using the UTF-8 charset, without loss: |
| * |
| * <pre>{@code |
| * Arrays.equals(bytes, new String(bytes, Internal.UTF_8).getBytes(Internal.UTF_8)) |
| * }</pre> |
| * |
| * <p>See the Unicode Standard,</br> Table 3-6. <em>UTF-8 Bit Distribution</em>,</br> Table 3-7. |
| * <em>Well Formed UTF-8 Byte Sequences</em>. |
| * |
| * <p>This class supports decoding of partial byte sequences, so that the bytes in a complete UTF-8 |
| * byte sequences can be stored in multiple segments. Methods typically return {@link #MALFORMED} if |
| * the partial byte sequence is definitely not well-formed, {@link #COMPLETE} if it is well-formed |
| * in the absence of additional input, or if the byte sequence apparently terminated in the middle |
| * of a character, an opaque integer "state" value containing enough information to decode the |
| * character when passed to a subsequent invocation of a partial decoding method. |
| * |
| * @author martinrb@google.com (Martin Buchholz) |
| */ |
| // TODO(nathanmittler): Copy changes in this class back to Guava |
| final class Utf8 { |
| |
| /** |
| * UTF-8 is a runtime hot spot so we attempt to provide heavily optimized implementations |
| * depending on what is available on the platform. The processor is the platform-optimized |
| * delegate for which all methods are delegated directly to. |
| */ |
| private static final Processor processor = |
| (UnsafeProcessor.isAvailable() && !Android.isOnAndroidDevice()) |
| ? new UnsafeProcessor() |
| : new SafeProcessor(); |
| |
| /** |
| * A mask used when performing unsafe reads to determine if a long value contains any non-ASCII |
| * characters (i.e. any byte >= 0x80). |
| */ |
| private static final long ASCII_MASK_LONG = 0x8080808080808080L; |
| |
| /** |
| * Maximum number of bytes per Java UTF-16 char in UTF-8. |
| * |
| * @see java.nio.charset.CharsetEncoder#maxBytesPerChar() |
| */ |
| static final int MAX_BYTES_PER_CHAR = 3; |
| |
| /** |
| * State value indicating that the byte sequence is well-formed and complete (no further bytes are |
| * needed to complete a character). |
| */ |
| public static final int COMPLETE = 0; |
| |
| /** State value indicating that the byte sequence is definitely not well-formed. */ |
| public static final int MALFORMED = -1; |
| |
| /** |
| * Used by {@code Unsafe} UTF-8 string validation logic to determine the minimum string length |
| * above which to employ an optimized algorithm for counting ASCII characters. The reason for this |
| * threshold is that for small strings, the optimization may not be beneficial or may even |
| * negatively impact performance since it requires additional logic to avoid unaligned reads (when |
| * calling {@code Unsafe.getLong}). This threshold guarantees that even if the initial offset is |
| * unaligned, we're guaranteed to make at least one call to {@code Unsafe.getLong()} which |
| * provides a performance improvement that entirely subsumes the cost of the additional logic. |
| */ |
| private static final int UNSAFE_COUNT_ASCII_THRESHOLD = 16; |
| |
| // Other state values include the partial bytes of the incomplete |
| // character to be decoded in the simplest way: we pack the bytes |
| // into the state int in little-endian order. For example: |
| // |
| // int state = byte1 ^ (byte2 << 8) ^ (byte3 << 16); |
| // |
| // Such a state is unpacked thus (note the ~ operation for byte2 to |
| // undo byte1's sign-extension bits): |
| // |
| // int byte1 = (byte) state; |
| // int byte2 = (byte) ~(state >> 8); |
| // int byte3 = (byte) (state >> 16); |
| // |
| // We cannot store a zero byte in the state because it would be |
| // indistinguishable from the absence of a byte. But we don't need |
| // to, because partial bytes must always be negative. When building |
| // a state, we ensure that byte1 is negative and subsequent bytes |
| // are valid trailing bytes. |
| |
| /** |
| * Returns {@code true} if the given byte array is a well-formed UTF-8 byte sequence. |
| * |
| * <p>This is a convenience method, equivalent to a call to {@code isValidUtf8(bytes, 0, |
| * bytes.length)}. |
| */ |
| public static boolean isValidUtf8(byte[] bytes) { |
| return processor.isValidUtf8(bytes, 0, bytes.length); |
| } |
| |
| /** |
| * Returns {@code true} if the given byte array slice is a well-formed UTF-8 byte sequence. The |
| * range of bytes to be checked extends from index {@code index}, inclusive, to {@code limit}, |
| * exclusive. |
| * |
| * <p>This is a convenience method, equivalent to {@code partialIsValidUtf8(bytes, index, limit) |
| * == Utf8.COMPLETE}. |
| */ |
| public static boolean isValidUtf8(byte[] bytes, int index, int limit) { |
| return processor.isValidUtf8(bytes, index, limit); |
| } |
| |
| /** |
| * Tells whether the given byte array slice is a well-formed, malformed, or incomplete UTF-8 byte |
| * sequence. The range of bytes to be checked extends from index {@code index}, inclusive, to |
| * {@code limit}, exclusive. |
| * |
| * @param state either {@link Utf8#COMPLETE} (if this is the initial decoding operation) or the |
| * value returned from a call to a partial decoding method for the previous bytes |
| * @return {@link #MALFORMED} if the partial byte sequence is definitely not well-formed, {@link |
| * #COMPLETE} if it is well-formed (no additional input needed), or if the byte sequence is |
| * "incomplete", i.e. apparently terminated in the middle of a character, an opaque integer |
| * "state" value containing enough information to decode the character when passed to a |
| * subsequent invocation of a partial decoding method. |
| */ |
| public static int partialIsValidUtf8(int state, byte[] bytes, int index, int limit) { |
| return processor.partialIsValidUtf8(state, bytes, index, limit); |
| } |
| |
| private static int incompleteStateFor(int byte1) { |
| return (byte1 > (byte) 0xF4) ? MALFORMED : byte1; |
| } |
| |
| private static int incompleteStateFor(int byte1, int byte2) { |
| return (byte1 > (byte) 0xF4 || byte2 > (byte) 0xBF) ? MALFORMED : byte1 ^ (byte2 << 8); |
| } |
| |
| private static int incompleteStateFor(int byte1, int byte2, int byte3) { |
| return (byte1 > (byte) 0xF4 || byte2 > (byte) 0xBF || byte3 > (byte) 0xBF) |
| ? MALFORMED |
| : byte1 ^ (byte2 << 8) ^ (byte3 << 16); |
| } |
| |
| private static int incompleteStateFor(byte[] bytes, int index, int limit) { |
| int byte1 = bytes[index - 1]; |
| switch (limit - index) { |
| case 0: |
| return incompleteStateFor(byte1); |
| case 1: |
| return incompleteStateFor(byte1, bytes[index]); |
| case 2: |
| return incompleteStateFor(byte1, bytes[index], bytes[index + 1]); |
| default: |
| throw new AssertionError(); |
| } |
| } |
| |
| private static int incompleteStateFor( |
| final ByteBuffer buffer, final int byte1, final int index, final int remaining) { |
| switch (remaining) { |
| case 0: |
| return incompleteStateFor(byte1); |
| case 1: |
| return incompleteStateFor(byte1, buffer.get(index)); |
| case 2: |
| return incompleteStateFor(byte1, buffer.get(index), buffer.get(index + 1)); |
| default: |
| throw new AssertionError(); |
| } |
| } |
| |
| // These UTF-8 handling methods are copied from Guava's Utf8 class with a modification to throw |
| // a protocol buffer local exception. This exception is then caught in CodedOutputStream so it can |
| // fallback to more lenient behavior. |
| |
| static class UnpairedSurrogateException extends IllegalArgumentException { |
| UnpairedSurrogateException(int index, int length) { |
| super("Unpaired surrogate at index " + index + " of " + length); |
| } |
| } |
| |
| /** |
| * Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string, this |
| * method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in both |
| * time and space. |
| * |
| * @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired |
| * surrogates) |
| */ |
| static int encodedLength(CharSequence sequence) { |
| // Warning to maintainers: this implementation is highly optimized. |
| int utf16Length = sequence.length(); |
| int utf8Length = utf16Length; |
| int i = 0; |
| |
| // This loop optimizes for pure ASCII. |
| while (i < utf16Length && sequence.charAt(i) < 0x80) { |
| i++; |
| } |
| |
| // This loop optimizes for chars less than 0x800. |
| for (; i < utf16Length; i++) { |
| char c = sequence.charAt(i); |
| if (c < 0x800) { |
| utf8Length += ((0x7f - c) >>> 31); // branch free! |
| } else { |
| utf8Length += encodedLengthGeneral(sequence, i); |
| break; |
| } |
| } |
| |
| if (utf8Length < utf16Length) { |
| // Necessary and sufficient condition for overflow because of maximum 3x expansion |
| throw new IllegalArgumentException( |
| "UTF-8 length does not fit in int: " + (utf8Length + (1L << 32))); |
| } |
| return utf8Length; |
| } |
| |
| private static int encodedLengthGeneral(CharSequence sequence, int start) { |
| int utf16Length = sequence.length(); |
| int utf8Length = 0; |
| for (int i = start; i < utf16Length; i++) { |
| char c = sequence.charAt(i); |
| if (c < 0x800) { |
| utf8Length += (0x7f - c) >>> 31; // branch free! |
| } else { |
| utf8Length += 2; |
| // jdk7+: if (Character.isSurrogate(c)) { |
| if (Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) { |
| // Check that we have a well-formed surrogate pair. |
| int cp = Character.codePointAt(sequence, i); |
| if (cp < MIN_SUPPLEMENTARY_CODE_POINT) { |
| throw new UnpairedSurrogateException(i, utf16Length); |
| } |
| i++; |
| } |
| } |
| } |
| return utf8Length; |
| } |
| |
| static int encode(CharSequence in, byte[] out, int offset, int length) { |
| return processor.encodeUtf8(in, out, offset, length); |
| } |
| // End Guava UTF-8 methods. |
| |
| /** |
| * Determines if the given {@link ByteBuffer} is a valid UTF-8 string. |
| * |
| * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) |
| * and the capabilities of the platform. |
| * |
| * @param buffer the buffer to check. |
| * @see Utf8#isValidUtf8(byte[], int, int) |
| */ |
| static boolean isValidUtf8(ByteBuffer buffer) { |
| return processor.isValidUtf8(buffer, buffer.position(), buffer.remaining()); |
| } |
| |
| /** |
| * Determines if the given {@link ByteBuffer} is a partially valid UTF-8 string. |
| * |
| * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) |
| * and the capabilities of the platform. |
| * |
| * @param buffer the buffer to check. |
| * @see Utf8#partialIsValidUtf8(int, byte[], int, int) |
| */ |
| static int partialIsValidUtf8(int state, ByteBuffer buffer, int index, int limit) { |
| return processor.partialIsValidUtf8(state, buffer, index, limit); |
| } |
| |
| /** |
| * Decodes the given UTF-8 portion of the {@link ByteBuffer} into a {@link String}. |
| * |
| * @throws InvalidProtocolBufferException if the input is not valid UTF-8. |
| */ |
| static String decodeUtf8(ByteBuffer buffer, int index, int size) |
| throws InvalidProtocolBufferException { |
| return processor.decodeUtf8(buffer, index, size); |
| } |
| |
| /** |
| * Decodes the given UTF-8 encoded byte array slice into a {@link String}. |
| * |
| * @throws InvalidProtocolBufferException if the input is not valid UTF-8. |
| */ |
| static String decodeUtf8(byte[] bytes, int index, int size) |
| throws InvalidProtocolBufferException { |
| return processor.decodeUtf8(bytes, index, size); |
| } |
| |
| /** |
| * Encodes the given characters to the target {@link ByteBuffer} using UTF-8 encoding. |
| * |
| * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) |
| * and the capabilities of the platform. |
| * |
| * @param in the source string to be encoded |
| * @param out the target buffer to receive the encoded string. |
| * @see Utf8#encode(CharSequence, byte[], int, int) |
| */ |
| static void encodeUtf8(CharSequence in, ByteBuffer out) { |
| processor.encodeUtf8(in, out); |
| } |
| |
| /** |
| * Counts (approximately) the number of consecutive ASCII characters in the given buffer. The byte |
| * order of the {@link ByteBuffer} does not matter, so performance can be improved if native byte |
| * order is used (i.e. no byte-swapping in {@link ByteBuffer#getLong(int)}). |
| * |
| * @param buffer the buffer to be scanned for ASCII chars |
| * @param index the starting index of the scan |
| * @param limit the limit within buffer for the scan |
| * @return the number of ASCII characters found. The stopping position will be at or before the |
| * first non-ASCII byte. |
| */ |
| private static int estimateConsecutiveAscii(ByteBuffer buffer, int index, int limit) { |
| int i = index; |
| final int lim = limit - 7; |
| // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). |
| // To speed things up further, we're reading longs instead of bytes so we use a mask to |
| // determine if any byte in the current long is non-ASCII. |
| for (; i < lim && (buffer.getLong(i) & ASCII_MASK_LONG) == 0; i += 8) {} |
| return i - index; |
| } |
| |
| /** A processor of UTF-8 strings, providing methods for checking validity and encoding. */ |
| // TODO(nathanmittler): Add support for Memory/MemoryBlock on Android. |
| abstract static class Processor { |
| /** |
| * Returns {@code true} if the given byte array slice is a well-formed UTF-8 byte sequence. The |
| * range of bytes to be checked extends from index {@code index}, inclusive, to {@code limit}, |
| * exclusive. |
| * |
| * <p>This is a convenience method, equivalent to {@code partialIsValidUtf8(bytes, index, limit) |
| * == Utf8.COMPLETE}. |
| */ |
| final boolean isValidUtf8(byte[] bytes, int index, int limit) { |
| return partialIsValidUtf8(COMPLETE, bytes, index, limit) == COMPLETE; |
| } |
| |
| /** |
| * Tells whether the given byte array slice is a well-formed, malformed, or incomplete UTF-8 |
| * byte sequence. The range of bytes to be checked extends from index {@code index}, inclusive, |
| * to {@code limit}, exclusive. |
| * |
| * @param state either {@link Utf8#COMPLETE} (if this is the initial decoding operation) or the |
| * value returned from a call to a partial decoding method for the previous bytes |
| * @return {@link #MALFORMED} if the partial byte sequence is definitely not well-formed, {@link |
| * #COMPLETE} if it is well-formed (no additional input needed), or if the byte sequence is |
| * "incomplete", i.e. apparently terminated in the middle of a character, an opaque integer |
| * "state" value containing enough information to decode the character when passed to a |
| * subsequent invocation of a partial decoding method. |
| */ |
| abstract int partialIsValidUtf8(int state, byte[] bytes, int index, int limit); |
| |
| /** |
| * Returns {@code true} if the given portion of the {@link ByteBuffer} is a well-formed UTF-8 |
| * byte sequence. The range of bytes to be checked extends from index {@code index}, inclusive, |
| * to {@code limit}, exclusive. |
| * |
| * <p>This is a convenience method, equivalent to {@code partialIsValidUtf8(bytes, index, limit) |
| * == Utf8.COMPLETE}. |
| */ |
| final boolean isValidUtf8(ByteBuffer buffer, int index, int limit) { |
| return partialIsValidUtf8(COMPLETE, buffer, index, limit) == COMPLETE; |
| } |
| |
| /** |
| * Indicates whether or not the given buffer contains a valid UTF-8 string. |
| * |
| * @param buffer the buffer to check. |
| * @return {@code true} if the given buffer contains a valid UTF-8 string. |
| */ |
| final int partialIsValidUtf8( |
| final int state, final ByteBuffer buffer, int index, final int limit) { |
| if (buffer.hasArray()) { |
| final int offset = buffer.arrayOffset(); |
| return partialIsValidUtf8(state, buffer.array(), offset + index, offset + limit); |
| } else if (buffer.isDirect()) { |
| return partialIsValidUtf8Direct(state, buffer, index, limit); |
| } |
| return partialIsValidUtf8Default(state, buffer, index, limit); |
| } |
| |
| /** Performs validation for direct {@link ByteBuffer} instances. */ |
| abstract int partialIsValidUtf8Direct( |
| final int state, final ByteBuffer buffer, int index, final int limit); |
| |
| /** |
| * Performs validation for {@link ByteBuffer} instances using the {@link ByteBuffer} API rather |
| * than potentially faster approaches. This first completes validation for the current character |
| * (provided by {@code state}) and then finishes validation for the sequence. |
| */ |
| final int partialIsValidUtf8Default( |
| final int state, final ByteBuffer buffer, int index, final int limit) { |
| if (state != COMPLETE) { |
| // The previous decoding operation was incomplete (or malformed). |
| // We look for a well-formed sequence consisting of bytes from |
| // the previous decoding operation (stored in state) together |
| // with bytes from the array slice. |
| // |
| // We expect such "straddler characters" to be rare. |
| |
| if (index >= limit) { // No bytes? No progress. |
| return state; |
| } |
| |
| byte byte1 = (byte) state; |
| // byte1 is never ASCII. |
| if (byte1 < (byte) 0xE0) { |
| // two-byte form |
| |
| // Simultaneously checks for illegal trailing-byte in |
| // leading position and overlong 2-byte form. |
| if (byte1 < (byte) 0xC2 |
| // byte2 trailing-byte test |
| || buffer.get(index++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else if (byte1 < (byte) 0xF0) { |
| // three-byte form |
| |
| // Get byte2 from saved state or array |
| byte byte2 = (byte) ~(state >> 8); |
| if (byte2 == 0) { |
| byte2 = buffer.get(index++); |
| if (index >= limit) { |
| return incompleteStateFor(byte1, byte2); |
| } |
| } |
| if (byte2 > (byte) 0xBF |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // illegal surrogate codepoint? |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| // byte3 trailing-byte test |
| || buffer.get(index++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else { |
| // four-byte form |
| |
| // Get byte2 and byte3 from saved state or array |
| byte byte2 = (byte) ~(state >> 8); |
| byte byte3 = 0; |
| if (byte2 == 0) { |
| byte2 = buffer.get(index++); |
| if (index >= limit) { |
| return incompleteStateFor(byte1, byte2); |
| } |
| } else { |
| byte3 = (byte) (state >> 16); |
| } |
| if (byte3 == 0) { |
| byte3 = buffer.get(index++); |
| if (index >= limit) { |
| return incompleteStateFor(byte1, byte2, byte3); |
| } |
| } |
| |
| // If we were called with state == MALFORMED, then byte1 is 0xFF, |
| // which never occurs in well-formed UTF-8, and so we will return |
| // MALFORMED again below. |
| |
| if (byte2 > (byte) 0xBF |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // if (byte1 > (byte) 0xF4 || |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| // byte3 trailing-byte test |
| || byte3 > (byte) 0xBF |
| // byte4 trailing-byte test |
| || buffer.get(index++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } |
| } |
| |
| // Finish validation for the sequence. |
| return partialIsValidUtf8(buffer, index, limit); |
| } |
| |
| /** |
| * Performs validation for {@link ByteBuffer} instances using the {@link ByteBuffer} API rather |
| * than potentially faster approaches. |
| */ |
| private static int partialIsValidUtf8(final ByteBuffer buffer, int index, final int limit) { |
| index += estimateConsecutiveAscii(buffer, index, limit); |
| |
| for (; ; ) { |
| // Optimize for interior runs of ASCII bytes. |
| // TODO(nathanmittler): Consider checking 8 bytes at a time after some threshold? |
| // Maybe after seeing a few in a row that are ASCII, go back to fast mode? |
| int byte1; |
| do { |
| if (index >= limit) { |
| return COMPLETE; |
| } |
| } while ((byte1 = buffer.get(index++)) >= 0); |
| |
| // If we're here byte1 is not ASCII. Only need to handle 2-4 byte forms. |
| if (byte1 < (byte) 0xE0) { |
| // Two-byte form (110xxxxx 10xxxxxx) |
| if (index >= limit) { |
| // Incomplete sequence |
| return byte1; |
| } |
| |
| // Simultaneously checks for illegal trailing-byte in |
| // leading position and overlong 2-byte form. |
| if (byte1 < (byte) 0xC2 || buffer.get(index) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| index++; |
| } else if (byte1 < (byte) 0xF0) { |
| // Three-byte form (1110xxxx 10xxxxxx 10xxxxxx) |
| if (index >= limit - 1) { |
| // Incomplete sequence |
| return incompleteStateFor(buffer, byte1, index, limit - index); |
| } |
| |
| final byte byte2 = buffer.get(index++); |
| if (byte2 > (byte) 0xBF |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // check for illegal surrogate codepoints |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| // byte3 trailing-byte test |
| || buffer.get(index) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| index++; |
| } else { |
| // Four-byte form (1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx) |
| if (index >= limit - 2) { |
| // Incomplete sequence |
| return incompleteStateFor(buffer, byte1, index, limit - index); |
| } |
| |
| // TODO(nathanmittler): Consider using getInt() to improve performance. |
| final int byte2 = buffer.get(index++); |
| if (byte2 > (byte) 0xBF |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // if (byte1 > (byte) 0xF4 || |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| // byte3 trailing-byte test |
| || buffer.get(index++) > (byte) 0xBF |
| // byte4 trailing-byte test |
| || buffer.get(index++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } |
| } |
| } |
| |
| /** |
| * Decodes the given byte array slice into a {@link String}. |
| * |
| * @throws InvalidProtocolBufferException if the byte array slice is not valid UTF-8. |
| */ |
| abstract String decodeUtf8(byte[] bytes, int index, int size) |
| throws InvalidProtocolBufferException; |
| |
| /** |
| * Decodes the given portion of the {@link ByteBuffer} into a {@link String}. |
| * |
| * @throws InvalidProtocolBufferException if the portion of the buffer is not valid UTF-8. |
| */ |
| final String decodeUtf8(ByteBuffer buffer, int index, int size) |
| throws InvalidProtocolBufferException { |
| if (buffer.hasArray()) { |
| final int offset = buffer.arrayOffset(); |
| return decodeUtf8(buffer.array(), offset + index, size); |
| } else if (buffer.isDirect()) { |
| return decodeUtf8Direct(buffer, index, size); |
| } |
| return decodeUtf8Default(buffer, index, size); |
| } |
| |
| /** Decodes direct {@link ByteBuffer} instances into {@link String}. */ |
| abstract String decodeUtf8Direct(ByteBuffer buffer, int index, int size) |
| throws InvalidProtocolBufferException; |
| |
| /** |
| * Decodes {@link ByteBuffer} instances using the {@link ByteBuffer} API rather than potentially |
| * faster approaches. |
| */ |
| final String decodeUtf8Default(ByteBuffer buffer, int index, int size) |
| throws InvalidProtocolBufferException { |
| // Bitwise OR combines the sign bits so any negative value fails the check. |
| if ((index | size | buffer.limit() - index - size) < 0) { |
| throw new ArrayIndexOutOfBoundsException( |
| String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), index, size)); |
| } |
| |
| int offset = index; |
| final int limit = offset + size; |
| |
| // The longest possible resulting String is the same as the number of input bytes, when it is |
| // all ASCII. For other cases, this over-allocates and we will truncate in the end. |
| char[] resultArr = new char[size]; |
| int resultPos = 0; |
| |
| // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). |
| // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). |
| while (offset < limit) { |
| byte b = buffer.get(offset); |
| if (!DecodeUtil.isOneByte(b)) { |
| break; |
| } |
| offset++; |
| DecodeUtil.handleOneByte(b, resultArr, resultPos++); |
| } |
| |
| while (offset < limit) { |
| byte byte1 = buffer.get(offset++); |
| if (DecodeUtil.isOneByte(byte1)) { |
| DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); |
| // It's common for there to be multiple ASCII characters in a run mixed in, so add an |
| // extra optimized loop to take care of these runs. |
| while (offset < limit) { |
| byte b = buffer.get(offset); |
| if (!DecodeUtil.isOneByte(b)) { |
| break; |
| } |
| offset++; |
| DecodeUtil.handleOneByte(b, resultArr, resultPos++); |
| } |
| } else if (DecodeUtil.isTwoBytes(byte1)) { |
| if (offset >= limit) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleTwoBytes( |
| byte1, /* byte2 */ buffer.get(offset++), resultArr, resultPos++); |
| } else if (DecodeUtil.isThreeBytes(byte1)) { |
| if (offset >= limit - 1) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleThreeBytes( |
| byte1, |
| /* byte2 */ buffer.get(offset++), |
| /* byte3 */ buffer.get(offset++), |
| resultArr, |
| resultPos++); |
| } else { |
| if (offset >= limit - 2) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleFourBytes( |
| byte1, |
| /* byte2 */ buffer.get(offset++), |
| /* byte3 */ buffer.get(offset++), |
| /* byte4 */ buffer.get(offset++), |
| resultArr, |
| resultPos++); |
| // 4-byte case requires two chars. |
| resultPos++; |
| } |
| } |
| |
| return new String(resultArr, 0, resultPos); |
| } |
| |
| /** |
| * Encodes an input character sequence ({@code in}) to UTF-8 in the target array ({@code out}). |
| * For a string, this method is similar to |
| * |
| * <pre>{@code |
| * byte[] a = string.getBytes(UTF_8); |
| * System.arraycopy(a, 0, bytes, offset, a.length); |
| * return offset + a.length; |
| * }</pre> |
| * |
| * but is more efficient in both time and space. One key difference is that this method requires |
| * paired surrogates, and therefore does not support chunking. While {@code |
| * String.getBytes(UTF_8)} replaces unpaired surrogates with the default replacement character, |
| * this method throws {@link UnpairedSurrogateException}. |
| * |
| * <p>To ensure sufficient space in the output buffer, either call {@link #encodedLength} to |
| * compute the exact amount needed, or leave room for {@code Utf8.MAX_BYTES_PER_CHAR * |
| * sequence.length()}, which is the largest possible number of bytes that any input can be |
| * encoded to. |
| * |
| * @param in the input character sequence to be encoded |
| * @param out the target array |
| * @param offset the starting offset in {@code bytes} to start writing at |
| * @param length the length of the {@code bytes}, starting from {@code offset} |
| * @throws UnpairedSurrogateException if {@code sequence} contains ill-formed UTF-16 (unpaired |
| * surrogates) |
| * @throws ArrayIndexOutOfBoundsException if {@code sequence} encoded in UTF-8 is longer than |
| * {@code bytes.length - offset} |
| * @return the new offset, equivalent to {@code offset + Utf8.encodedLength(sequence)} |
| */ |
| abstract int encodeUtf8(CharSequence in, byte[] out, int offset, int length); |
| |
| /** |
| * Encodes an input character sequence ({@code in}) to UTF-8 in the target buffer ({@code out}). |
| * Upon returning from this method, the {@code out} position will point to the position after |
| * the last encoded byte. This method requires paired surrogates, and therefore does not support |
| * chunking. |
| * |
| * <p>To ensure sufficient space in the output buffer, either call {@link #encodedLength} to |
| * compute the exact amount needed, or leave room for {@code Utf8.MAX_BYTES_PER_CHAR * |
| * in.length()}, which is the largest possible number of bytes that any input can be encoded to. |
| * |
| * @param in the source character sequence to be encoded |
| * @param out the target buffer |
| * @throws UnpairedSurrogateException if {@code in} contains ill-formed UTF-16 (unpaired |
| * surrogates) |
| * @throws ArrayIndexOutOfBoundsException if {@code in} encoded in UTF-8 is longer than {@code |
| * out.remaining()} |
| */ |
| final void encodeUtf8(CharSequence in, ByteBuffer out) { |
| if (out.hasArray()) { |
| final int offset = out.arrayOffset(); |
| int endIndex = Utf8.encode(in, out.array(), offset + out.position(), out.remaining()); |
| out.position(endIndex - offset); |
| } else if (out.isDirect()) { |
| encodeUtf8Direct(in, out); |
| } else { |
| encodeUtf8Default(in, out); |
| } |
| } |
| |
| /** Encodes the input character sequence to a direct {@link ByteBuffer} instance. */ |
| abstract void encodeUtf8Direct(CharSequence in, ByteBuffer out); |
| |
| /** |
| * Encodes the input character sequence to a {@link ByteBuffer} instance using the {@link |
| * ByteBuffer} API, rather than potentially faster approaches. |
| */ |
| final void encodeUtf8Default(CharSequence in, ByteBuffer out) { |
| final int inLength = in.length(); |
| int outIx = out.position(); |
| int inIx = 0; |
| |
| // Since ByteBuffer.putXXX() already checks boundaries for us, no need to explicitly check |
| // access. Assume the buffer is big enough and let it handle the out of bounds exception |
| // if it occurs. |
| try { |
| // Designed to take advantage of |
| // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination |
| for (char c; inIx < inLength && (c = in.charAt(inIx)) < 0x80; ++inIx) { |
| out.put(outIx + inIx, (byte) c); |
| } |
| if (inIx == inLength) { |
| // Successfully encoded the entire string. |
| out.position(outIx + inIx); |
| return; |
| } |
| |
| outIx += inIx; |
| for (char c; inIx < inLength; ++inIx, ++outIx) { |
| c = in.charAt(inIx); |
| if (c < 0x80) { |
| // One byte (0xxx xxxx) |
| out.put(outIx, (byte) c); |
| } else if (c < 0x800) { |
| // Two bytes (110x xxxx 10xx xxxx) |
| |
| // Benchmarks show put performs better than putShort here (for HotSpot). |
| out.put(outIx++, (byte) (0xC0 | (c >>> 6))); |
| out.put(outIx, (byte) (0x80 | (0x3F & c))); |
| } else if (c < MIN_SURROGATE || MAX_SURROGATE < c) { |
| // Three bytes (1110 xxxx 10xx xxxx 10xx xxxx) |
| // Maximum single-char code point is 0xFFFF, 16 bits. |
| |
| // Benchmarks show put performs better than putShort here (for HotSpot). |
| out.put(outIx++, (byte) (0xE0 | (c >>> 12))); |
| out.put(outIx++, (byte) (0x80 | (0x3F & (c >>> 6)))); |
| out.put(outIx, (byte) (0x80 | (0x3F & c))); |
| } else { |
| // Four bytes (1111 xxxx 10xx xxxx 10xx xxxx 10xx xxxx) |
| |
| // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 |
| // bytes |
| final char low; |
| if (inIx + 1 == inLength || !isSurrogatePair(c, (low = in.charAt(++inIx)))) { |
| throw new UnpairedSurrogateException(inIx, inLength); |
| } |
| // TODO(nathanmittler): Consider using putInt() to improve performance. |
| int codePoint = toCodePoint(c, low); |
| out.put(outIx++, (byte) ((0xF << 4) | (codePoint >>> 18))); |
| out.put(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12)))); |
| out.put(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6)))); |
| out.put(outIx, (byte) (0x80 | (0x3F & codePoint))); |
| } |
| } |
| |
| // Successfully encoded the entire string. |
| out.position(outIx); |
| } catch (IndexOutOfBoundsException e) { |
| // TODO(nathanmittler): Consider making the API throw IndexOutOfBoundsException instead. |
| |
| // If we failed in the outer ASCII loop, outIx will not have been updated. In this case, |
| // use inIx to determine the bad write index. |
| int badWriteIndex = out.position() + Math.max(inIx, outIx - out.position() + 1); |
| throw new ArrayIndexOutOfBoundsException( |
| "Failed writing " + in.charAt(inIx) + " at index " + badWriteIndex); |
| } |
| } |
| } |
| |
| /** {@link Processor} implementation that does not use any {@code sun.misc.Unsafe} methods. */ |
| static final class SafeProcessor extends Processor { |
| @Override |
| int partialIsValidUtf8(int state, byte[] bytes, int index, int limit) { |
| if (state != COMPLETE) { |
| // The previous decoding operation was incomplete (or malformed). |
| // We look for a well-formed sequence consisting of bytes from |
| // the previous decoding operation (stored in state) together |
| // with bytes from the array slice. |
| // |
| // We expect such "straddler characters" to be rare. |
| |
| if (index >= limit) { // No bytes? No progress. |
| return state; |
| } |
| int byte1 = (byte) state; |
| // byte1 is never ASCII. |
| if (byte1 < (byte) 0xE0) { |
| // two-byte form |
| |
| // Simultaneously checks for illegal trailing-byte in |
| // leading position and overlong 2-byte form. |
| if (byte1 < (byte) 0xC2 |
| // byte2 trailing-byte test |
| || bytes[index++] > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else if (byte1 < (byte) 0xF0) { |
| // three-byte form |
| |
| // Get byte2 from saved state or array |
| int byte2 = (byte) ~(state >> 8); |
| if (byte2 == 0) { |
| byte2 = bytes[index++]; |
| if (index >= limit) { |
| return incompleteStateFor(byte1, byte2); |
| } |
| } |
| if (byte2 > (byte) 0xBF |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // illegal surrogate codepoint? |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| // byte3 trailing-byte test |
| || bytes[index++] > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else { |
| // four-byte form |
| |
| // Get byte2 and byte3 from saved state or array |
| int byte2 = (byte) ~(state >> 8); |
| int byte3 = 0; |
| if (byte2 == 0) { |
| byte2 = bytes[index++]; |
| if (index >= limit) { |
| return incompleteStateFor(byte1, byte2); |
| } |
| } else { |
| byte3 = (byte) (state >> 16); |
| } |
| if (byte3 == 0) { |
| byte3 = bytes[index++]; |
| if (index >= limit) { |
| return incompleteStateFor(byte1, byte2, byte3); |
| } |
| } |
| |
| // If we were called with state == MALFORMED, then byte1 is 0xFF, |
| // which never occurs in well-formed UTF-8, and so we will return |
| // MALFORMED again below. |
| |
| if (byte2 > (byte) 0xBF |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // if (byte1 > (byte) 0xF4 || |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| // byte3 trailing-byte test |
| || byte3 > (byte) 0xBF |
| // byte4 trailing-byte test |
| || bytes[index++] > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } |
| } |
| |
| return partialIsValidUtf8(bytes, index, limit); |
| } |
| |
| @Override |
| int partialIsValidUtf8Direct(int state, ByteBuffer buffer, int index, int limit) { |
| // For safe processing, we have to use the ByteBuffer API. |
| return partialIsValidUtf8Default(state, buffer, index, limit); |
| } |
| |
| @Override |
| String decodeUtf8(byte[] bytes, int index, int size) throws InvalidProtocolBufferException { |
| // Bitwise OR combines the sign bits so any negative value fails the check. |
| if ((index | size | bytes.length - index - size) < 0) { |
| throw new ArrayIndexOutOfBoundsException( |
| String.format("buffer length=%d, index=%d, size=%d", bytes.length, index, size)); |
| } |
| |
| int offset = index; |
| final int limit = offset + size; |
| |
| // The longest possible resulting String is the same as the number of input bytes, when it is |
| // all ASCII. For other cases, this over-allocates and we will truncate in the end. |
| char[] resultArr = new char[size]; |
| int resultPos = 0; |
| |
| // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). |
| // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). |
| while (offset < limit) { |
| byte b = bytes[offset]; |
| if (!DecodeUtil.isOneByte(b)) { |
| break; |
| } |
| offset++; |
| DecodeUtil.handleOneByte(b, resultArr, resultPos++); |
| } |
| |
| while (offset < limit) { |
| byte byte1 = bytes[offset++]; |
| if (DecodeUtil.isOneByte(byte1)) { |
| DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); |
| // It's common for there to be multiple ASCII characters in a run mixed in, so add an |
| // extra optimized loop to take care of these runs. |
| while (offset < limit) { |
| byte b = bytes[offset]; |
| if (!DecodeUtil.isOneByte(b)) { |
| break; |
| } |
| offset++; |
| DecodeUtil.handleOneByte(b, resultArr, resultPos++); |
| } |
| } else if (DecodeUtil.isTwoBytes(byte1)) { |
| if (offset >= limit) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleTwoBytes(byte1, /* byte2 */ bytes[offset++], resultArr, resultPos++); |
| } else if (DecodeUtil.isThreeBytes(byte1)) { |
| if (offset >= limit - 1) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleThreeBytes( |
| byte1, |
| /* byte2 */ bytes[offset++], |
| /* byte3 */ bytes[offset++], |
| resultArr, |
| resultPos++); |
| } else { |
| if (offset >= limit - 2) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleFourBytes( |
| byte1, |
| /* byte2 */ bytes[offset++], |
| /* byte3 */ bytes[offset++], |
| /* byte4 */ bytes[offset++], |
| resultArr, |
| resultPos++); |
| // 4-byte case requires two chars. |
| resultPos++; |
| } |
| } |
| |
| return new String(resultArr, 0, resultPos); |
| } |
| |
| @Override |
| String decodeUtf8Direct(ByteBuffer buffer, int index, int size) |
| throws InvalidProtocolBufferException { |
| // For safe processing, we have to use the ByteBufferAPI. |
| return decodeUtf8Default(buffer, index, size); |
| } |
| |
| @Override |
| int encodeUtf8(CharSequence in, byte[] out, int offset, int length) { |
| int utf16Length = in.length(); |
| int j = offset; |
| int i = 0; |
| int limit = offset + length; |
| // Designed to take advantage of |
| // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination |
| for (char c; i < utf16Length && i + j < limit && (c = in.charAt(i)) < 0x80; i++) { |
| out[j + i] = (byte) c; |
| } |
| if (i == utf16Length) { |
| return j + utf16Length; |
| } |
| j += i; |
| for (char c; i < utf16Length; i++) { |
| c = in.charAt(i); |
| if (c < 0x80 && j < limit) { |
| out[j++] = (byte) c; |
| } else if (c < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes |
| out[j++] = (byte) ((0xF << 6) | (c >>> 6)); |
| out[j++] = (byte) (0x80 | (0x3F & c)); |
| } else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) && j <= limit - 3) { |
| // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes |
| out[j++] = (byte) ((0xF << 5) | (c >>> 12)); |
| out[j++] = (byte) (0x80 | (0x3F & (c >>> 6))); |
| out[j++] = (byte) (0x80 | (0x3F & c)); |
| } else if (j <= limit - 4) { |
| // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, |
| // four UTF-8 bytes |
| final char low; |
| if (i + 1 == in.length() || !Character.isSurrogatePair(c, (low = in.charAt(++i)))) { |
| throw new UnpairedSurrogateException((i - 1), utf16Length); |
| } |
| int codePoint = Character.toCodePoint(c, low); |
| out[j++] = (byte) ((0xF << 4) | (codePoint >>> 18)); |
| out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 12))); |
| out[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 6))); |
| out[j++] = (byte) (0x80 | (0x3F & codePoint)); |
| } else { |
| // If we are surrogates and we're not a surrogate pair, always throw an |
| // UnpairedSurrogateException instead of an ArrayOutOfBoundsException. |
| if ((Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) |
| && (i + 1 == in.length() || !Character.isSurrogatePair(c, in.charAt(i + 1)))) { |
| throw new UnpairedSurrogateException(i, utf16Length); |
| } |
| throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + j); |
| } |
| } |
| return j; |
| } |
| |
| @Override |
| void encodeUtf8Direct(CharSequence in, ByteBuffer out) { |
| // For safe processing, we have to use the ByteBuffer API. |
| encodeUtf8Default(in, out); |
| } |
| |
| private static int partialIsValidUtf8(byte[] bytes, int index, int limit) { |
| // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). |
| // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). |
| while (index < limit && bytes[index] >= 0) { |
| index++; |
| } |
| |
| return (index >= limit) ? COMPLETE : partialIsValidUtf8NonAscii(bytes, index, limit); |
| } |
| |
| private static int partialIsValidUtf8NonAscii(byte[] bytes, int index, int limit) { |
| for (; ; ) { |
| int byte1; |
| int byte2; |
| |
| // Optimize for interior runs of ASCII bytes. |
| do { |
| if (index >= limit) { |
| return COMPLETE; |
| } |
| } while ((byte1 = bytes[index++]) >= 0); |
| |
| if (byte1 < (byte) 0xE0) { |
| // two-byte form |
| |
| if (index >= limit) { |
| // Incomplete sequence |
| return byte1; |
| } |
| |
| // Simultaneously checks for illegal trailing-byte in |
| // leading position and overlong 2-byte form. |
| if (byte1 < (byte) 0xC2 || bytes[index++] > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else if (byte1 < (byte) 0xF0) { |
| // three-byte form |
| |
| if (index >= limit - 1) { // incomplete sequence |
| return incompleteStateFor(bytes, index, limit); |
| } |
| if ((byte2 = bytes[index++]) > (byte) 0xBF |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // check for illegal surrogate codepoints |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| // byte3 trailing-byte test |
| || bytes[index++] > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else { |
| // four-byte form |
| |
| if (index >= limit - 2) { // incomplete sequence |
| return incompleteStateFor(bytes, index, limit); |
| } |
| if ((byte2 = bytes[index++]) > (byte) 0xBF |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // if (byte1 > (byte) 0xF4 || |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| // byte3 trailing-byte test |
| || bytes[index++] > (byte) 0xBF |
| // byte4 trailing-byte test |
| || bytes[index++] > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } |
| } |
| } |
| } |
| |
| /** {@link Processor} that uses {@code sun.misc.Unsafe} where possible to improve performance. */ |
| static final class UnsafeProcessor extends Processor { |
| /** Indicates whether or not all required unsafe operations are supported on this platform. */ |
| static boolean isAvailable() { |
| return hasUnsafeArrayOperations() && hasUnsafeByteBufferOperations(); |
| } |
| |
| @Override |
| int partialIsValidUtf8(int state, byte[] bytes, final int index, final int limit) { |
| // Bitwise OR combines the sign bits so any negative value fails the check. |
| if ((index | limit | bytes.length - limit) < 0) { |
| throw new ArrayIndexOutOfBoundsException( |
| String.format("Array length=%d, index=%d, limit=%d", bytes.length, index, limit)); |
| } |
| long offset = index; |
| final long offsetLimit = limit; |
| if (state != COMPLETE) { |
| // The previous decoding operation was incomplete (or malformed). |
| // We look for a well-formed sequence consisting of bytes from |
| // the previous decoding operation (stored in state) together |
| // with bytes from the array slice. |
| // |
| // We expect such "straddler characters" to be rare. |
| |
| if (offset >= offsetLimit) { // No bytes? No progress. |
| return state; |
| } |
| int byte1 = (byte) state; |
| // byte1 is never ASCII. |
| if (byte1 < (byte) 0xE0) { |
| // two-byte form |
| |
| // Simultaneously checks for illegal trailing-byte in |
| // leading position and overlong 2-byte form. |
| if (byte1 < (byte) 0xC2 |
| // byte2 trailing-byte test |
| || UnsafeUtil.getByte(bytes, offset++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else if (byte1 < (byte) 0xF0) { |
| // three-byte form |
| |
| // Get byte2 from saved state or array |
| int byte2 = (byte) ~(state >> 8); |
| if (byte2 == 0) { |
| byte2 = UnsafeUtil.getByte(bytes, offset++); |
| if (offset >= offsetLimit) { |
| return incompleteStateFor(byte1, byte2); |
| } |
| } |
| if (byte2 > (byte) 0xBF |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // illegal surrogate codepoint? |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| // byte3 trailing-byte test |
| || UnsafeUtil.getByte(bytes, offset++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else { |
| // four-byte form |
| |
| // Get byte2 and byte3 from saved state or array |
| int byte2 = (byte) ~(state >> 8); |
| int byte3 = 0; |
| if (byte2 == 0) { |
| byte2 = UnsafeUtil.getByte(bytes, offset++); |
| if (offset >= offsetLimit) { |
| return incompleteStateFor(byte1, byte2); |
| } |
| } else { |
| byte3 = (byte) (state >> 16); |
| } |
| if (byte3 == 0) { |
| byte3 = UnsafeUtil.getByte(bytes, offset++); |
| if (offset >= offsetLimit) { |
| return incompleteStateFor(byte1, byte2, byte3); |
| } |
| } |
| |
| // If we were called with state == MALFORMED, then byte1 is 0xFF, |
| // which never occurs in well-formed UTF-8, and so we will return |
| // MALFORMED again below. |
| |
| if (byte2 > (byte) 0xBF |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // if (byte1 > (byte) 0xF4 || |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| // byte3 trailing-byte test |
| || byte3 > (byte) 0xBF |
| // byte4 trailing-byte test |
| || UnsafeUtil.getByte(bytes, offset++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } |
| } |
| |
| return partialIsValidUtf8(bytes, offset, (int) (offsetLimit - offset)); |
| } |
| |
| @Override |
| int partialIsValidUtf8Direct( |
| final int state, ByteBuffer buffer, final int index, final int limit) { |
| // Bitwise OR combines the sign bits so any negative value fails the check. |
| if ((index | limit | buffer.limit() - limit) < 0) { |
| throw new ArrayIndexOutOfBoundsException( |
| String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), index, limit)); |
| } |
| long address = addressOffset(buffer) + index; |
| final long addressLimit = address + (limit - index); |
| if (state != COMPLETE) { |
| // The previous decoding operation was incomplete (or malformed). |
| // We look for a well-formed sequence consisting of bytes from |
| // the previous decoding operation (stored in state) together |
| // with bytes from the array slice. |
| // |
| // We expect such "straddler characters" to be rare. |
| |
| if (address >= addressLimit) { // No bytes? No progress. |
| return state; |
| } |
| |
| final int byte1 = (byte) state; |
| // byte1 is never ASCII. |
| if (byte1 < (byte) 0xE0) { |
| // two-byte form |
| |
| // Simultaneously checks for illegal trailing-byte in |
| // leading position and overlong 2-byte form. |
| if (byte1 < (byte) 0xC2 |
| // byte2 trailing-byte test |
| || UnsafeUtil.getByte(address++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else if (byte1 < (byte) 0xF0) { |
| // three-byte form |
| |
| // Get byte2 from saved state or array |
| int byte2 = (byte) ~(state >> 8); |
| if (byte2 == 0) { |
| byte2 = UnsafeUtil.getByte(address++); |
| if (address >= addressLimit) { |
| return incompleteStateFor(byte1, byte2); |
| } |
| } |
| if (byte2 > (byte) 0xBF |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // illegal surrogate codepoint? |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| // byte3 trailing-byte test |
| || UnsafeUtil.getByte(address++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else { |
| // four-byte form |
| |
| // Get byte2 and byte3 from saved state or array |
| int byte2 = (byte) ~(state >> 8); |
| int byte3 = 0; |
| if (byte2 == 0) { |
| byte2 = UnsafeUtil.getByte(address++); |
| if (address >= addressLimit) { |
| return incompleteStateFor(byte1, byte2); |
| } |
| } else { |
| byte3 = (byte) (state >> 16); |
| } |
| if (byte3 == 0) { |
| byte3 = UnsafeUtil.getByte(address++); |
| if (address >= addressLimit) { |
| return incompleteStateFor(byte1, byte2, byte3); |
| } |
| } |
| |
| // If we were called with state == MALFORMED, then byte1 is 0xFF, |
| // which never occurs in well-formed UTF-8, and so we will return |
| // MALFORMED again below. |
| |
| if (byte2 > (byte) 0xBF |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // if (byte1 > (byte) 0xF4 || |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| // byte3 trailing-byte test |
| || byte3 > (byte) 0xBF |
| // byte4 trailing-byte test |
| || UnsafeUtil.getByte(address++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } |
| } |
| |
| return partialIsValidUtf8(address, (int) (addressLimit - address)); |
| } |
| |
| @Override |
| String decodeUtf8(byte[] bytes, int index, int size) throws InvalidProtocolBufferException { |
| if ((index | size | bytes.length - index - size) < 0) { |
| throw new ArrayIndexOutOfBoundsException( |
| String.format("buffer length=%d, index=%d, size=%d", bytes.length, index, size)); |
| } |
| |
| int offset = index; |
| final int limit = offset + size; |
| |
| // The longest possible resulting String is the same as the number of input bytes, when it is |
| // all ASCII. For other cases, this over-allocates and we will truncate in the end. |
| char[] resultArr = new char[size]; |
| int resultPos = 0; |
| |
| // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). |
| // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). |
| while (offset < limit) { |
| byte b = UnsafeUtil.getByte(bytes, offset); |
| if (!DecodeUtil.isOneByte(b)) { |
| break; |
| } |
| offset++; |
| DecodeUtil.handleOneByte(b, resultArr, resultPos++); |
| } |
| |
| while (offset < limit) { |
| byte byte1 = UnsafeUtil.getByte(bytes, offset++); |
| if (DecodeUtil.isOneByte(byte1)) { |
| DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); |
| // It's common for there to be multiple ASCII characters in a run mixed in, so add an |
| // extra optimized loop to take care of these runs. |
| while (offset < limit) { |
| byte b = UnsafeUtil.getByte(bytes, offset); |
| if (!DecodeUtil.isOneByte(b)) { |
| break; |
| } |
| offset++; |
| DecodeUtil.handleOneByte(b, resultArr, resultPos++); |
| } |
| } else if (DecodeUtil.isTwoBytes(byte1)) { |
| if (offset >= limit) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleTwoBytes( |
| byte1, /* byte2 */ UnsafeUtil.getByte(bytes, offset++), resultArr, resultPos++); |
| } else if (DecodeUtil.isThreeBytes(byte1)) { |
| if (offset >= limit - 1) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleThreeBytes( |
| byte1, |
| /* byte2 */ UnsafeUtil.getByte(bytes, offset++), |
| /* byte3 */ UnsafeUtil.getByte(bytes, offset++), |
| resultArr, |
| resultPos++); |
| } else { |
| if (offset >= limit - 2) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleFourBytes( |
| byte1, |
| /* byte2 */ UnsafeUtil.getByte(bytes, offset++), |
| /* byte3 */ UnsafeUtil.getByte(bytes, offset++), |
| /* byte4 */ UnsafeUtil.getByte(bytes, offset++), |
| resultArr, |
| resultPos++); |
| // 4-byte case requires two chars. |
| resultPos++; |
| } |
| } |
| |
| return new String(resultArr, 0, resultPos); |
| } |
| |
| @Override |
| String decodeUtf8Direct(ByteBuffer buffer, int index, int size) |
| throws InvalidProtocolBufferException { |
| // Bitwise OR combines the sign bits so any negative value fails the check. |
| if ((index | size | buffer.limit() - index - size) < 0) { |
| throw new ArrayIndexOutOfBoundsException( |
| String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), index, size)); |
| } |
| long address = UnsafeUtil.addressOffset(buffer) + index; |
| final long addressLimit = address + size; |
| |
| // The longest possible resulting String is the same as the number of input bytes, when it is |
| // all ASCII. For other cases, this over-allocates and we will truncate in the end. |
| char[] resultArr = new char[size]; |
| int resultPos = 0; |
| |
| // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). |
| // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). |
| while (address < addressLimit) { |
| byte b = UnsafeUtil.getByte(address); |
| if (!DecodeUtil.isOneByte(b)) { |
| break; |
| } |
| address++; |
| DecodeUtil.handleOneByte(b, resultArr, resultPos++); |
| } |
| |
| while (address < addressLimit) { |
| byte byte1 = UnsafeUtil.getByte(address++); |
| if (DecodeUtil.isOneByte(byte1)) { |
| DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); |
| // It's common for there to be multiple ASCII characters in a run mixed in, so add an |
| // extra optimized loop to take care of these runs. |
| while (address < addressLimit) { |
| byte b = UnsafeUtil.getByte(address); |
| if (!DecodeUtil.isOneByte(b)) { |
| break; |
| } |
| address++; |
| DecodeUtil.handleOneByte(b, resultArr, resultPos++); |
| } |
| } else if (DecodeUtil.isTwoBytes(byte1)) { |
| if (address >= addressLimit) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleTwoBytes( |
| byte1, /* byte2 */ UnsafeUtil.getByte(address++), resultArr, resultPos++); |
| } else if (DecodeUtil.isThreeBytes(byte1)) { |
| if (address >= addressLimit - 1) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleThreeBytes( |
| byte1, |
| /* byte2 */ UnsafeUtil.getByte(address++), |
| /* byte3 */ UnsafeUtil.getByte(address++), |
| resultArr, |
| resultPos++); |
| } else { |
| if (address >= addressLimit - 2) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| DecodeUtil.handleFourBytes( |
| byte1, |
| /* byte2 */ UnsafeUtil.getByte(address++), |
| /* byte3 */ UnsafeUtil.getByte(address++), |
| /* byte4 */ UnsafeUtil.getByte(address++), |
| resultArr, |
| resultPos++); |
| // 4-byte case requires two chars. |
| resultPos++; |
| } |
| } |
| |
| return new String(resultArr, 0, resultPos); |
| } |
| |
| @Override |
| int encodeUtf8(final CharSequence in, final byte[] out, final int offset, final int length) { |
| long outIx = offset; |
| final long outLimit = outIx + length; |
| final int inLimit = in.length(); |
| if (inLimit > length || out.length - length < offset) { |
| // Not even enough room for an ASCII-encoded string. |
| throw new ArrayIndexOutOfBoundsException( |
| "Failed writing " + in.charAt(inLimit - 1) + " at index " + (offset + length)); |
| } |
| |
| // Designed to take advantage of |
| // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination |
| int inIx = 0; |
| for (char c; inIx < inLimit && (c = in.charAt(inIx)) < 0x80; ++inIx) { |
| UnsafeUtil.putByte(out, outIx++, (byte) c); |
| } |
| if (inIx == inLimit) { |
| // We're done, it was ASCII encoded. |
| return (int) outIx; |
| } |
| |
| for (char c; inIx < inLimit; ++inIx) { |
| c = in.charAt(inIx); |
| if (c < 0x80 && outIx < outLimit) { |
| UnsafeUtil.putByte(out, outIx++, (byte) c); |
| } else if (c < 0x800 && outIx <= outLimit - 2L) { // 11 bits, two UTF-8 bytes |
| UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 6) | (c >>> 6))); |
| UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & c))); |
| } else if ((c < MIN_SURROGATE || MAX_SURROGATE < c) && outIx <= outLimit - 3L) { |
| // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes |
| UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 5) | (c >>> 12))); |
| UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (c >>> 6)))); |
| UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & c))); |
| } else if (outIx <= outLimit - 4L) { |
| // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 |
| // bytes |
| final char low; |
| if (inIx + 1 == inLimit || !isSurrogatePair(c, (low = in.charAt(++inIx)))) { |
| throw new UnpairedSurrogateException((inIx - 1), inLimit); |
| } |
| int codePoint = toCodePoint(c, low); |
| UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 4) | (codePoint >>> 18))); |
| UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12)))); |
| UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6)))); |
| UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & codePoint))); |
| } else { |
| if ((MIN_SURROGATE <= c && c <= MAX_SURROGATE) |
| && (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) { |
| // We are surrogates and we're not a surrogate pair. |
| throw new UnpairedSurrogateException(inIx, inLimit); |
| } |
| // Not enough space in the output buffer. |
| throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + outIx); |
| } |
| } |
| |
| // All bytes have been encoded. |
| return (int) outIx; |
| } |
| |
| @Override |
| void encodeUtf8Direct(CharSequence in, ByteBuffer out) { |
| final long address = addressOffset(out); |
| long outIx = address + out.position(); |
| final long outLimit = address + out.limit(); |
| final int inLimit = in.length(); |
| if (inLimit > outLimit - outIx) { |
| // Not even enough room for an ASCII-encoded string. |
| throw new ArrayIndexOutOfBoundsException( |
| "Failed writing " + in.charAt(inLimit - 1) + " at index " + out.limit()); |
| } |
| |
| // Designed to take advantage of |
| // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination |
| int inIx = 0; |
| for (char c; inIx < inLimit && (c = in.charAt(inIx)) < 0x80; ++inIx) { |
| UnsafeUtil.putByte(outIx++, (byte) c); |
| } |
| if (inIx == inLimit) { |
| // We're done, it was ASCII encoded. |
| out.position((int) (outIx - address)); |
| return; |
| } |
| |
| for (char c; inIx < inLimit; ++inIx) { |
| c = in.charAt(inIx); |
| if (c < 0x80 && outIx < outLimit) { |
| UnsafeUtil.putByte(outIx++, (byte) c); |
| } else if (c < 0x800 && outIx <= outLimit - 2L) { // 11 bits, two UTF-8 bytes |
| UnsafeUtil.putByte(outIx++, (byte) ((0xF << 6) | (c >>> 6))); |
| UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & c))); |
| } else if ((c < MIN_SURROGATE || MAX_SURROGATE < c) && outIx <= outLimit - 3L) { |
| // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes |
| UnsafeUtil.putByte(outIx++, (byte) ((0xF << 5) | (c >>> 12))); |
| UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & (c >>> 6)))); |
| UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & c))); |
| } else if (outIx <= outLimit - 4L) { |
| // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 |
| // bytes |
| final char low; |
| if (inIx + 1 == inLimit || !isSurrogatePair(c, (low = in.charAt(++inIx)))) { |
| throw new UnpairedSurrogateException((inIx - 1), inLimit); |
| } |
| int codePoint = toCodePoint(c, low); |
| UnsafeUtil.putByte(outIx++, (byte) ((0xF << 4) | (codePoint >>> 18))); |
| UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12)))); |
| UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6)))); |
| UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & codePoint))); |
| } else { |
| if ((MIN_SURROGATE <= c && c <= MAX_SURROGATE) |
| && (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) { |
| // We are surrogates and we're not a surrogate pair. |
| throw new UnpairedSurrogateException(inIx, inLimit); |
| } |
| // Not enough space in the output buffer. |
| throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + outIx); |
| } |
| } |
| |
| // All bytes have been encoded. |
| out.position((int) (outIx - address)); |
| } |
| |
| /** |
| * Counts (approximately) the number of consecutive ASCII characters starting from the given |
| * position, using the most efficient method available to the platform. |
| * |
| * @param bytes the array containing the character sequence |
| * @param offset the offset position of the index (same as index + arrayBaseOffset) |
| * @param maxChars the maximum number of characters to count |
| * @return the number of ASCII characters found. The stopping position will be at or before the |
| * first non-ASCII byte. |
| */ |
| private static int unsafeEstimateConsecutiveAscii( |
| byte[] bytes, long offset, final int maxChars) { |
| if (maxChars < UNSAFE_COUNT_ASCII_THRESHOLD) { |
| // Don't bother with small strings. |
| return 0; |
| } |
| |
| for (int i = 0; i < maxChars; i++) { |
| if (UnsafeUtil.getByte(bytes, offset++) < 0) { |
| return i; |
| } |
| } |
| return maxChars; |
| } |
| |
| /** |
| * Same as {@link Utf8#estimateConsecutiveAscii(ByteBuffer, int, int)} except that it uses the |
| * most efficient method available to the platform. |
| */ |
| private static int unsafeEstimateConsecutiveAscii(long address, final int maxChars) { |
| int remaining = maxChars; |
| if (remaining < UNSAFE_COUNT_ASCII_THRESHOLD) { |
| // Don't bother with small strings. |
| return 0; |
| } |
| |
| // Read bytes until 8-byte aligned so that we can read longs in the loop below. |
| // We do this by ANDing the address with 7 to determine the number of bytes that need to |
| // be read before we're 8-byte aligned. |
| final int unaligned = 8 - ((int) address & 7); |
| for (int j = unaligned; j > 0; j--) { |
| if (UnsafeUtil.getByte(address++) < 0) { |
| return unaligned - j; |
| } |
| } |
| |
| // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). |
| // To speed things up further, we're reading longs instead of bytes so we use a mask to |
| // determine if any byte in the current long is non-ASCII. |
| remaining -= unaligned; |
| for (; |
| remaining >= 8 && (UnsafeUtil.getLong(address) & ASCII_MASK_LONG) == 0; |
| address += 8, remaining -= 8) {} |
| return maxChars - remaining; |
| } |
| |
| private static int partialIsValidUtf8(final byte[] bytes, long offset, int remaining) { |
| // Skip past ASCII characters as quickly as possible. |
| final int skipped = unsafeEstimateConsecutiveAscii(bytes, offset, remaining); |
| remaining -= skipped; |
| offset += skipped; |
| |
| for (; ; ) { |
| // Optimize for interior runs of ASCII bytes. |
| // TODO(nathanmittler): Consider checking 8 bytes at a time after some threshold? |
| // Maybe after seeing a few in a row that are ASCII, go back to fast mode? |
| int byte1 = 0; |
| for (; remaining > 0 && (byte1 = UnsafeUtil.getByte(bytes, offset++)) >= 0; --remaining) {} |
| if (remaining == 0) { |
| return COMPLETE; |
| } |
| remaining--; |
| |
| // If we're here byte1 is not ASCII. Only need to handle 2-4 byte forms. |
| if (byte1 < (byte) 0xE0) { |
| // Two-byte form (110xxxxx 10xxxxxx) |
| if (remaining == 0) { |
| // Incomplete sequence |
| return byte1; |
| } |
| remaining--; |
| |
| // Simultaneously checks for illegal trailing-byte in |
| // leading position and overlong 2-byte form. |
| if (byte1 < (byte) 0xC2 || UnsafeUtil.getByte(bytes, offset++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else if (byte1 < (byte) 0xF0) { |
| // Three-byte form (1110xxxx 10xxxxxx 10xxxxxx) |
| if (remaining < 2) { |
| // Incomplete sequence |
| return unsafeIncompleteStateFor(bytes, byte1, offset, remaining); |
| } |
| remaining -= 2; |
| |
| final int byte2; |
| if ((byte2 = UnsafeUtil.getByte(bytes, offset++)) > (byte) 0xBF |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // check for illegal surrogate codepoints |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| // byte3 trailing-byte test |
| || UnsafeUtil.getByte(bytes, offset++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else { |
| // Four-byte form (1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx) |
| if (remaining < 3) { |
| // Incomplete sequence |
| return unsafeIncompleteStateFor(bytes, byte1, offset, remaining); |
| } |
| remaining -= 3; |
| |
| final int byte2; |
| if ((byte2 = UnsafeUtil.getByte(bytes, offset++)) > (byte) 0xBF |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // if (byte1 > (byte) 0xF4 || |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| // byte3 trailing-byte test |
| || UnsafeUtil.getByte(bytes, offset++) > (byte) 0xBF |
| // byte4 trailing-byte test |
| || UnsafeUtil.getByte(bytes, offset++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } |
| } |
| } |
| |
| private static int partialIsValidUtf8(long address, int remaining) { |
| // Skip past ASCII characters as quickly as possible. |
| final int skipped = unsafeEstimateConsecutiveAscii(address, remaining); |
| address += skipped; |
| remaining -= skipped; |
| |
| for (; ; ) { |
| // Optimize for interior runs of ASCII bytes. |
| // TODO(nathanmittler): Consider checking 8 bytes at a time after some threshold? |
| // Maybe after seeing a few in a row that are ASCII, go back to fast mode? |
| int byte1 = 0; |
| for (; remaining > 0 && (byte1 = UnsafeUtil.getByte(address++)) >= 0; --remaining) {} |
| if (remaining == 0) { |
| return COMPLETE; |
| } |
| remaining--; |
| |
| if (byte1 < (byte) 0xE0) { |
| // Two-byte form |
| |
| if (remaining == 0) { |
| // Incomplete sequence |
| return byte1; |
| } |
| remaining--; |
| |
| // Simultaneously checks for illegal trailing-byte in |
| // leading position and overlong 2-byte form. |
| if (byte1 < (byte) 0xC2 || UnsafeUtil.getByte(address++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else if (byte1 < (byte) 0xF0) { |
| // Three-byte form |
| |
| if (remaining < 2) { |
| // Incomplete sequence |
| return unsafeIncompleteStateFor(address, byte1, remaining); |
| } |
| remaining -= 2; |
| |
| final byte byte2 = UnsafeUtil.getByte(address++); |
| if (byte2 > (byte) 0xBF |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // check for illegal surrogate codepoints |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| // byte3 trailing-byte test |
| || UnsafeUtil.getByte(address++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } else { |
| // Four-byte form |
| |
| if (remaining < 3) { |
| // Incomplete sequence |
| return unsafeIncompleteStateFor(address, byte1, remaining); |
| } |
| remaining -= 3; |
| |
| final byte byte2 = UnsafeUtil.getByte(address++); |
| if (byte2 > (byte) 0xBF |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // if (byte1 > (byte) 0xF4 || |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| // byte3 trailing-byte test |
| || UnsafeUtil.getByte(address++) > (byte) 0xBF |
| // byte4 trailing-byte test |
| || UnsafeUtil.getByte(address++) > (byte) 0xBF) { |
| return MALFORMED; |
| } |
| } |
| } |
| } |
| |
| private static int unsafeIncompleteStateFor( |
| byte[] bytes, int byte1, long offset, int remaining) { |
| switch (remaining) { |
| case 0: |
| return incompleteStateFor(byte1); |
| case 1: |
| return incompleteStateFor(byte1, UnsafeUtil.getByte(bytes, offset)); |
| case 2: |
| return incompleteStateFor( |
| byte1, UnsafeUtil.getByte(bytes, offset), UnsafeUtil.getByte(bytes, offset + 1)); |
| default: |
| throw new AssertionError(); |
| } |
| } |
| |
| private static int unsafeIncompleteStateFor(long address, final int byte1, int remaining) { |
| switch (remaining) { |
| case 0: |
| return incompleteStateFor(byte1); |
| case 1: |
| return incompleteStateFor(byte1, UnsafeUtil.getByte(address)); |
| case 2: |
| return incompleteStateFor( |
| byte1, UnsafeUtil.getByte(address), UnsafeUtil.getByte(address + 1)); |
| default: |
| throw new AssertionError(); |
| } |
| } |
| } |
| |
| /** |
| * Utility methods for decoding bytes into {@link String}. Callers are responsible for extracting |
| * bytes (possibly using Unsafe methods), and checking remaining bytes. All other UTF-8 validity |
| * checks and codepoint conversion happen in this class. |
| */ |
| private static class DecodeUtil { |
| |
| /** Returns whether this is a single-byte codepoint (i.e., ASCII) with the form '0XXXXXXX'. */ |
| private static boolean isOneByte(byte b) { |
| return b >= 0; |
| } |
| |
| /** Returns whether this is a two-byte codepoint with the form '10XXXXXX'. */ |
| private static boolean isTwoBytes(byte b) { |
| return b < (byte) 0xE0; |
| } |
| |
| /** Returns whether this is a three-byte codepoint with the form '110XXXXX'. */ |
| private static boolean isThreeBytes(byte b) { |
| return b < (byte) 0xF0; |
| } |
| |
| private static void handleOneByte(byte byte1, char[] resultArr, int resultPos) { |
| resultArr[resultPos] = (char) byte1; |
| } |
| |
| private static void handleTwoBytes(byte byte1, byte byte2, char[] resultArr, int resultPos) |
| throws InvalidProtocolBufferException { |
| // Simultaneously checks for illegal trailing-byte in leading position (<= '11000000') and |
| // overlong 2-byte, '11000001'. |
| if (byte1 < (byte) 0xC2 || isNotTrailingByte(byte2)) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| resultArr[resultPos] = (char) (((byte1 & 0x1F) << 6) | trailingByteValue(byte2)); |
| } |
| |
| private static void handleThreeBytes( |
| byte byte1, byte byte2, byte byte3, char[] resultArr, int resultPos) |
| throws InvalidProtocolBufferException { |
| if (isNotTrailingByte(byte2) |
| // overlong? 5 most significant bits must not all be zero |
| || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) |
| // check for illegal surrogate codepoints |
| || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) |
| || isNotTrailingByte(byte3)) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| resultArr[resultPos] = |
| (char) |
| (((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) | trailingByteValue(byte3)); |
| } |
| |
| private static void handleFourBytes( |
| byte byte1, byte byte2, byte byte3, byte byte4, char[] resultArr, int resultPos) |
| throws InvalidProtocolBufferException { |
| if (isNotTrailingByte(byte2) |
| // Check that 1 <= plane <= 16. Tricky optimized form of: |
| // valid 4-byte leading byte? |
| // if (byte1 > (byte) 0xF4 || |
| // overlong? 4 most significant bits must not all be zero |
| // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || |
| // codepoint larger than the highest code point (U+10FFFF)? |
| // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) |
| || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 |
| || isNotTrailingByte(byte3) |
| || isNotTrailingByte(byte4)) { |
| throw InvalidProtocolBufferException.invalidUtf8(); |
| } |
| int codepoint = |
| ((byte1 & 0x07) << 18) |
| | (trailingByteValue(byte2) << 12) |
| | (trailingByteValue(byte3) << 6) |
| | trailingByteValue(byte4); |
| resultArr[resultPos] = DecodeUtil.highSurrogate(codepoint); |
| resultArr[resultPos + 1] = DecodeUtil.lowSurrogate(codepoint); |
| } |
| |
| /** Returns whether the byte is not a valid continuation of the form '10XXXXXX'. */ |
| private static boolean isNotTrailingByte(byte b) { |
| return b > (byte) 0xBF; |
| } |
| |
| /** Returns the actual value of the trailing byte (removes the prefix '10') for composition. */ |
| private static int trailingByteValue(byte b) { |
| return b & 0x3F; |
| } |
| |
| private static char highSurrogate(int codePoint) { |
| return (char) |
| ((MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10)) + (codePoint >>> 10)); |
| } |
| |
| private static char lowSurrogate(int codePoint) { |
| return (char) (MIN_LOW_SURROGATE + (codePoint & 0x3ff)); |
| } |
| } |
| |
| private Utf8() {} |
| } |