unibrow::Utf8 Class Reference

#include <unicode.h>

Collaboration diagram for unibrow::Utf8:

Static Public Member Functions
static uchar	Length (uchar chr, int previous)
static unsigned	EncodeOneByte (char *out, uint8_t c)
static unsigned	Encode (char *out, uchar c, int previous, bool replace_invalid=false)
static uchar	CalculateValue (const byte *str, unsigned length, unsigned *cursor)
static uchar	ValueOf (const byte *str, unsigned length, unsigned *cursor)

Static Public Attributes
static const uchar	kBadChar = 0xFFFD
static const unsigned	kMaxEncodedSize = 4
static const unsigned	kMaxOneByteChar = 0x7f
static const unsigned	kMaxTwoByteChar = 0x7ff
static const unsigned	kMaxThreeByteChar = 0xffff
static const unsigned	kMaxFourByteChar = 0x1fffff
static const unsigned	kBytesSavedByCombiningSurrogates = 2
static const unsigned	kSizeOfUnmatchedSurrogate = 3
static const unsigned	kMax16BitCodeUnitSize = 3

Detailed Description

Definition at line 125 of file unicode.h.

Member Function Documentation

CalculateValue()

uchar unibrow::Utf8::CalculateValue ( const byte *  str, unsigned  length, unsigned *  cursor  )

static

Definition at line 191 of file unicode.cc.

                                             {
  // We only get called for non-ASCII characters.
  if (length == 1) {
    *cursor += 1;
    return kBadChar;
  }
  byte first = str[0];
  byte second = str[1] ^ 0x80;
  if (second & 0xC0) {
    *cursor += 1;
    return kBadChar;
  }
  if (first < 0xE0) {
    if (first < 0xC0) {
      *cursor += 1;
      return kBadChar;
    }
    uchar code_point = ((first << 6) | second) & kMaxTwoByteChar;
    if (code_point <= kMaxOneByteChar) {
      *cursor += 1;
      return kBadChar;
    }
    *cursor += 2;
    return code_point;
  }
  if (length == 2) {
    *cursor += 1;
    return kBadChar;
  }
  byte third = str[2] ^ 0x80;
  if (third & 0xC0) {
    *cursor += 1;
    return kBadChar;
  }
  if (first < 0xF0) {
    uchar code_point = ((((first << 6) | second) << 6) | third)
        & kMaxThreeByteChar;
    if (code_point <= kMaxTwoByteChar) {
      *cursor += 1;
      return kBadChar;
    }
    *cursor += 3;
    return code_point;
  }
  if (length == 3) {
    *cursor += 1;
    return kBadChar;
  }
  byte fourth = str[3] ^ 0x80;
  if (fourth & 0xC0) {
    *cursor += 1;
    return kBadChar;
  }
  if (first < 0xF8) {
    uchar code_point = (((((first << 6 | second) << 6) | third) << 6) | fourth)
        & kMaxFourByteChar;
    if (code_point <= kMaxThreeByteChar) {
      *cursor += 1;
      return kBadChar;
    }
    *cursor += 4;
    return code_point;
  }
  *cursor += 1;
  return kBadChar;
}

References kBadChar, kMaxFourByteChar, kMaxOneByteChar, kMaxThreeByteChar, and kMaxTwoByteChar.

Referenced by v8::internal::Utf8ToUtf16CharacterStream::CopyChars(), v8::internal::HeapSnapshotJSONSerializer::SerializeString(), and ValueOf().

Here is the caller graph for this function:

Encode()

unsigned unibrow::Utf8::Encode ( char *  out, uchar  c, int  previous, bool  replace_invalid = false  )

inlinestatic

Definition at line 91 of file unicode-inl.h.

                                            {
  static const int kMask = ~(1 << 6);
  if (c <= kMaxOneByteChar) {
    str[0] = c;
    return 1;
  } else if (c <= kMaxTwoByteChar) {
    str[0] = 0xC0 | (c >> 6);
    str[1] = 0x80 | (c & kMask);
    return 2;
  } else if (c <= kMaxThreeByteChar) {
    if (Utf16::IsSurrogatePair(previous, c)) {
      const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
      return Encode(str - kUnmatchedSize,
                    Utf16::CombineSurrogatePair(previous, c),
                    Utf16::kNoPreviousCharacter,
                    replace_invalid) - kUnmatchedSize;
    } else if (replace_invalid &&
               (Utf16::IsLeadSurrogate(c) ||
               Utf16::IsTrailSurrogate(c))) {
      c = kBadChar;
    }
    str[0] = 0xE0 | (c >> 12);
    str[1] = 0x80 | ((c >> 6) & kMask);
    str[2] = 0x80 | (c & kMask);
    return 3;
  } else {
    str[0] = 0xF0 | (c >> 18);
    str[1] = 0x80 | ((c >> 12) & kMask);
    str[2] = 0x80 | ((c >> 6) & kMask);
    str[3] = 0x80 | (c & kMask);
    return 4;
  }
}

References unibrow::Utf16::CombineSurrogatePair(), unibrow::Utf16::IsLeadSurrogate(), unibrow::Utf16::IsSurrogatePair(), unibrow::Utf16::IsTrailSurrogate(), kBadChar, kMaxOneByteChar, kMaxThreeByteChar, kMaxTwoByteChar, unibrow::Utf16::kNoPreviousCharacter, and kSizeOfUnmatchedSurrogate.

Referenced by v8::internal::CodeEventLogger::NameBuffer::AppendString(), and v8::internal::String::ToCString().

Here is the call graph for this function:

Here is the caller graph for this function:

EncodeOneByte()

unsigned unibrow::Utf8::EncodeOneByte ( char *  out, uint8_t  c  )

inlinestatic

Definition at line 76 of file unicode-inl.h.

                                                 {
  static const int kMask = ~(1 << 6);
  if (c <= kMaxOneByteChar) {
    str[0] = c;
    return 1;
  }
  str[0] = 0xC0 | (c >> 6);
  str[1] = 0x80 | (c & kMask);
  return 2;
}

References kMaxOneByteChar.

Length()

unsigned unibrow::Utf8::Length ( uchar  chr, int  previous  )

inlinestatic

Definition at line 140 of file unicode-inl.h.

                                           {
  if (c <= kMaxOneByteChar) {
    return 1;
  } else if (c <= kMaxTwoByteChar) {
    return 2;
  } else if (c <= kMaxThreeByteChar) {
    if (Utf16::IsTrailSurrogate(c) &&
        Utf16::IsLeadSurrogate(previous)) {
      return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates;
    }
    return 3;
  } else {
    return 4;
  }
}

References unibrow::Utf16::IsLeadSurrogate(), unibrow::Utf16::IsTrailSurrogate(), kBytesSavedByCombiningSurrogates, kMaxOneByteChar, kMaxThreeByteChar, kMaxTwoByteChar, and kSizeOfUnmatchedSurrogate.

Referenced by v8::internal::CodeEventLogger::NameBuffer::AppendString(), v8::internal::String::ToCString(), and v8::Utf8LengthHelper::Visitor::VisitTwoByteString().

Here is the call graph for this function:

Here is the caller graph for this function:

ValueOf()

uchar unibrow::Utf8::ValueOf ( const byte *  str, unsigned  length, unsigned *  cursor  )

inlinestatic

Definition at line 129 of file unicode-inl.h.

                                                                        {
  if (length <= 0) return kBadChar;
  byte first = bytes[0];
  // Characters between 0000 and 0007F are encoded as a single character
  if (first <= kMaxOneByteChar) {
    *cursor += 1;
    return first;
  }
  return CalculateValue(bytes, length, cursor);
}

References CalculateValue(), kBadChar, and kMaxOneByteChar.

Referenced by v8::internal::StringHasher::ComputeUtf8Hash(), v8::internal::String::IsUtf8EqualTo(), unibrow::Utf8DecoderBase::Reset(), v8::internal::WriteTwoByteData(), and unibrow::Utf8DecoderBase::WriteUtf16Slow().

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

kBadChar

const uchar unibrow::Utf8::kBadChar = 0xFFFD

static

Definition at line 139 of file unicode.h.

Referenced by CalculateValue(), Encode(), v8::internal::HeapSnapshotJSONSerializer::SerializeString(), ValueOf(), and v8::internal::WriteTwoByteData().

kBytesSavedByCombiningSurrogates

const unsigned unibrow::Utf8::kBytesSavedByCombiningSurrogates = 2

static

Definition at line 148 of file unicode.h.

Referenced by Length(), v8::Utf8LengthHelper::MergeLeafLeft(), v8::Utf8LengthHelper::MergeLeafRight(), and v8::Utf8LengthHelper::MergeTerminal().

kMax16BitCodeUnitSize

const unsigned unibrow::Utf8::kMax16BitCodeUnitSize = 3

static

Definition at line 152 of file unicode.h.

Referenced by v8::String::WriteUtf8().

kMaxEncodedSize

const unsigned unibrow::Utf8::kMaxEncodedSize = 4

static

Definition at line 140 of file unicode.h.

Referenced by v8::internal::String::IsUtf8EqualTo(), and unibrow::Utf8DecoderBase::WriteUtf16Slow().

kMaxFourByteChar

const unsigned unibrow::Utf8::kMaxFourByteChar = 0x1fffff

static

Definition at line 144 of file unicode.h.

Referenced by CalculateValue().

kMaxOneByteChar

const unsigned unibrow::Utf8::kMaxOneByteChar = 0x7f

static

Definition at line 141 of file unicode.h.

Referenced by v8::internal::CodeEventLogger::NameBuffer::AppendString(), CalculateValue(), v8::internal::StringHasher::ComputeUtf8Hash(), v8::internal::Utf8ToUtf16CharacterStream::CopyChars(), Encode(), EncodeOneByte(), v8::internal::ExternalStreamingStream::HandleUtf8SplitCharacters(), Length(), v8::internal::String::NonAsciiStart(), v8::internal::Utf8CharacterBack(), v8::internal::Utf8CharacterForward(), and ValueOf().

kMaxThreeByteChar

const unsigned unibrow::Utf8::kMaxThreeByteChar = 0xffff

static

Definition at line 143 of file unicode.h.

Referenced by CalculateValue(), Encode(), and Length().

kMaxTwoByteChar

const unsigned unibrow::Utf8::kMaxTwoByteChar = 0x7ff

static

Definition at line 142 of file unicode.h.

Referenced by CalculateValue(), Encode(), and Length().

kSizeOfUnmatchedSurrogate

const unsigned unibrow::Utf8::kSizeOfUnmatchedSurrogate = 3

static

Definition at line 149 of file unicode.h.

Referenced by Encode(), and Length().

---

The documentation for this class was generated from the following files:

• /mnt/V8SourceCode/src/unicode.h
• /mnt/V8SourceCode/src/unicode-inl.h
• /mnt/V8SourceCode/src/unicode.cc