#include <scanner-character-streams.h>

Inheritance diagram for v8::internal::ExternalStreamingStream:

Collaboration diagram for v8::internal::ExternalStreamingStream:

Public Member Functions
	ExternalStreamingStream (ScriptCompiler::ExternalSourceStream *source_stream, v8::ScriptCompiler::StreamedSource::Encoding encoding)

virtual	~ExternalStreamingStream ()

virtual unsigned	BufferSeekForward (unsigned delta) OVERRIDE

virtual unsigned	FillBuffer (unsigned position)

Public Member Functions inherited from v8::internal::BufferedUtf16CharacterStream
	BufferedUtf16CharacterStream ()

virtual	~BufferedUtf16CharacterStream ()

virtual void	PushBack (uc32 character)

Public Member Functions inherited from v8::internal::Utf16CharacterStream
	Utf16CharacterStream ()

virtual	~Utf16CharacterStream ()

uc32	Advance ()

unsigned	pos () const

unsigned	SeekForward (unsigned code_unit_count)

Private Member Functions
void	HandleUtf8SplitCharacters (unsigned *data_in_buffer)

Private Attributes
ScriptCompiler::ExternalSourceStream *	source_stream_

v8::ScriptCompiler::StreamedSource::Encoding	encoding_

const uint8_t *	current_data_

unsigned	current_data_offset_

unsigned	current_data_length_

uint8_t	utf8_split_char_buffer_ [4]

unsigned	utf8_split_char_buffer_length_

Additional Inherited Members
Protected Member Functions inherited from v8::internal::BufferedUtf16CharacterStream
virtual unsigned	SlowSeekForward (unsigned delta)

virtual bool	ReadBlock ()

virtual void	SlowPushBack (uc16 character)

Protected Attributes inherited from v8::internal::BufferedUtf16CharacterStream
const uc16 *	pushback_limit_

uc16	buffer_ [kBufferSize]

Protected Attributes inherited from v8::internal::Utf16CharacterStream
const uint16_t *	buffer_cursor_

const uint16_t *	buffer_end_

unsigned	pos_

Static Protected Attributes inherited from v8::internal::BufferedUtf16CharacterStream
static const unsigned	kBufferSize = 512

static const unsigned	kPushBackStepSize = 16

Static Protected Attributes inherited from v8::internal::Utf16CharacterStream
static const uc32	kEndOfInput = -1

Detailed Description

Definition at line 81 of file scanner-character-streams.h.

Constructor & Destructor Documentation

◆ ExternalStreamingStream()

v8::internal::ExternalStreamingStream::ExternalStreamingStream	(	ScriptCompiler::ExternalSourceStream *	source_stream,
		v8::ScriptCompiler::StreamedSource::Encoding	encoding
	)

inline

Definition at line 83 of file scanner-character-streams.h.

       : source_stream_(source_stream),
         encoding_(encoding),
         current_data_(NULL),
         current_data_offset_(0),
         current_data_length_(0),
         utf8_split_char_buffer_length_(0) {}

◆ ~ExternalStreamingStream()

virtual v8::internal::ExternalStreamingStream::~ExternalStreamingStream ( )

inlinevirtual

Definition at line 92 of file scanner-character-streams.h.

92 { delete[] current_data_; }

References current_data_.

Member Function Documentation

◆ BufferSeekForward()

virtual unsigned v8::internal::ExternalStreamingStream::BufferSeekForward ( unsigned delta )

inlinevirtual

Implements v8::internal::BufferedUtf16CharacterStream.

Definition at line 94 of file scanner-character-streams.h.

                                                               {
     // We never need to seek forward when streaming scripts. We only seek
     // forward when we want to parse a function whose location we already know,
     // and when streaming, we don't know the locations of anything we haven't
     // seen yet.
     UNREACHABLE();
     return 0;
   }

References UNREACHABLE.

◆ FillBuffer()

unsigned v8::internal::ExternalStreamingStream::FillBuffer ( unsigned position )

virtual

Implements v8::internal::BufferedUtf16CharacterStream.

Definition at line 319 of file scanner-character-streams.cc.

                                                               {
   // Ignore "position" which is the position in the decoded data. Instead,
   // ExternalStreamingStream keeps track of the position in the raw data.
   unsigned data_in_buffer = 0;
   // Note that the UTF-8 decoder might not be able to fill the buffer
   // completely; it will typically leave the last character empty (see
   // Utf8ToUtf16CharacterStream::CopyChars).
   while (data_in_buffer < kBufferSize - 1) {
     if (current_data_ == NULL) {
       // GetSomeData will wait until the embedder has enough data. Here's an
       // interface between the API which uses size_t (which is the correct type
       // here) and the internal parts which use unsigned. TODO(marja): make the
       // internal parts use size_t too.
       current_data_length_ =
           static_cast<unsigned>(source_stream_->GetMoreData(&current_data_));
       current_data_offset_ = 0;
       bool data_ends = current_data_length_ == 0;
  
       // A caveat: a data chunk might end with bytes from an incomplete UTF-8
       // character (the rest of the bytes will be in the next chunk).
       if (encoding_ == ScriptCompiler::StreamedSource::UTF8) {
         HandleUtf8SplitCharacters(&data_in_buffer);
         if (!data_ends && current_data_offset_ == current_data_length_) {
           // The data stream didn't end, but we used all the data in the
           // chunk. This will only happen when the chunk was really small. We
           // don't handle the case where a UTF-8 character is split over several
           // chunks; in that case V8 won't crash, but it will be a parse error.
           delete[] current_data_;
           current_data_ = NULL;
           current_data_length_ = 0;
           current_data_offset_ = 0;
           continue;  // Request a new chunk.
         }
       }
  
       // Did the data stream end?
       if (data_ends) {
         DCHECK(utf8_split_char_buffer_length_ == 0);
         return data_in_buffer;
       }
     }
  
     // Fill the buffer from current_data_.
     unsigned new_offset = 0;
     unsigned new_chars_in_buffer =
         CopyCharsHelper(buffer_ + data_in_buffer, kBufferSize - data_in_buffer,
                         current_data_ + current_data_offset_, &new_offset,
                         current_data_length_ - current_data_offset_, encoding_);
     data_in_buffer += new_chars_in_buffer;
     current_data_offset_ += new_offset;
     DCHECK(data_in_buffer <= kBufferSize);
  
     // Did we use all the data in the data chunk?
     if (current_data_offset_ == current_data_length_) {
       delete[] current_data_;
       current_data_ = NULL;
       current_data_length_ = 0;
       current_data_offset_ = 0;
     }
   }
   return data_in_buffer;
 }

References v8::internal::BufferedUtf16CharacterStream::buffer_, v8::internal::anonymous_namespace{scanner-character-streams.cc}::CopyCharsHelper(), current_data_, current_data_length_, current_data_offset_, DCHECK, encoding_, v8::ScriptCompiler::ExternalSourceStream::GetMoreData(), HandleUtf8SplitCharacters(), v8::internal::BufferedUtf16CharacterStream::kBufferSize, NULL, source_stream_, v8::ScriptCompiler::StreamedSource::UTF8, and utf8_split_char_buffer_length_.

Here is the call graph for this function:

◆ HandleUtf8SplitCharacters()

void v8::internal::ExternalStreamingStream::HandleUtf8SplitCharacters ( unsigned * data_in_buffer )

private

Definition at line 382 of file scanner-character-streams.cc.

                               {
   // First check if we have leftover data from the last chunk.
   unibrow::uchar c;
   if (utf8_split_char_buffer_length_ > 0) {
     // Move the bytes which are part of the split character (which started in
     // the previous chunk) into utf8_split_char_buffer_.
     while (current_data_offset_ < current_data_length_ &&
            utf8_split_char_buffer_length_ < 4 &&
            (c = current_data_[current_data_offset_]) >
                unibrow::Utf8::kMaxOneByteChar) {
       utf8_split_char_buffer_[utf8_split_char_buffer_length_] = c;
       ++utf8_split_char_buffer_length_;
       ++current_data_offset_;
     }
  
     // Convert the data in utf8_split_char_buffer_.
     unsigned new_offset = 0;
     unsigned new_chars_in_buffer =
         CopyCharsHelper(buffer_ + *data_in_buffer,
                         kBufferSize - *data_in_buffer, utf8_split_char_buffer_,
                         &new_offset, utf8_split_char_buffer_length_, encoding_);
     *data_in_buffer += new_chars_in_buffer;
     // Make sure we used all the data.
     DCHECK(new_offset == utf8_split_char_buffer_length_);
     DCHECK(*data_in_buffer <= kBufferSize);
  
     utf8_split_char_buffer_length_ = 0;
   }
  
   // Move bytes which are part of an incomplete character from the end of the
   // current chunk to utf8_split_char_buffer_. They will be converted when the
   // next data chunk arrives. Note that all valid UTF-8 characters are at most 4
   // bytes long, but if the data is invalid, we can have character values bigger
   // than unibrow::Utf8::kMaxOneByteChar for more than 4 consecutive bytes.
   while (current_data_length_ > current_data_offset_ &&
          (c = current_data_[current_data_length_ - 1]) >
              unibrow::Utf8::kMaxOneByteChar &&
          utf8_split_char_buffer_length_ < 4) {
     --current_data_length_;
     ++utf8_split_char_buffer_length_;
   }
   CHECK(utf8_split_char_buffer_length_ <= 4);
   for (unsigned i = 0; i < utf8_split_char_buffer_length_; ++i) {
     utf8_split_char_buffer_[i] = current_data_[current_data_length_ + i];
   }
 }