v8::internal::Scanner Class Reference

#include <scanner.h>

Collaboration diagram for v8::internal::Scanner:

Classes
class	LiteralScope
struct	Location
struct	TokenDesc

Public Member Functions
	Scanner (UnicodeCache *scanner_contants)
void	Initialize (Utf16CharacterStream *source)
Token::Value	Next ()
Token::Value	current_token ()
Location	location () const
Token::Value	peek () const
Location	peek_location () const
bool	literal_contains_escapes () const
bool	is_literal_contextual_keyword (Vector< const char > keyword)
bool	is_next_contextual_keyword (Vector< const char > keyword)
const AstRawString *	CurrentSymbol (AstValueFactory *ast_value_factory)
const AstRawString *	NextSymbol (AstValueFactory *ast_value_factory)
double	DoubleValue ()
bool	UnescapedLiteralMatches (const char *data, int length)
void	IsGetOrSet (bool *is_get, bool *is_set)
int	FindNumber (DuplicateFinder *finder, int value)
int	FindSymbol (DuplicateFinder *finder, int value)
UnicodeCache *	unicode_cache ()
Location	octal_position () const
void	clear_octal_position ()
void	SeekForward (int pos)
bool	HarmonyScoping () const
void	SetHarmonyScoping (bool scoping)
bool	HarmonyModules () const
void	SetHarmonyModules (bool modules)
bool	HarmonyNumericLiterals () const
void	SetHarmonyNumericLiterals (bool numeric_literals)
bool	HarmonyClasses () const
void	SetHarmonyClasses (bool classes)
bool	HasAnyLineTerminatorBeforeNext () const
bool	ScanRegExpPattern (bool seen_equal)
bool	ScanRegExpFlags ()
const LiteralBuffer *	source_url () const
const LiteralBuffer *	source_mapping_url () const
bool	IdentifierIsFutureStrictReserved (const AstRawString *string) const

Static Public Attributes
static const int	kNoOctalLocation = -1

Private Member Functions
uc32	ScanOctalEscape (uc32 c, int length)
void	Init ()
void	StartLiteral ()
	INLINE (void AddLiteralChar(uc32 c))
void	TerminateLiteral ()
void	DropLiteral ()
void	AddLiteralCharAdvance ()
void	Advance ()
void	PushBack (uc32 ch)
Token::Value	Select (Token::Value tok)
Token::Value	Select (uc32 next, Token::Value then, Token::Value else_)
Vector< const uint8_t >	literal_one_byte_string ()
Vector< const uint16_t >	literal_two_byte_string ()
bool	is_literal_one_byte ()
int	literal_length () const
Vector< const uint8_t >	next_literal_one_byte_string ()
Vector< const uint16_t >	next_literal_two_byte_string ()
bool	is_next_literal_one_byte ()
int	next_literal_length () const
uc32	ScanHexNumber (int expected_length)
void	Scan ()
bool	SkipWhiteSpace ()
Token::Value	SkipSingleLineComment ()
Token::Value	SkipSourceURLComment ()
void	TryToParseSourceURLComment ()
Token::Value	SkipMultiLineComment ()
Token::Value	ScanHtmlComment ()
void	ScanDecimalDigits ()
Token::Value	ScanNumber (bool seen_period)
Token::Value	ScanIdentifierOrKeyword ()
Token::Value	ScanIdentifierSuffix (LiteralScope *literal)
Token::Value	ScanString ()
bool	ScanEscape ()
uc32	ScanIdentifierUnicodeEscape ()
bool	ScanLiteralUnicodeEscape ()
int	source_pos ()

Private Attributes
UnicodeCache *	unicode_cache_
LiteralBuffer	literal_buffer1_
LiteralBuffer	literal_buffer2_
LiteralBuffer	source_url_
LiteralBuffer	source_mapping_url_
TokenDesc	current_
TokenDesc	next_
Utf16CharacterStream *	source_
Location	octal_pos_
uc32	c0_
bool	has_line_terminator_before_next_
bool	has_multiline_comment_before_next_
bool	harmony_scoping_
bool	harmony_modules_
bool	harmony_numeric_literals_
bool	harmony_classes_

Static Private Attributes
static const int	kCharacterLookaheadBufferSize = 1

Detailed Description

Definition at line 305 of file scanner.h.

Constructor & Destructor Documentation

Scanner()

v8::internal::Scanner::Scanner ( UnicodeCache *  scanner_contants )

explicit

Definition at line 34 of file scanner.cc.

    : unicode_cache_(unicode_cache),
      octal_pos_(Location::invalid()),
      harmony_scoping_(false),
      harmony_modules_(false),
      harmony_numeric_literals_(false),
      harmony_classes_(false) { }

Member Function Documentation

AddLiteralCharAdvance()

void v8::internal::Scanner::AddLiteralCharAdvance ( )

inlineprivate

Definition at line 515 of file scanner.h.

                                      {
    AddLiteralChar(c0_);
    Advance();
  }

References Advance(), and c0_.

Referenced by ScanDecimalDigits(), ScanNumber(), ScanRegExpFlags(), and ScanRegExpPattern().

Here is the call graph for this function:

Here is the caller graph for this function:

Advance()

void v8::internal::Scanner::Advance ( )

inlineprivate

Definition at line 521 of file scanner.h.

{ c0_ = source_->Advance(); }

References v8::internal::Utf16CharacterStream::Advance(), c0_, and source_.

Referenced by AddLiteralCharAdvance(), Init(), Next(), Scan(), ScanEscape(), ScanHexNumber(), ScanHtmlComment(), ScanIdentifierOrKeyword(), ScanIdentifierSuffix(), ScanIdentifierUnicodeEscape(), ScanLiteralUnicodeEscape(), ScanOctalEscape(), ScanRegExpFlags(), ScanRegExpPattern(), ScanString(), SeekForward(), Select(), SkipMultiLineComment(), SkipSingleLineComment(), SkipSourceURLComment(), SkipWhiteSpace(), and TryToParseSourceURLComment().

Here is the call graph for this function:

Here is the caller graph for this function:

clear_octal_position()

void v8::internal::Scanner::clear_octal_position ( )

inline

Definition at line 415 of file scanner.h.

{ octal_pos_ = Location::invalid(); }

References v8::internal::Scanner::Location::invalid(), and octal_pos_.

Referenced by v8::internal::ParserBase< Traits >::CheckOctalLiteral().

Here is the call graph for this function:

Here is the caller graph for this function:

current_token()

Token::Value v8::internal::Scanner::current_token ( )

inline

Definition at line 353 of file scanner.h.

{ return current_.token; }

References current_, and v8::internal::Scanner::TokenDesc::token.

Referenced by v8::internal::PreParserTraits::GetSymbol().

Here is the caller graph for this function:

CurrentSymbol()

const AstRawString * v8::internal::Scanner::CurrentSymbol

(

AstValueFactory *

ast_value_factory

)

Definition at line 1195 of file scanner.cc.

                                                                             {
  if (is_literal_one_byte()) {
    return ast_value_factory->GetOneByteString(literal_one_byte_string());
  }
  return ast_value_factory->GetTwoByteString(literal_two_byte_string());
}

References v8::internal::AstValueFactory::GetOneByteString(), v8::internal::AstValueFactory::GetTwoByteString(), is_literal_one_byte(), literal_one_byte_string(), and literal_two_byte_string().

Referenced by v8::internal::ParserTraits::GetSymbol().

Here is the call graph for this function:

Here is the caller graph for this function:

DoubleValue()

double v8::internal::Scanner::DoubleValue

(

)

Definition at line 1211 of file scanner.cc.

                            {
  DCHECK(is_literal_one_byte());
  return StringToDouble(
      unicode_cache_,
      literal_one_byte_string(),
      ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY);
}

References v8::internal::ALLOW_BINARY, v8::internal::ALLOW_HEX, v8::internal::ALLOW_IMPLICIT_OCTAL, v8::internal::ALLOW_OCTAL, DCHECK, is_literal_one_byte(), literal_one_byte_string(), v8::internal::StringToDouble(), and unicode_cache_.

Referenced by v8::internal::ParserTraits::ExpressionFromLiteral(), and v8::internal::ParserTraits::GetNumberAsSymbol().

Here is the call graph for this function:

Here is the caller graph for this function:

DropLiteral()

void v8::internal::Scanner::DropLiteral ( )

inlineprivate

Definition at line 511 of file scanner.h.

                            {
    next_.literal_chars = NULL;
  }

References v8::internal::Scanner::TokenDesc::literal_chars, next_, and NULL.

Referenced by v8::internal::Scanner::LiteralScope::~LiteralScope().

Here is the caller graph for this function:

FindNumber()

int v8::internal::Scanner::FindNumber	(	DuplicateFinder *	finder,
		int	value
	)

Definition at line 1220 of file scanner.cc.

                                                          {
  return finder->AddNumber(literal_one_byte_string(), value);
}

References v8::internal::DuplicateFinder::AddNumber(), and literal_one_byte_string().

Referenced by v8::internal::ParserBase< Traits >::ObjectLiteralChecker::CheckProperty().

Here is the call graph for this function:

Here is the caller graph for this function:

FindSymbol()

int v8::internal::Scanner::FindSymbol	(	DuplicateFinder *	finder,
		int	value
	)

Definition at line 1225 of file scanner.cc.

                                                          {
  if (is_literal_one_byte()) {
    return finder->AddOneByteSymbol(literal_one_byte_string(), value);
  }
  return finder->AddTwoByteSymbol(literal_two_byte_string(), value);
}

References v8::internal::DuplicateFinder::AddOneByteSymbol(), v8::internal::DuplicateFinder::AddTwoByteSymbol(), is_literal_one_byte(), literal_one_byte_string(), and literal_two_byte_string().

Referenced by v8::internal::ParserBase< Traits >::ObjectLiteralChecker::CheckProperty(), and v8::internal::PreParser::ParseFunctionLiteral().

Here is the call graph for this function:

Here is the caller graph for this function:

HarmonyClasses()

bool v8::internal::Scanner::HarmonyClasses ( ) const

inline

Definition at line 441 of file scanner.h.

                              {
    return harmony_classes_;
  }

References harmony_classes_.

Referenced by v8::internal::ParserBase< Traits >::allow_classes().

Here is the caller graph for this function:

HarmonyModules()

bool v8::internal::Scanner::HarmonyModules ( ) const

inline

Definition at line 429 of file scanner.h.

                              {
    return harmony_modules_;
  }

References harmony_modules_.

Referenced by v8::internal::ParserBase< Traits >::allow_modules().

Here is the caller graph for this function:

HarmonyNumericLiterals()

bool v8::internal::Scanner::HarmonyNumericLiterals ( ) const

inline

Definition at line 435 of file scanner.h.

                                      {
    return harmony_numeric_literals_;
  }

References harmony_numeric_literals_.

Referenced by v8::internal::ParserBase< Traits >::allow_harmony_numeric_literals().

Here is the caller graph for this function:

HarmonyScoping()

bool v8::internal::Scanner::HarmonyScoping ( ) const

inline

Definition at line 423 of file scanner.h.

                              {
    return harmony_scoping_;
  }

References harmony_scoping_.

Referenced by v8::internal::ParserBase< Traits >::allow_harmony_scoping().

Here is the caller graph for this function:

HasAnyLineTerminatorBeforeNext()

bool v8::internal::Scanner::HasAnyLineTerminatorBeforeNext ( ) const

inline

Definition at line 450 of file scanner.h.

                                              {
    return has_line_terminator_before_next_ ||
           has_multiline_comment_before_next_;
  }

References has_line_terminator_before_next_, and has_multiline_comment_before_next_.

IdentifierIsFutureStrictReserved()

bool v8::internal::Scanner::IdentifierIsFutureStrictReserved

(

const AstRawString *

string

)

const

Definition at line 1017 of file scanner.cc.

                                      {
  // Keywords are always 1-byte strings.
  return string->is_one_byte() &&
         Token::FUTURE_STRICT_RESERVED_WORD ==
             KeywordOrIdentifierToken(string->raw_data(), string->length(),
                                      harmony_scoping_, harmony_modules_,
                                      harmony_classes_);
}

References harmony_classes_, harmony_modules_, harmony_scoping_, v8::internal::KeywordOrIdentifierToken(), v8::internal::AstRawString::length(), and v8::internal::AstRawString::raw_data().

Referenced by v8::internal::ParserTraits::IsFutureStrictReserved().

Here is the call graph for this function:

Here is the caller graph for this function:

Init()

void v8::internal::Scanner::Init ( )

inlineprivate

Definition at line 483 of file scanner.h.

              {
    // Set c0_ (one character ahead)
    STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
    Advance();
    // Initialize current_ to not refer to a literal.
    current_.literal_chars = NULL;
  }

References Advance(), current_, kCharacterLookaheadBufferSize, v8::internal::Scanner::TokenDesc::literal_chars, NULL, and v8::internal::STATIC_ASSERT().

Referenced by Initialize().

Here is the call graph for this function:

Here is the caller graph for this function:

Initialize()

void v8::internal::Scanner::Initialize

(

Utf16CharacterStream *

source

)

Definition at line 43 of file scanner.cc.

                                                     {
  source_ = source;
  // Need to capture identifiers in order to recognize "get" and "set"
  // in object literals.
  Init();
  // Skip initial whitespace allowing HTML comment ends just like
  // after a newline and scan first token.
  has_line_terminator_before_next_ = true;
  SkipWhiteSpace();
  Scan();
}

References has_line_terminator_before_next_, Init(), Scan(), SkipWhiteSpace(), and source_.

Referenced by v8::internal::Parser::ParseLazy(), v8::internal::Parser::ParseOnBackground(), and v8::internal::Parser::ParseProgram().

Here is the call graph for this function:

Here is the caller graph for this function:

INLINE()

v8::internal::Scanner::INLINE ( void   AddLiteralCharuc32 c )

inlineprivate

Definition at line 499 of file scanner.h.

                                      {
    DCHECK_NOT_NULL(next_.literal_chars);
    next_.literal_chars->AddChar(c);
  }

References DCHECK_NOT_NULL, v8::internal::Scanner::TokenDesc::literal_chars, and next_.

is_literal_contextual_keyword()

bool v8::internal::Scanner::is_literal_contextual_keyword ( Vector< const char >  keyword )

inline

Definition at line 374 of file scanner.h.

                                                                 {
    DCHECK_NOT_NULL(current_.literal_chars);
    return current_.literal_chars->is_contextual_keyword(keyword);
  }

References current_, DCHECK_NOT_NULL, v8::internal::LiteralBuffer::is_contextual_keyword(), and v8::internal::Scanner::TokenDesc::literal_chars.

Here is the call graph for this function:

is_literal_one_byte()

bool v8::internal::Scanner::is_literal_one_byte ( )

inlineprivate

Definition at line 556 of file scanner.h.

                             {
    DCHECK_NOT_NULL(current_.literal_chars);
    return current_.literal_chars->is_one_byte();
  }

References current_, DCHECK_NOT_NULL, v8::internal::LiteralBuffer::is_one_byte(), and v8::internal::Scanner::TokenDesc::literal_chars.

Referenced by CurrentSymbol(), DoubleValue(), FindSymbol(), IsGetOrSet(), and UnescapedLiteralMatches().

Here is the call graph for this function:

Here is the caller graph for this function:

is_next_contextual_keyword()

bool v8::internal::Scanner::is_next_contextual_keyword ( Vector< const char >  keyword )

inline

Definition at line 378 of file scanner.h.

                                                              {
    DCHECK_NOT_NULL(next_.literal_chars);
    return next_.literal_chars->is_contextual_keyword(keyword);
  }

References DCHECK_NOT_NULL, v8::internal::LiteralBuffer::is_contextual_keyword(), v8::internal::Scanner::TokenDesc::literal_chars, and next_.

Here is the call graph for this function:

is_next_literal_one_byte()

bool v8::internal::Scanner::is_next_literal_one_byte ( )

inlineprivate

Definition at line 574 of file scanner.h.

                                  {
    DCHECK_NOT_NULL(next_.literal_chars);
    return next_.literal_chars->is_one_byte();
  }

References DCHECK_NOT_NULL, v8::internal::LiteralBuffer::is_one_byte(), v8::internal::Scanner::TokenDesc::literal_chars, and next_.

Referenced by NextSymbol().

Here is the call graph for this function:

Here is the caller graph for this function:

IsGetOrSet()

void v8::internal::Scanner::IsGetOrSet ( bool *  is_get, bool *  is_set  )

inline

Definition at line 397 of file scanner.h.

                                              {
    if (is_literal_one_byte() &&
        literal_length() == 3 &&
        !literal_contains_escapes()) {
      const char* token =
          reinterpret_cast<const char*>(literal_one_byte_string().start());
      *is_get = strncmp(token, "get", 3) == 0;
      *is_set = !*is_get && strncmp(token, "set", 3) == 0;
    }
  }

References is_literal_one_byte(), literal_contains_escapes(), literal_length(), and literal_one_byte_string().

Here is the call graph for this function:

literal_contains_escapes()

bool v8::internal::Scanner::literal_contains_escapes ( ) const

inline

Definition at line 365 of file scanner.h.

                                        {
    Location location = current_.location;
    int source_length = (location.end_pos - location.beg_pos);
    if (current_.token == Token::STRING) {
      // Subtract delimiters.
      source_length -= 2;
    }
    return current_.literal_chars->length() != source_length;
  }

References v8::internal::Scanner::Location::beg_pos, current_, v8::internal::Scanner::Location::end_pos, v8::internal::LiteralBuffer::length(), v8::internal::Scanner::TokenDesc::literal_chars, location(), v8::internal::Scanner::TokenDesc::location, STRING, and v8::internal::Scanner::TokenDesc::token.

Referenced by IsGetOrSet(), and UnescapedLiteralMatches().

Here is the call graph for this function:

Here is the caller graph for this function:

literal_length()

int v8::internal::Scanner::literal_length ( ) const

inlineprivate

Definition at line 560 of file scanner.h.

                             {
    DCHECK_NOT_NULL(current_.literal_chars);
    return current_.literal_chars->length();
  }

References current_, DCHECK_NOT_NULL, v8::internal::LiteralBuffer::length(), and v8::internal::Scanner::TokenDesc::literal_chars.

Referenced by IsGetOrSet(), and UnescapedLiteralMatches().

Here is the call graph for this function:

Here is the caller graph for this function:

literal_one_byte_string()

Vector<const uint8_t> v8::internal::Scanner::literal_one_byte_string ( )

inlineprivate

Definition at line 548 of file scanner.h.

                                                  {
    DCHECK_NOT_NULL(current_.literal_chars);
    return current_.literal_chars->one_byte_literal();
  }

References current_, DCHECK_NOT_NULL, v8::internal::Scanner::TokenDesc::literal_chars, and v8::internal::LiteralBuffer::one_byte_literal().

Referenced by CurrentSymbol(), DoubleValue(), FindNumber(), FindSymbol(), IsGetOrSet(), and UnescapedLiteralMatches().

Here is the call graph for this function:

Here is the caller graph for this function:

literal_two_byte_string()

Vector<const uint16_t> v8::internal::Scanner::literal_two_byte_string ( )

inlineprivate

Definition at line 552 of file scanner.h.

                                                   {
    DCHECK_NOT_NULL(current_.literal_chars);
    return current_.literal_chars->two_byte_literal();
  }

References current_, DCHECK_NOT_NULL, v8::internal::Scanner::TokenDesc::literal_chars, and v8::internal::LiteralBuffer::two_byte_literal().

Referenced by CurrentSymbol(), and FindSymbol().

Here is the call graph for this function:

Here is the caller graph for this function:

location()

Location v8::internal::Scanner::location ( ) const

inline

Definition at line 356 of file scanner.h.

{ return current_.location; }

References current_, and v8::internal::Scanner::TokenDesc::location.

Referenced by v8::internal::Parser::DesugarLetBindingsInForStatement(), v8::internal::Parser::DoParseProgram(), literal_contains_escapes(), v8::internal::Parser::ParseFunctionLiteral(), v8::internal::PreParser::ParseFunctionLiteral(), v8::internal::Parser::ParseReturnStatement(), v8::internal::PreParser::ParseStatement(), v8::internal::ParserBase< Traits >::position(), v8::internal::PreParser::PreParseLazyFunction(), v8::internal::ParserTraits::ReportMessage(), v8::internal::ParserBase< Traits >::ReportMessage(), and v8::internal::Parser::SkipLazyFunctionBody().

Here is the caller graph for this function:

Next()

Token::Value v8::internal::Scanner::Next

(

)

Definition at line 219 of file scanner.cc.

                         {
  current_ = next_;
  has_line_terminator_before_next_ = false;
  has_multiline_comment_before_next_ = false;
  if (static_cast<unsigned>(c0_) <= 0x7f) {
    Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]);
    if (token != Token::ILLEGAL) {
      int pos = source_pos();
      next_.token = token;
      next_.location.beg_pos = pos;
      next_.location.end_pos = pos + 1;
      Advance();
      return current_.token;
    }
  }
  Scan();
  return current_.token;
}

References Advance(), v8::internal::Scanner::Location::beg_pos, c0_, current_, v8::internal::Scanner::Location::end_pos, has_line_terminator_before_next_, has_multiline_comment_before_next_, v8::internal::Scanner::TokenDesc::location, next_, v8::internal::one_char_tokens, Scan(), source_pos(), and v8::internal::Scanner::TokenDesc::token.

Referenced by v8::internal::ParserBase< Traits >::INLINE().

Here is the call graph for this function:

Here is the caller graph for this function:

next_literal_length()

int v8::internal::Scanner::next_literal_length ( ) const

inlineprivate

Definition at line 578 of file scanner.h.

                                  {
    DCHECK_NOT_NULL(next_.literal_chars);
    return next_.literal_chars->length();
  }

References DCHECK_NOT_NULL, v8::internal::LiteralBuffer::length(), v8::internal::Scanner::TokenDesc::literal_chars, and next_.

Here is the call graph for this function:

next_literal_one_byte_string()

Vector<const uint8_t> v8::internal::Scanner::next_literal_one_byte_string ( )

inlineprivate

Definition at line 566 of file scanner.h.

                                                       {
    DCHECK_NOT_NULL(next_.literal_chars);
    return next_.literal_chars->one_byte_literal();
  }

References DCHECK_NOT_NULL, v8::internal::Scanner::TokenDesc::literal_chars, next_, and v8::internal::LiteralBuffer::one_byte_literal().

Referenced by NextSymbol().

Here is the call graph for this function:

Here is the caller graph for this function:

next_literal_two_byte_string()

Vector<const uint16_t> v8::internal::Scanner::next_literal_two_byte_string ( )

inlineprivate

Definition at line 570 of file scanner.h.

                                                        {
    DCHECK_NOT_NULL(next_.literal_chars);
    return next_.literal_chars->two_byte_literal();
  }

References DCHECK_NOT_NULL, v8::internal::Scanner::TokenDesc::literal_chars, next_, and v8::internal::LiteralBuffer::two_byte_literal().

Referenced by NextSymbol().

Here is the call graph for this function:

Here is the caller graph for this function:

NextSymbol()

const AstRawString * v8::internal::Scanner::NextSymbol

(

AstValueFactory *

ast_value_factory

)

Definition at line 1203 of file scanner.cc.

                                                                          {
  if (is_next_literal_one_byte()) {
    return ast_value_factory->GetOneByteString(next_literal_one_byte_string());
  }
  return ast_value_factory->GetTwoByteString(next_literal_two_byte_string());
}

References v8::internal::AstValueFactory::GetOneByteString(), v8::internal::AstValueFactory::GetTwoByteString(), is_next_literal_one_byte(), next_literal_one_byte_string(), and next_literal_two_byte_string().

Referenced by v8::internal::ParserTraits::GetNextSymbol().

Here is the call graph for this function:

Here is the caller graph for this function:

octal_position()

Location v8::internal::Scanner::octal_position ( ) const

inline

Definition at line 414 of file scanner.h.

{ return octal_pos_; }

References octal_pos_.

Referenced by v8::internal::ParserBase< Traits >::CheckOctalLiteral().

Here is the caller graph for this function:

peek()

Token::Value v8::internal::Scanner::peek ( ) const

inline

Definition at line 361 of file scanner.h.

{ return next_.token; }

References next_, and v8::internal::Scanner::TokenDesc::token.

Referenced by v8::internal::ParserBase< Traits >::INLINE().

Here is the caller graph for this function:

peek_location()

Location v8::internal::Scanner::peek_location ( ) const

inline

Definition at line 363 of file scanner.h.

{ return next_.location; }

References v8::internal::Scanner::TokenDesc::location, and next_.

Referenced by v8::internal::Parser::ParseForStatement(), v8::internal::PreParser::ParseLazyFunctionLiteralBody(), v8::internal::Parser::ParseSourceElements(), v8::internal::PreParser::ParseStatement(), v8::internal::PreParser::ParseVariableDeclarations(), v8::internal::ParserBase< Traits >::peek_position(), and v8::internal::PreParser::PreParseProgram().

Here is the caller graph for this function:

PushBack()

void v8::internal::Scanner::PushBack ( uc32  ch )

inlineprivate

Definition at line 522 of file scanner.h.

                         {
    source_->PushBack(c0_);
    c0_ = ch;
  }

References c0_, v8::internal::Utf16CharacterStream::PushBack(), and source_.

Referenced by Scan(), ScanHexNumber(), ScanHtmlComment(), ScanIdentifierUnicodeEscape(), ScanLiteralUnicodeEscape(), and SkipWhiteSpace().

Here is the call graph for this function:

Here is the caller graph for this function:

Scan()

void v8::internal::Scanner::Scan ( )

private

Definition at line 411 of file scanner.cc.

                   {
  next_.literal_chars = NULL;
  Token::Value token;
  do {
    // Remember the position of the next token
    next_.location.beg_pos = source_pos();
 
    switch (c0_) {
      case ' ':
      case '\t':
        Advance();
        token = Token::WHITESPACE;
        break;
 
      case '\n':
        Advance();
        has_line_terminator_before_next_ = true;
        token = Token::WHITESPACE;
        break;
 
      case '"': case '\'':
        token = ScanString();
        break;
 
      case '<':
        // < <= << <<= <!--
        Advance();
        if (c0_ == '=') {
          token = Select(Token::LTE);
        } else if (c0_ == '<') {
          token = Select('=', Token::ASSIGN_SHL, Token::SHL);
        } else if (c0_ == '!') {
          token = ScanHtmlComment();
        } else {
          token = Token::LT;
        }
        break;
 
      case '>':
        // > >= >> >>= >>> >>>=
        Advance();
        if (c0_ == '=') {
          token = Select(Token::GTE);
        } else if (c0_ == '>') {
          // >> >>= >>> >>>=
          Advance();
          if (c0_ == '=') {
            token = Select(Token::ASSIGN_SAR);
          } else if (c0_ == '>') {
            token = Select('=', Token::ASSIGN_SHR, Token::SHR);
          } else {
            token = Token::SAR;
          }
        } else {
          token = Token::GT;
        }
        break;
 
      case '=':
        // = == === =>
        Advance();
        if (c0_ == '=') {
          token = Select('=', Token::EQ_STRICT, Token::EQ);
        } else if (c0_ == '>') {
          token = Select(Token::ARROW);
        } else {
          token = Token::ASSIGN;
        }
        break;
 
      case '!':
        // ! != !==
        Advance();
        if (c0_ == '=') {
          token = Select('=', Token::NE_STRICT, Token::NE);
        } else {
          token = Token::NOT;
        }
        break;
 
      case '+':
        // + ++ +=
        Advance();
        if (c0_ == '+') {
          token = Select(Token::INC);
        } else if (c0_ == '=') {
          token = Select(Token::ASSIGN_ADD);
        } else {
          token = Token::ADD;
        }
        break;
 
      case '-':
        // - -- --> -=
        Advance();
        if (c0_ == '-') {
          Advance();
          if (c0_ == '>' && has_line_terminator_before_next_) {
            // For compatibility with SpiderMonkey, we skip lines that
            // start with an HTML comment end '-->'.
            token = SkipSingleLineComment();
          } else {
            token = Token::DEC;
          }
        } else if (c0_ == '=') {
          token = Select(Token::ASSIGN_SUB);
        } else {
          token = Token::SUB;
        }
        break;
 
      case '*':
        // * *=
        token = Select('=', Token::ASSIGN_MUL, Token::MUL);
        break;
 
      case '%':
        // % %=
        token = Select('=', Token::ASSIGN_MOD, Token::MOD);
        break;
 
      case '/':
        // /  // /* /=
        Advance();
        if (c0_ == '/') {
          Advance();
          if (c0_ == '@' || c0_ == '#') {
            Advance();
            token = SkipSourceURLComment();
          } else {
            PushBack(c0_);
            token = SkipSingleLineComment();
          }
        } else if (c0_ == '*') {
          token = SkipMultiLineComment();
        } else if (c0_ == '=') {
          token = Select(Token::ASSIGN_DIV);
        } else {
          token = Token::DIV;
        }
        break;
 
      case '&':
        // & && &=
        Advance();
        if (c0_ == '&') {
          token = Select(Token::AND);
        } else if (c0_ == '=') {
          token = Select(Token::ASSIGN_BIT_AND);
        } else {
          token = Token::BIT_AND;
        }
        break;
 
      case '|':
        // | || |=
        Advance();
        if (c0_ == '|') {
          token = Select(Token::OR);
        } else if (c0_ == '=') {
          token = Select(Token::ASSIGN_BIT_OR);
        } else {
          token = Token::BIT_OR;
        }
        break;
 
      case '^':
        // ^ ^=
        token = Select('=', Token::ASSIGN_BIT_XOR, Token::BIT_XOR);
        break;
 
      case '.':
        // . Number
        Advance();
        if (IsDecimalDigit(c0_)) {
          token = ScanNumber(true);
        } else {
          token = Token::PERIOD;
        }
        break;
 
      case ':':
        token = Select(Token::COLON);
        break;
 
      case ';':
        token = Select(Token::SEMICOLON);
        break;
 
      case ',':
        token = Select(Token::COMMA);
        break;
 
      case '(':
        token = Select(Token::LPAREN);
        break;
 
      case ')':
        token = Select(Token::RPAREN);
        break;
 
      case '[':
        token = Select(Token::LBRACK);
        break;
 
      case ']':
        token = Select(Token::RBRACK);
        break;
 
      case '{':
        token = Select(Token::LBRACE);
        break;
 
      case '}':
        token = Select(Token::RBRACE);
        break;
 
      case '?':
        token = Select(Token::CONDITIONAL);
        break;
 
      case '~':
        token = Select(Token::BIT_NOT);
        break;
 
      default:
        if (unicode_cache_->IsIdentifierStart(c0_)) {
          token = ScanIdentifierOrKeyword();
        } else if (IsDecimalDigit(c0_)) {
          token = ScanNumber(false);
        } else if (SkipWhiteSpace()) {
          token = Token::WHITESPACE;
        } else if (c0_ < 0) {
          token = Token::EOS;
        } else {
          token = Select(Token::ILLEGAL);
        }
        break;
    }
 
    // Continue scanning for tokens as long as we're just skipping
    // whitespace.
  } while (token == Token::WHITESPACE);
 
  next_.location.end_pos = source_pos();
  next_.token = token;
}

References v8::internal::ADD, Advance(), v8::internal::AND, v8::internal::Scanner::Location::beg_pos, c0_, COMMA, v8::internal::DIV, v8::internal::Scanner::Location::end_pos, v8::internal::EQ, has_line_terminator_before_next_, v8::internal::IsDecimalDigit(), v8::internal::UnicodeCache::IsIdentifierStart(), v8::internal::Scanner::TokenDesc::literal_chars, v8::internal::Scanner::TokenDesc::location, v8::internal::MUL, next_, v8::internal::NOT, NULL, v8::internal::OR, PushBack(), ScanHtmlComment(), ScanIdentifierOrKeyword(), ScanNumber(), ScanString(), Select(), SkipMultiLineComment(), SkipSingleLineComment(), SkipSourceURLComment(), SkipWhiteSpace(), source_pos(), v8::internal::SUB, v8::internal::Scanner::TokenDesc::token, and unicode_cache_.

Referenced by Initialize(), Next(), and SeekForward().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanDecimalDigits()

void v8::internal::Scanner::ScanDecimalDigits ( )

private

Definition at line 780 of file scanner.cc.

                                {
  while (IsDecimalDigit(c0_))
    AddLiteralCharAdvance();
}

References AddLiteralCharAdvance(), c0_, and v8::internal::IsDecimalDigit().

Referenced by ScanNumber().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanEscape()

bool v8::internal::Scanner::ScanEscape ( )

private

Definition at line 681 of file scanner.cc.

                         {
  uc32 c = c0_;
  Advance();
 
  // Skip escaped newlines.
  if (unicode_cache_->IsLineTerminator(c)) {
    // Allow CR+LF newlines in multiline string literals.
    if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance();
    // Allow LF+CR newlines in multiline string literals.
    if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance();
    return true;
  }
 
  switch (c) {
    case '\'':  // fall through
    case '"' :  // fall through
    case '\\': break;
    case 'b' : c = '\b'; break;
    case 'f' : c = '\f'; break;
    case 'n' : c = '\n'; break;
    case 'r' : c = '\r'; break;
    case 't' : c = '\t'; break;
    case 'u' : {
      c = ScanHexNumber(4);
      if (c < 0) return false;
      break;
    }
    case 'v' : c = '\v'; break;
    case 'x' : {
      c = ScanHexNumber(2);
      if (c < 0) return false;
      break;
    }
    case '0' :  // fall through
    case '1' :  // fall through
    case '2' :  // fall through
    case '3' :  // fall through
    case '4' :  // fall through
    case '5' :  // fall through
    case '6' :  // fall through
    case '7' : c = ScanOctalEscape(c, 2); break;
  }
 
  // According to ECMA-262, section 7.8.4, characters not covered by the
  // above cases should be illegal, but they are commonly handled as
  // non-escaped characters by JS VMs.
  AddLiteralChar(c);
  return true;
}

References Advance(), c0_, v8::internal::IsCarriageReturn(), v8::internal::IsLineFeed(), v8::internal::UnicodeCache::IsLineTerminator(), ScanHexNumber(), ScanOctalEscape(), and unicode_cache_.

Referenced by ScanString().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanHexNumber()

uc32 v8::internal::Scanner::ScanHexNumber ( int  expected_length )

private

Definition at line 56 of file scanner.cc.

                                               {
  DCHECK(expected_length <= 4);  // prevent overflow
 
  uc32 digits[4] = { 0, 0, 0, 0 };
  uc32 x = 0;
  for (int i = 0; i < expected_length; i++) {
    digits[i] = c0_;
    int d = HexValue(c0_);
    if (d < 0) {
      // According to ECMA-262, 3rd, 7.8.4, page 18, these hex escapes
      // should be illegal, but other JS VMs just return the
      // non-escaped version of the original character.
 
      // Push back digits that we have advanced past.
      for (int j = i-1; j >= 0; j--) {
        PushBack(digits[j]);
      }
      return -1;
    }
    x = x * 16 + d;
    Advance();
  }
 
  return x;
}

References Advance(), c0_, DCHECK, v8::internal::HexValue(), and PushBack().

Referenced by ScanEscape(), and ScanIdentifierUnicodeEscape().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanHtmlComment()

Token::Value v8::internal::Scanner::ScanHtmlComment ( )

private

Definition at line 396 of file scanner.cc.

                                    {
  // Check for <!-- comments.
  DCHECK(c0_ == '!');
  Advance();
  if (c0_ == '-') {
    Advance();
    if (c0_ == '-') return SkipSingleLineComment();
    PushBack('-');  // undo Advance()
  }
  PushBack('!');  // undo Advance()
  DCHECK(c0_ == '!');
  return Token::LT;
}

References Advance(), c0_, DCHECK, PushBack(), and SkipSingleLineComment().

Referenced by Scan().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanIdentifierOrKeyword()

Token::Value v8::internal::Scanner::ScanIdentifierOrKeyword ( )

private

Definition at line 1028 of file scanner.cc.

                                            {
  DCHECK(unicode_cache_->IsIdentifierStart(c0_));
  LiteralScope literal(this);
  // Scan identifier start character.
  if (c0_ == '\\') {
    uc32 c = ScanIdentifierUnicodeEscape();
    // Only allow legal identifier start characters.
    if (c < 0 ||
        c == '\\' ||  // No recursive escapes.
        !unicode_cache_->IsIdentifierStart(c)) {
      return Token::ILLEGAL;
    }
    AddLiteralChar(c);
    return ScanIdentifierSuffix(&literal);
  }
 
  uc32 first_char = c0_;
  Advance();
  AddLiteralChar(first_char);
 
  // Scan the rest of the identifier characters.
  while (unicode_cache_->IsIdentifierPart(c0_)) {
    if (c0_ != '\\') {
      uc32 next_char = c0_;
      Advance();
      AddLiteralChar(next_char);
      continue;
    }
    // Fallthrough if no longer able to complete keyword.
    return ScanIdentifierSuffix(&literal);
  }
 
  literal.Complete();
 
  if (next_.literal_chars->is_one_byte()) {
    Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal();
    return KeywordOrIdentifierToken(chars.start(),
                                    chars.length(),
                                    harmony_scoping_,
                                    harmony_modules_,
                                    harmony_classes_);
  }
 
  return Token::IDENTIFIER;
}

References Advance(), c0_, v8::internal::Scanner::LiteralScope::Complete(), DCHECK, harmony_classes_, harmony_modules_, harmony_scoping_, v8::internal::LiteralBuffer::is_one_byte(), v8::internal::UnicodeCache::IsIdentifierPart(), v8::internal::UnicodeCache::IsIdentifierStart(), v8::internal::KeywordOrIdentifierToken(), v8::internal::Vector< T >::length(), v8::internal::Scanner::TokenDesc::literal_chars, next_, v8::internal::LiteralBuffer::one_byte_literal(), ScanIdentifierSuffix(), ScanIdentifierUnicodeEscape(), v8::internal::Vector< T >::start(), and unicode_cache_.

Referenced by Scan().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanIdentifierSuffix()

Token::Value v8::internal::Scanner::ScanIdentifierSuffix ( LiteralScope *  literal )

private

Definition at line 1075 of file scanner.cc.

                                                              {
  // Scan the rest of the identifier characters.
  while (unicode_cache_->IsIdentifierPart(c0_)) {
    if (c0_ == '\\') {
      uc32 c = ScanIdentifierUnicodeEscape();
      // Only allow legal identifier part characters.
      if (c < 0 ||
          c == '\\' ||
          !unicode_cache_->IsIdentifierPart(c)) {
        return Token::ILLEGAL;
      }
      AddLiteralChar(c);
    } else {
      AddLiteralChar(c0_);
      Advance();
    }
  }
  literal->Complete();
 
  return Token::IDENTIFIER;
}

References Advance(), c0_, v8::internal::Scanner::LiteralScope::Complete(), v8::internal::UnicodeCache::IsIdentifierPart(), ScanIdentifierUnicodeEscape(), and unicode_cache_.

Referenced by ScanIdentifierOrKeyword().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanIdentifierUnicodeEscape()

uc32 v8::internal::Scanner::ScanIdentifierUnicodeEscape ( )

private

Definition at line 892 of file scanner.cc.

                                          {
  Advance();
  if (c0_ != 'u') return -1;
  Advance();
  uc32 result = ScanHexNumber(4);
  if (result < 0) PushBack('u');
  return result;
}

References Advance(), c0_, PushBack(), and ScanHexNumber().

Referenced by ScanIdentifierOrKeyword(), and ScanIdentifierSuffix().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanLiteralUnicodeEscape()

bool v8::internal::Scanner::ScanLiteralUnicodeEscape ( )

private

Definition at line 1145 of file scanner.cc.

                                       {
  DCHECK(c0_ == '\\');
  uc32 chars_read[6] = {'\\', 'u', 0, 0, 0, 0};
  Advance();
  int i = 1;
  if (c0_ == 'u') {
    i++;
    while (i < 6) {
      Advance();
      if (!IsHexDigit(c0_)) break;
      chars_read[i] = c0_;
      i++;
    }
  }
  if (i < 6) {
    // Incomplete escape. Undo all advances and return false.
    while (i > 0) {
      i--;
      PushBack(chars_read[i]);
    }
    return false;
  }
  // Complete escape. Add all chars to current literal buffer.
  for (int i = 0; i < 6; i++) {
    AddLiteralChar(chars_read[i]);
  }
  return true;
}

References Advance(), c0_, DCHECK, v8::internal::IsHexDigit(), and PushBack().

Referenced by ScanRegExpFlags().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanNumber()

Token::Value v8::internal::Scanner::ScanNumber ( bool  seen_period )

private

Definition at line 786 of file scanner.cc.

                                               {
  DCHECK(IsDecimalDigit(c0_));  // the first digit of the number or the fraction
 
  enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL;
 
  LiteralScope literal(this);
  if (seen_period) {
    // we have already seen a decimal point of the float
    AddLiteralChar('.');
    ScanDecimalDigits();  // we know we have at least one digit
 
  } else {
    // if the first character is '0' we must check for octals and hex
    if (c0_ == '0') {
      int start_pos = source_pos();  // For reporting octal positions.
      AddLiteralCharAdvance();
 
      // either 0, 0exxx, 0Exxx, 0.xxx, a hex number, a binary number or
      // an octal number.
      if (c0_ == 'x' || c0_ == 'X') {
        // hex number
        kind = HEX;
        AddLiteralCharAdvance();
        if (!IsHexDigit(c0_)) {
          // we must have at least one hex digit after 'x'/'X'
          return Token::ILLEGAL;
        }
        while (IsHexDigit(c0_)) {
          AddLiteralCharAdvance();
        }
      } else if (harmony_numeric_literals_ && (c0_ == 'o' || c0_ == 'O')) {
        kind = OCTAL;
        AddLiteralCharAdvance();
        if (!IsOctalDigit(c0_)) {
          // we must have at least one octal digit after 'o'/'O'
          return Token::ILLEGAL;
        }
        while (IsOctalDigit(c0_)) {
          AddLiteralCharAdvance();
        }
      } else if (harmony_numeric_literals_ && (c0_ == 'b' || c0_ == 'B')) {
        kind = BINARY;
        AddLiteralCharAdvance();
        if (!IsBinaryDigit(c0_)) {
          // we must have at least one binary digit after 'b'/'B'
          return Token::ILLEGAL;
        }
        while (IsBinaryDigit(c0_)) {
          AddLiteralCharAdvance();
        }
      } else if ('0' <= c0_ && c0_ <= '7') {
        // (possible) octal number
        kind = IMPLICIT_OCTAL;
        while (true) {
          if (c0_ == '8' || c0_ == '9') {
            kind = DECIMAL;
            break;
          }
          if (c0_  < '0' || '7'  < c0_) {
            // Octal literal finished.
            octal_pos_ = Location(start_pos, source_pos());
            break;
          }
          AddLiteralCharAdvance();
        }
      }
    }
 
    // Parse decimal digits and allow trailing fractional part.
    if (kind == DECIMAL) {
      ScanDecimalDigits();  // optional
      if (c0_ == '.') {
        AddLiteralCharAdvance();
        ScanDecimalDigits();  // optional
      }
    }
  }
 
  // scan exponent, if any
  if (c0_ == 'e' || c0_ == 'E') {
    DCHECK(kind != HEX);  // 'e'/'E' must be scanned as part of the hex number
    if (kind != DECIMAL) return Token::ILLEGAL;
    // scan exponent
    AddLiteralCharAdvance();
    if (c0_ == '+' || c0_ == '-')
      AddLiteralCharAdvance();
    if (!IsDecimalDigit(c0_)) {
      // we must have at least one decimal digit after 'e'/'E'
      return Token::ILLEGAL;
    }
    ScanDecimalDigits();
  }
 
  // The source character immediately following a numeric literal must
  // not be an identifier start or a decimal digit; see ECMA-262
  // section 7.8.3, page 17 (note that we read only one decimal digit
  // if the value is 0).
  if (IsDecimalDigit(c0_) || unicode_cache_->IsIdentifierStart(c0_))
    return Token::ILLEGAL;
 
  literal.Complete();
 
  return Token::NUMBER;
}

References AddLiteralCharAdvance(), c0_, v8::internal::Scanner::LiteralScope::Complete(), DCHECK, harmony_numeric_literals_, v8::internal::IsBinaryDigit(), v8::internal::IsDecimalDigit(), v8::internal::IsHexDigit(), v8::internal::UnicodeCache::IsIdentifierStart(), v8::internal::IsOctalDigit(), octal_pos_, ScanDecimalDigits(), source_pos(), and unicode_cache_.

Referenced by Scan().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanOctalEscape()

uc32 v8::internal::Scanner::ScanOctalEscape ( uc32  c, int  length  )

private

Definition at line 734 of file scanner.cc.

                                                {
  uc32 x = c - '0';
  int i = 0;
  for (; i < length; i++) {
    int d = c0_ - '0';
    if (d < 0 || d > 7) break;
    int nx = x * 8 + d;
    if (nx >= 256) break;
    x = nx;
    Advance();
  }
  // Anything except '\0' is an octal escape sequence, illegal in strict mode.
  // Remember the position of octal escape sequences so that an error
  // can be reported later (in strict mode).
  // We don't report the error immediately, because the octal escape can
  // occur before the "use strict" directive.
  if (c != '0' || i > 0) {
    octal_pos_ = Location(source_pos() - i - 1, source_pos() - 1);
  }
  return x;
}

References Advance(), c0_, octal_pos_, and source_pos().

Referenced by ScanEscape().

Here is the call graph for this function:

Here is the caller graph for this function:

ScanRegExpFlags()

bool v8::internal::Scanner::ScanRegExpFlags

(

)

Definition at line 1175 of file scanner.cc.

                              {
  // Scan regular expression flags.
  LiteralScope literal(this);
  while (unicode_cache_->IsIdentifierPart(c0_)) {
    if (c0_ != '\\') {
      AddLiteralCharAdvance();
    } else {
      if (!ScanLiteralUnicodeEscape()) {
        break;
      }
      Advance();
    }
  }
  literal.Complete();
 
  next_.location.end_pos = source_pos() - 1;
  return true;
}

References AddLiteralCharAdvance(), Advance(), c0_, v8::internal::Scanner::LiteralScope::Complete(), v8::internal::Scanner::Location::end_pos, v8::internal::UnicodeCache::IsIdentifierPart(), v8::internal::Scanner::TokenDesc::location, next_, ScanLiteralUnicodeEscape(), source_pos(), and unicode_cache_.

Here is the call graph for this function:

ScanRegExpPattern()

bool v8::internal::Scanner::ScanRegExpPattern

(

bool

seen_equal

)

Definition at line 1098 of file scanner.cc.

                                               {
  // Scan: ('/' | '/=') RegularExpressionBody '/' RegularExpressionFlags
  bool in_character_class = false;
 
  // Previous token is either '/' or '/=', in the second case, the
  // pattern starts at =.
  next_.location.beg_pos = source_pos() - (seen_equal ? 2 : 1);
  next_.location.end_pos = source_pos() - (seen_equal ? 1 : 0);
 
  // Scan regular expression body: According to ECMA-262, 3rd, 7.8.5,
  // the scanner should pass uninterpreted bodies to the RegExp
  // constructor.
  LiteralScope literal(this);
  if (seen_equal) {
    AddLiteralChar('=');
  }
 
  while (c0_ != '/' || in_character_class) {
    if (unicode_cache_->IsLineTerminator(c0_) || c0_ < 0) return false;
    if (c0_ == '\\') {  // Escape sequence.
      AddLiteralCharAdvance();
      if (unicode_cache_->IsLineTerminator(c0_) || c0_ < 0) return false;
      AddLiteralCharAdvance();
      // If the escape allows more characters, i.e., \x??, \u????, or \c?,
      // only "safe" characters are allowed (letters, digits, underscore),
      // otherwise the escape isn't valid and the invalid character has
      // its normal meaning. I.e., we can just continue scanning without
      // worrying whether the following characters are part of the escape
      // or not, since any '/', '\\' or '[' is guaranteed to not be part
      // of the escape sequence.
 
      // TODO(896): At some point, parse RegExps more throughly to capture
      // octal esacpes in strict mode.
    } else {  // Unescaped character.
      if (c0_ == '[') in_character_class = true;
      if (c0_ == ']') in_character_class = false;
      AddLiteralCharAdvance();
    }
  }
  Advance();  // consume '/'
 
  literal.Complete();
 
  return true;
}

References AddLiteralCharAdvance(), Advance(), v8::internal::Scanner::Location::beg_pos, c0_, v8::internal::Scanner::LiteralScope::Complete(), v8::internal::Scanner::Location::end_pos, v8::internal::UnicodeCache::IsLineTerminator(), v8::internal::Scanner::TokenDesc::location, next_, source_pos(), and unicode_cache_.

Here is the call graph for this function:

ScanString()

Token::Value v8::internal::Scanner::ScanString ( )

private

Definition at line 757 of file scanner.cc.

                               {
  uc32 quote = c0_;
  Advance();  // consume quote
 
  LiteralScope literal(this);
  while (c0_ != quote && c0_ >= 0
         && !unicode_cache_->IsLineTerminator(c0_)) {
    uc32 c = c0_;
    Advance();
    if (c == '\\') {
      if (c0_ < 0 || !ScanEscape()) return Token::ILLEGAL;
    } else {
      AddLiteralChar(c);
    }
  }
  if (c0_ != quote) return Token::ILLEGAL;
  literal.Complete();
 
  Advance();  // consume quote
  return Token::STRING;
}

References Advance(), c0_, v8::internal::Scanner::LiteralScope::Complete(), v8::internal::UnicodeCache::IsLineTerminator(), ScanEscape(), STRING, and unicode_cache_.

Referenced by Scan().

Here is the call graph for this function:

Here is the caller graph for this function:

SeekForward()

void v8::internal::Scanner::SeekForward

(

int

pos

)

Definition at line 660 of file scanner.cc.

                                 {
  // After this call, we will have the token at the given position as
  // the "next" token. The "current" token will be invalid.
  if (pos == next_.location.beg_pos) return;
  int current_pos = source_pos();
  DCHECK_EQ(next_.location.end_pos, current_pos);
  // Positions inside the lookahead token aren't supported.
  DCHECK(pos >= current_pos);
  if (pos != current_pos) {
    source_->SeekForward(pos - source_->pos());
    Advance();
    // This function is only called to seek to the location
    // of the end of a function (at the "}" token). It doesn't matter
    // whether there was a line terminator in the part we skip.
    has_line_terminator_before_next_ = false;
    has_multiline_comment_before_next_ = false;
  }
  Scan();
}

References Advance(), v8::internal::Scanner::Location::beg_pos, DCHECK, DCHECK_EQ, v8::internal::Scanner::Location::end_pos, has_line_terminator_before_next_, has_multiline_comment_before_next_, v8::internal::Scanner::TokenDesc::location, next_, v8::internal::Utf16CharacterStream::pos(), Scan(), v8::internal::Utf16CharacterStream::SeekForward(), source_, and source_pos().

Referenced by v8::internal::Parser::SkipLazyFunctionBody().

Here is the call graph for this function:

Here is the caller graph for this function:

Select() [1/2]

Token::Value v8::internal::Scanner::Select ( Token::Value  tok )

inlineprivate

Definition at line 527 of file scanner.h.

                                           {
    Advance();
    return tok;
  }

References Advance().

Referenced by Scan().

Here is the call graph for this function:

Here is the caller graph for this function:

Select() [2/2]

Token::Value v8::internal::Scanner::Select ( uc32  next, Token::Value  then, Token::Value  else_  )

inlineprivate

Definition at line 532 of file scanner.h.

                                                                         {
    Advance();
    if (c0_ == next) {
      Advance();
      return then;
    } else {
      return else_;
    }
  }

References Advance(), and c0_.

Here is the call graph for this function:

SetHarmonyClasses()

void v8::internal::Scanner::SetHarmonyClasses ( bool  classes )

inline

Definition at line 444 of file scanner.h.

                                       {
    harmony_classes_ = classes;
  }

References harmony_classes_.

Referenced by v8::internal::ParserBase< Traits >::set_allow_classes().

Here is the caller graph for this function:

SetHarmonyModules()

void v8::internal::Scanner::SetHarmonyModules ( bool  modules )

inline

Definition at line 432 of file scanner.h.

                                       {
    harmony_modules_ = modules;
  }

References harmony_modules_.

Referenced by v8::internal::ParserBase< Traits >::set_allow_modules().

Here is the caller graph for this function:

SetHarmonyNumericLiterals()

void v8::internal::Scanner::SetHarmonyNumericLiterals ( bool  numeric_literals )

inline

Definition at line 438 of file scanner.h.

                                                        {
    harmony_numeric_literals_ = numeric_literals;
  }

References harmony_numeric_literals_.

Referenced by v8::internal::ParserBase< Traits >::set_allow_harmony_numeric_literals().

Here is the caller graph for this function:

SetHarmonyScoping()

void v8::internal::Scanner::SetHarmonyScoping ( bool  scoping )

inline

Definition at line 426 of file scanner.h.

                                       {
    harmony_scoping_ = scoping;
  }

References harmony_scoping_.

Referenced by v8::internal::ParserBase< Traits >::set_allow_harmony_scoping().

Here is the caller graph for this function:

SkipMultiLineComment()

Token::Value v8::internal::Scanner::SkipMultiLineComment ( )

private

Definition at line 370 of file scanner.cc.

                                         {
  DCHECK(c0_ == '*');
  Advance();
 
  while (c0_ >= 0) {
    uc32 ch = c0_;
    Advance();
    if (unicode_cache_->IsLineTerminator(ch)) {
      // Following ECMA-262, section 7.4, a comment containing
      // a newline will make the comment count as a line-terminator.
      has_multiline_comment_before_next_ = true;
    }
    // If we have reached the end of the multi-line comment, we
    // consume the '/' and insert a whitespace. This way all
    // multi-line comments are treated as whitespace.
    if (ch == '*' && c0_ == '/') {
      c0_ = ' ';
      return Token::WHITESPACE;
    }
  }
 
  // Unterminated multi-line comment.
  return Token::ILLEGAL;
}

References Advance(), c0_, DCHECK, has_multiline_comment_before_next_, v8::internal::UnicodeCache::IsLineTerminator(), and unicode_cache_.

Referenced by Scan().

Here is the call graph for this function:

Here is the caller graph for this function:

SkipSingleLineComment()

Token::Value v8::internal::Scanner::SkipSingleLineComment ( )

private

Definition at line 292 of file scanner.cc.

                                          {
  Advance();
 
  // The line terminator at the end of the line is not considered
  // to be part of the single-line comment; it is recognized
  // separately by the lexical grammar and becomes part of the
  // stream of input elements for the syntactic grammar (see
  // ECMA-262, section 7.4).
  while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) {
    Advance();
  }
 
  return Token::WHITESPACE;
}

References Advance(), c0_, v8::internal::UnicodeCache::IsLineTerminator(), and unicode_cache_.

Referenced by Scan(), ScanHtmlComment(), and SkipWhiteSpace().

Here is the call graph for this function:

Here is the caller graph for this function:

SkipSourceURLComment()

Token::Value v8::internal::Scanner::SkipSourceURLComment ( )

private

Definition at line 308 of file scanner.cc.

                                         {
  TryToParseSourceURLComment();
  while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) {
    Advance();
  }
 
  return Token::WHITESPACE;
}

References Advance(), c0_, v8::internal::UnicodeCache::IsLineTerminator(), TryToParseSourceURLComment(), and unicode_cache_.

Referenced by Scan().

Here is the call graph for this function:

Here is the caller graph for this function:

SkipWhiteSpace()

bool v8::internal::Scanner::SkipWhiteSpace ( )

private

Definition at line 252 of file scanner.cc.

                             {
  int start_position = source_pos();
 
  while (true) {
    while (true) {
      // Advance as long as character is a WhiteSpace or LineTerminator.
      // Remember if the latter is the case.
      if (unicode_cache_->IsLineTerminator(c0_)) {
        has_line_terminator_before_next_ = true;
      } else if (!unicode_cache_->IsWhiteSpace(c0_) &&
                 !IsLittleEndianByteOrderMark(c0_)) {
        break;
      }
      Advance();
    }
 
    // If there is an HTML comment end '-->' at the beginning of a
    // line (with only whitespace in front of it), we treat the rest
    // of the line as a comment. This is in line with the way
    // SpiderMonkey handles it.
    if (c0_ == '-' && has_line_terminator_before_next_) {
      Advance();
      if (c0_ == '-') {
        Advance();
        if (c0_ == '>') {
          // Treat the rest of the line as a comment.
          SkipSingleLineComment();
          // Continue skipping white space after the comment.
          continue;
        }
        PushBack('-');  // undo Advance()
      }
      PushBack('-');  // undo Advance()
    }
    // Return whether or not we skipped any characters.
    return source_pos() != start_position;
  }
}

References Advance(), c0_, has_line_terminator_before_next_, v8::internal::UnicodeCache::IsLineTerminator(), v8::internal::IsLittleEndianByteOrderMark(), v8::internal::UnicodeCache::IsWhiteSpace(), PushBack(), SkipSingleLineComment(), source_pos(), and unicode_cache_.

Referenced by Initialize(), and Scan().

Here is the call graph for this function:

Here is the caller graph for this function:

source_mapping_url()

const LiteralBuffer* v8::internal::Scanner::source_mapping_url ( ) const

inline

Definition at line 463 of file scanner.h.

                                                  {
    return &source_mapping_url_;
  }

References source_mapping_url_.

Referenced by v8::internal::Parser::HandleSourceURLComments().

Here is the caller graph for this function:

source_pos()

int v8::internal::Scanner::source_pos ( )

inlineprivate

Definition at line 616 of file scanner.h.

                   {
    return source_->pos() - kCharacterLookaheadBufferSize;
  }

References kCharacterLookaheadBufferSize, v8::internal::Utf16CharacterStream::pos(), and source_.

Referenced by Next(), Scan(), ScanNumber(), ScanOctalEscape(), ScanRegExpFlags(), ScanRegExpPattern(), SeekForward(), and SkipWhiteSpace().

Here is the call graph for this function:

Here is the caller graph for this function:

source_url()

const LiteralBuffer* v8::internal::Scanner::source_url ( ) const

inline

Definition at line 462 of file scanner.h.

{ return &source_url_; }

References source_url_.

Referenced by v8::internal::Parser::HandleSourceURLComments().

Here is the caller graph for this function:

StartLiteral()

void v8::internal::Scanner::StartLiteral ( )

inlineprivate

Definition at line 492 of file scanner.h.

                             {
    LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ?
            &literal_buffer2_ : &literal_buffer1_;
    free_buffer->Reset();
    next_.literal_chars = free_buffer;
  }

References current_, literal_buffer1_, literal_buffer2_, v8::internal::Scanner::TokenDesc::literal_chars, next_, and v8::internal::LiteralBuffer::Reset().

Referenced by v8::internal::Scanner::LiteralScope::LiteralScope().

Here is the call graph for this function:

Here is the caller graph for this function:

TerminateLiteral()

void v8::internal::Scanner::TerminateLiteral ( )

inlineprivate

Definition at line 505 of file scanner.h.

                                 {
    // Does nothing in the current implementation.
  }

Referenced by v8::internal::Scanner::LiteralScope::Complete().

Here is the caller graph for this function:

TryToParseSourceURLComment()

void v8::internal::Scanner::TryToParseSourceURLComment ( )

private

Definition at line 318 of file scanner.cc.

                                         {
  // Magic comments are of the form: //[#@]\s<name>=\s*<value>\s*.* and this
  // function will just return if it cannot parse a magic comment.
  if (!unicode_cache_->IsWhiteSpace(c0_))
    return;
  Advance();
  LiteralBuffer name;
  while (c0_ >= 0 && !unicode_cache_->IsWhiteSpaceOrLineTerminator(c0_) &&
         c0_ != '=') {
    name.AddChar(c0_);
    Advance();
  }
  if (!name.is_one_byte()) return;
  Vector<const uint8_t> name_literal = name.one_byte_literal();
  LiteralBuffer* value;
  if (name_literal == STATIC_CHAR_VECTOR("sourceURL")) {
    value = &source_url_;
  } else if (name_literal == STATIC_CHAR_VECTOR("sourceMappingURL")) {
    value = &source_mapping_url_;
  } else {
    return;
  }
  if (c0_ != '=')
    return;
  Advance();
  value->Reset();
  while (c0_ >= 0 && unicode_cache_->IsWhiteSpace(c0_)) {
    Advance();
  }
  while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) {
    // Disallowed characters.
    if (c0_ == '"' || c0_ == '\'') {
      value->Reset();
      return;
    }
    if (unicode_cache_->IsWhiteSpace(c0_)) {
      break;
    }
    value->AddChar(c0_);
    Advance();
  }
  // Allow whitespace at the end.
  while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) {
    if (!unicode_cache_->IsWhiteSpace(c0_)) {
      value->Reset();
      break;
    }
    Advance();
  }
}

References Advance(), c0_, v8::internal::UnicodeCache::IsLineTerminator(), v8::internal::UnicodeCache::IsWhiteSpace(), v8::internal::UnicodeCache::IsWhiteSpaceOrLineTerminator(), name, v8::internal::LiteralBuffer::Reset(), source_mapping_url_, source_url_, STATIC_CHAR_VECTOR, and unicode_cache_.

Referenced by SkipSourceURLComment().

Here is the call graph for this function:

Here is the caller graph for this function:

UnescapedLiteralMatches()

bool v8::internal::Scanner::UnescapedLiteralMatches ( const char *  data, int  length  )

inline

Definition at line 387 of file scanner.h.

                                                             {
    if (is_literal_one_byte() &&
        literal_length() == length &&
        !literal_contains_escapes()) {
      const char* token =
          reinterpret_cast<const char*>(literal_one_byte_string().start());
      return !strncmp(token, data, length);
    }
    return false;
  }

References is_literal_one_byte(), literal_contains_escapes(), literal_length(), and literal_one_byte_string().

Referenced by v8::internal::PreParserTraits::ExpressionFromString(), and v8::internal::PreParserTraits::GetSymbol().

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_cache()

UnicodeCache* v8::internal::Scanner::unicode_cache ( )

inline

Definition at line 411 of file scanner.h.

{ return unicode_cache_; }

References unicode_cache_.

Member Data Documentation

c0_

uc32 v8::internal::Scanner::c0_

private

Definition at line 641 of file scanner.h.

Referenced by AddLiteralCharAdvance(), Advance(), Next(), PushBack(), Scan(), ScanDecimalDigits(), ScanEscape(), ScanHexNumber(), ScanHtmlComment(), ScanIdentifierOrKeyword(), ScanIdentifierSuffix(), ScanIdentifierUnicodeEscape(), ScanLiteralUnicodeEscape(), ScanNumber(), ScanOctalEscape(), ScanRegExpFlags(), ScanRegExpPattern(), ScanString(), Select(), SkipMultiLineComment(), SkipSingleLineComment(), SkipSourceURLComment(), SkipWhiteSpace(), and TryToParseSourceURLComment().

current_

TokenDesc v8::internal::Scanner::current_

private

Definition at line 630 of file scanner.h.

Referenced by current_token(), Init(), is_literal_contextual_keyword(), is_literal_one_byte(), literal_contains_escapes(), literal_length(), literal_one_byte_string(), literal_two_byte_string(), location(), Next(), and StartLiteral().

harmony_classes_

bool v8::internal::Scanner::harmony_classes_

private

Definition at line 657 of file scanner.h.

Referenced by HarmonyClasses(), IdentifierIsFutureStrictReserved(), ScanIdentifierOrKeyword(), and SetHarmonyClasses().

harmony_modules_

bool v8::internal::Scanner::harmony_modules_

private

Definition at line 653 of file scanner.h.

Referenced by HarmonyModules(), IdentifierIsFutureStrictReserved(), ScanIdentifierOrKeyword(), and SetHarmonyModules().

harmony_numeric_literals_

bool v8::internal::Scanner::harmony_numeric_literals_

private

Definition at line 655 of file scanner.h.

Referenced by HarmonyNumericLiterals(), ScanNumber(), and SetHarmonyNumericLiterals().

harmony_scoping_

bool v8::internal::Scanner::harmony_scoping_

private

Definition at line 651 of file scanner.h.

Referenced by HarmonyScoping(), IdentifierIsFutureStrictReserved(), ScanIdentifierOrKeyword(), and SetHarmonyScoping().

has_line_terminator_before_next_

bool v8::internal::Scanner::has_line_terminator_before_next_

private

Definition at line 646 of file scanner.h.

Referenced by HasAnyLineTerminatorBeforeNext(), Initialize(), Next(), Scan(), SeekForward(), and SkipWhiteSpace().

has_multiline_comment_before_next_

bool v8::internal::Scanner::has_multiline_comment_before_next_

private

Definition at line 649 of file scanner.h.

Referenced by HasAnyLineTerminatorBeforeNext(), Next(), SeekForward(), and SkipMultiLineComment().

kCharacterLookaheadBufferSize

const int v8::internal::Scanner::kCharacterLookaheadBufferSize = 1

staticprivate

Definition at line 477 of file scanner.h.

Referenced by Init(), and source_pos().

kNoOctalLocation

const int v8::internal::Scanner::kNoOctalLocation = -1

static

Definition at line 344 of file scanner.h.

literal_buffer1_

LiteralBuffer v8::internal::Scanner::literal_buffer1_

private

Definition at line 623 of file scanner.h.

Referenced by StartLiteral().

literal_buffer2_

LiteralBuffer v8::internal::Scanner::literal_buffer2_

private

Definition at line 624 of file scanner.h.

Referenced by StartLiteral().

next_

TokenDesc v8::internal::Scanner::next_

private

Definition at line 631 of file scanner.h.

Referenced by DropLiteral(), INLINE(), is_next_contextual_keyword(), is_next_literal_one_byte(), Next(), next_literal_length(), next_literal_one_byte_string(), next_literal_two_byte_string(), peek(), peek_location(), Scan(), ScanIdentifierOrKeyword(), ScanRegExpFlags(), ScanRegExpPattern(), SeekForward(), and StartLiteral().

octal_pos_

Location v8::internal::Scanner::octal_pos_

private

Definition at line 638 of file scanner.h.

Referenced by clear_octal_position(), octal_position(), ScanNumber(), and ScanOctalEscape().

source_

Utf16CharacterStream* v8::internal::Scanner::source_

private

Definition at line 634 of file scanner.h.

Referenced by Advance(), Initialize(), PushBack(), SeekForward(), and source_pos().

source_mapping_url_

LiteralBuffer v8::internal::Scanner::source_mapping_url_

private

Definition at line 628 of file scanner.h.

Referenced by source_mapping_url(), and TryToParseSourceURLComment().

source_url_

LiteralBuffer v8::internal::Scanner::source_url_

private

Definition at line 627 of file scanner.h.

Referenced by source_url(), and TryToParseSourceURLComment().

unicode_cache_

UnicodeCache* v8::internal::Scanner::unicode_cache_

private

Definition at line 620 of file scanner.h.

Referenced by DoubleValue(), Scan(), ScanEscape(), ScanIdentifierOrKeyword(), ScanIdentifierSuffix(), ScanNumber(), ScanRegExpFlags(), ScanRegExpPattern(), ScanString(), SkipMultiLineComment(), SkipSingleLineComment(), SkipSourceURLComment(), SkipWhiteSpace(), TryToParseSourceURLComment(), and unicode_cache().

---

The documentation for this class was generated from the following files:

• /mnt/V8SourceCode/src/scanner.h
• /mnt/V8SourceCode/src/scanner.cc