179 lines
9.0 KiB
Plaintext
179 lines
9.0 KiB
Plaintext
|
/*
|
||
|
|
* Copyright (C) 2009 Google Inc. All rights reserved.
|
||
|
|
*
|
||
|
|
* Redistribution and use in source and binary forms, with or without
|
||
|
|
* modification, are permitted provided that the following conditions are
|
||
|
|
* met:
|
||
|
|
*
|
||
|
|
* * Redistributions of source code must retain the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer.
|
||
|
|
* * Redistributions in binary form must reproduce the above
|
||
|
|
* copyright notice, this list of conditions and the following disclaimer
|
||
|
|
* in the documentation and/or other materials provided with the
|
||
|
|
* distribution.
|
||
|
|
* * Neither the name of Google Inc. nor the names of its
|
||
|
|
* contributors may be used to endorse or promote products derived from
|
||
|
|
* this software without specific prior written permission.
|
||
|
|
*
|
||
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||
|
|
*/
|
||
|
|
|
||
|
|
// Generate js file as follows:
|
||
|
|
//
|
||
|
|
// re2c -isc WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js \
|
||
|
|
// | sed 's|^yy\([^:]*\)*\:|case \1:|' \
|
||
|
|
// | sed 's|[*]cursor[+][+]|this._charAt(cursor++)|' \
|
||
|
|
// | sed 's|[[*][+][+]cursor|this._charAt(++cursor)|' \
|
||
|
|
// | sed 's|[*]cursor|this._charAt(cursor)|' \
|
||
|
|
// | sed 's|yych = \*\([^;]*\)|yych = this._charAt\1|' \
|
||
|
|
// | sed 's|goto case \([^;]*\)|{ gotoCase = \1; continue; }|' \
|
||
|
|
// | sed 's|unsigned\ int|var|' \
|
||
|
|
// | sed 's|var\ yych|case 1: var yych|'
|
||
|
|
|
||
|
|
WebInspector.SourceJavaScriptTokenizer = function()
|
||
|
|
{
|
||
|
|
WebInspector.SourceTokenizer.call(this);
|
||
|
|
|
||
|
|
this._keywords = [
|
||
|
|
"null", "true", "false", "break", "case", "catch", "const", "default", "finally", "for",
|
||
|
|
"instanceof", "new", "var", "continue", "function", "return", "void", "delete", "if",
|
||
|
|
"this", "do", "while", "else", "in", "switch", "throw", "try", "typeof", "debugger",
|
||
|
|
"class", "enum", "export", "extends", "import", "super", "get", "set", "with"
|
||
|
|
].keySet();
|
||
|
|
|
||
|
|
this._lexConditions = {
|
||
|
|
DIV: 0,
|
||
|
|
NODIV: 1,
|
||
|
|
COMMENT: 2,
|
||
|
|
DSTRING: 3,
|
||
|
|
SSTRING: 4,
|
||
|
|
REGEX: 5
|
||
|
|
};
|
||
|
|
|
||
|
|
this.case_DIV = 1000;
|
||
|
|
this.case_NODIV = 1001;
|
||
|
|
this.case_COMMENT = 1002;
|
||
|
|
this.case_DSTRING = 1003;
|
||
|
|
this.case_SSTRING = 1004;
|
||
|
|
this.case_REGEX = 1005;
|
||
|
|
|
||
|
|
this.initialCondition = { lexCondition: this._lexConditions.NODIV }
|
||
|
|
this.condition = this.initialCondition;
|
||
|
|
}
|
||
|
|
|
||
|
|
WebInspector.SourceJavaScriptTokenizer.prototype = {
|
||
|
|
nextToken: function(cursor)
|
||
|
|
{
|
||
|
|
var cursorOnEnter = cursor;
|
||
|
|
var gotoCase = 1;
|
||
|
|
while (1) {
|
||
|
|
switch (gotoCase)
|
||
|
|
// Following comment is replaced with generated state machine.
|
||
|
|
/*!re2c
|
||
|
|
re2c:define:YYCTYPE = "var";
|
||
|
|
re2c:define:YYCURSOR = cursor;
|
||
|
|
re2c:define:YYGETCONDITION = "this.getLexCondition";
|
||
|
|
re2c:define:YYSETCONDITION = "this.setLexCondition";
|
||
|
|
re2c:condprefix = "case this.case_";
|
||
|
|
re2c:condenumprefix = "this._lexConditions.";
|
||
|
|
re2c:yyfill:enable = 0;
|
||
|
|
re2c:labelprefix = "case ";
|
||
|
|
re2c:indent:top = 2;
|
||
|
|
re2c:indent:string = " ";
|
||
|
|
|
||
|
|
LineComment = "//" [^\r\n]*;
|
||
|
|
CommentContent = ([^*\r\n] | ("*"+[^/*]))*;
|
||
|
|
Comment = "/*" CommentContent "*"+ "/";
|
||
|
|
CommentStart = "/*" CommentContent [\r\n];
|
||
|
|
CommentEnd = CommentContent "*"+ "/";
|
||
|
|
|
||
|
|
DecimalDigit = [0-9];
|
||
|
|
NonZeroDigit = [1-9];
|
||
|
|
OctalDigit = [0-7];
|
||
|
|
HexDigit = [0-9a-fA-F];
|
||
|
|
SignedInteger = ("+"|"-")? DecimalDigit+;
|
||
|
|
ExponentPart = ("e" | "E") SignedInteger;
|
||
|
|
DecimalIntegerLiteral = "0" | NonZeroDigit DecimalDigit*;
|
||
|
|
DecimalLiteral = DecimalIntegerLiteral "." DecimalDigit* ExponentPart? | "." DecimalDigit+ ExponentPart? | DecimalIntegerLiteral ExponentPart?;
|
||
|
|
HexIntegerLiteral = "0" ("x"|"X") HexDigit+;
|
||
|
|
OctalIntegerLiteral = "0" OctalDigit+;
|
||
|
|
NumericLiteral = DecimalLiteral | HexIntegerLiteral | OctalIntegerLiteral;
|
||
|
|
|
||
|
|
Punctuation = [\!\%\&\(\*\+\,\-\.\:\;\<\=\>\?\[\]\^\{\|\}\~] | "!=" | "!==" | "%=" | "&&" | "&=" | "*=" | "++" | "+=" | "--" | "-=" | "<<" | "<<=" | "<=" | "==" | "===" | ">=" | ">>" | ">>=" | ">>>" | ">>>=" | "^=" | "|=" | "||";
|
||
|
|
Division = "/" | "/=";
|
||
|
|
RightParen = ")";
|
||
|
|
|
||
|
|
Letter = [a-zA-Z\x80-\xFF];
|
||
|
|
UnicodeEscapeSequence = "\\u" HexDigit HexDigit HexDigit HexDigit;
|
||
|
|
|
||
|
|
IdentifierStart = Letter | "_" | "$" | UnicodeEscapeSequence;
|
||
|
|
IdentifierPart = IdentifierStart | DecimalDigit;
|
||
|
|
Identifier = IdentifierStart IdentifierPart *;
|
||
|
|
|
||
|
|
DoubleStringContent = ([^\r\n\"\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*;
|
||
|
|
SingleStringContent = ([^\r\n\'\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*;
|
||
|
|
StringLiteral = "\"" DoubleStringContent "\"" | "'" SingleStringContent "'";
|
||
|
|
DoubleStringStart = "\"" DoubleStringContent "\\" [\r\n];
|
||
|
|
DoubleStringEnd = DoubleStringContent "\"";
|
||
|
|
SingleStringStart = "'" SingleStringContent "\\" [\r\n];
|
||
|
|
SingleStringEnd = SingleStringContent "'";
|
||
|
|
|
||
|
|
BackslashSequence = "\\" [^\r\n];
|
||
|
|
RegexSet = "[" ([^\r\n*\\/] | BackslashSequence)* "]";
|
||
|
|
RegexFirstChar = [^\r\n*\\/\[\]] | BackslashSequence | RegexSet;
|
||
|
|
RegexChar = [^\r\n\\/\[\]] | BackslashSequence | RegexSet;
|
||
|
|
RegexContent = RegexChar*;
|
||
|
|
Regex = "/" RegexFirstChar RegexContent "/" [igm]*;
|
||
|
|
RegexStart = "/" RegexFirstChar RegexContent "\\";
|
||
|
|
RegexEnd = RegexContent "/" [igm]*;
|
||
|
|
|
||
|
|
<DIV,NODIV> LineComment { this.tokenType = "javascript-comment"; return cursor; }
|
||
|
|
<DIV,NODIV> Comment { this.tokenType = "javascript-comment"; return cursor; }
|
||
|
|
<DIV,NODIV> CommentStart => COMMENT { this.tokenType = "javascript-comment"; return cursor; }
|
||
|
|
<COMMENT> CommentContent => COMMENT { this.tokenType = "javascript-comment"; return cursor; }
|
||
|
|
<COMMENT> CommentEnd => NODIV { this.tokenType = "javascript-comment"; return cursor; }
|
||
|
|
|
||
|
|
<DIV,NODIV> StringLiteral { this.tokenType = "javascript-string"; return cursor; }
|
||
|
|
<DIV,NODIV> DoubleStringStart => DSTRING { this.tokenType = "javascript-string"; return cursor; }
|
||
|
|
<DSTRING> DoubleStringContent => DSTRING { this.tokenType = "javascript-string"; return cursor; }
|
||
|
|
<DSTRING> DoubleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; }
|
||
|
|
<DIV,NODIV> SingleStringStart => SSTRING { this.tokenType = "javascript-string"; return cursor; }
|
||
|
|
<SSTRING> SingleStringContent => SSTRING { this.tokenType = "javascript-string"; return cursor; }
|
||
|
|
<SSTRING> SingleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; }
|
||
|
|
|
||
|
|
<NODIV> Regex { this.tokenType = "javascript-regexp"; return cursor; }
|
||
|
|
<NODIV> RegexStart => REGEX { this.tokenType = "javascript-regexp"; return cursor; }
|
||
|
|
<REGEX> RegexContent => REGEX { this.tokenType = "javascript-regexp"; return cursor; }
|
||
|
|
<REGEX> RegexEnd => NODIV { this.tokenType = "javascript-regexp"; return cursor; }
|
||
|
|
|
||
|
|
<DIV,NODIV> NumericLiteral => DIV { this.tokenType = "javascript-number"; return cursor; }
|
||
|
|
<DIV,NODIV> Identifier => DIV
|
||
|
|
{
|
||
|
|
var token = this._line.substring(cursorOnEnter, cursor);
|
||
|
|
if (token in this._keywords)
|
||
|
|
this.tokenType = "javascript-keyword";
|
||
|
|
else
|
||
|
|
this.tokenType = "javascript-ident";
|
||
|
|
return cursor;
|
||
|
|
}
|
||
|
|
<DIV,NODIV> RightParen => DIV { this.tokenType = null; return cursor; }
|
||
|
|
<DIV,NODIV> Punctuation => NODIV { this.tokenType = null; return cursor; }
|
||
|
|
<DIV> Division => NODIV { this.tokenType = null; return cursor; }
|
||
|
|
<*> [^] { this.tokenType = null; return cursor; }
|
||
|
|
*/
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
WebInspector.SourceJavaScriptTokenizer.prototype.__proto__ = WebInspector.SourceTokenizer.prototype;
|