[Pkg-javascript-commits] [node-acorn-jsx] 356/484: Change token structure of template literals
Bastien Roucariès
rouca at moszumanska.debian.org
Sat Aug 19 14:20:56 UTC 2017
This is an automated email from the git hooks/post-receive script.
rouca pushed a commit to branch master
in repository node-acorn-jsx.
commit 2cb3dbcb4191a9ec60e4c6ed0fc3d7ee13fd6b5d
Author: Marijn Haverbeke <marijnh at gmail.com>
Date: Thu Dec 11 14:11:39 2014 +0100
Change token structure of template literals
Fix various template parsing issues, makes tokenizer useable from outside
the parser.
Closes #169
Closes #173
---
acorn.js | 123 +++++++++++++++++--------------------
test/tests-harmony.js | 163 ++++++++++++++++++++++++++++++++++++++++++++------
2 files changed, 200 insertions(+), 86 deletions(-)
diff --git a/acorn.js b/acorn.js
index e39c326..d32eaf2 100644
--- a/acorn.js
+++ b/acorn.js
@@ -302,11 +302,12 @@
var metParenL;
- // This is used by parser for detecting if it's inside ES6
- // Template String. If it is, it should treat '$' as prefix before
- // '{expression}' and everything else as string literals.
+ // This is used by the tokenizer to track the template strings it is
+ // inside, and count the amount of open braces seen inside them, to
+ // be able to switch back to a template token when the } to match ${
+ // is encountered. It will hold an array of integers.
- var inTemplate;
+ var templates;
function initParserState() {
lastStart = lastEnd = tokPos;
@@ -409,7 +410,7 @@
var _braceR = {type: "}"}, _parenL = {type: "(", beforeExpr: true}, _parenR = {type: ")"};
var _comma = {type: ",", beforeExpr: true}, _semi = {type: ";", beforeExpr: true};
var _colon = {type: ":", beforeExpr: true}, _dot = {type: "."}, _question = {type: "?", beforeExpr: true};
- var _arrow = {type: "=>", beforeExpr: true}, _bquote = {type: "`"}, _dollarBraceL = {type: "${", beforeExpr: true};
+ var _arrow = {type: "=>", beforeExpr: true}, _template = {type: "template"}, _templateContinued = {type: "templateContinued"};
var _ellipsis = {type: "...", prefix: true, beforeExpr: true};
// Operators. These carry several kinds of properties to help the
@@ -452,8 +453,8 @@
parenL: _parenL, parenR: _parenR, comma: _comma, semi: _semi, colon: _colon,
dot: _dot, ellipsis: _ellipsis, question: _question, slash: _slash, eq: _eq,
name: _name, eof: _eof, num: _num, regexp: _regexp, string: _string,
- arrow: _arrow, bquote: _bquote, dollarBraceL: _dollarBraceL, star: _star,
- assign: _assign};
+ arrow: _arrow, template: _template, templateContinued: _templateContinued, star: _star,
+ assign: _assign};
for (var kw in keywordTypes) exports.tokTypes["_" + kw] = keywordTypes[kw];
// This is a trick taken from Esprima. It turns out that, on
@@ -548,6 +549,10 @@
var newline = /[\n\r\u2028\u2029]/;
+ function isNewLine(code) {
+ return code === 10 || code === 13 || code === 0x2028 || code == 0x2029;
+ }
+
// Matches a whole line break (where CRLF is considered a single
// line break). Used to count lines.
@@ -598,7 +603,7 @@
}
tokRegexpAllowed = true;
metParenL = 0;
- inTemplate = false;
+ templates = [];
}
// Called at the end of every token. Sets `tokEnd`, `tokVal`, and
@@ -788,25 +793,6 @@
return finishOp(code === 61 ? _eq : _prefix, 1);
}
- // Get token inside ES6 template (special rules work there).
-
- function getTemplateToken(code) {
- // '`' and '${' have special meanings, but they should follow
- // string (can be empty)
- if (tokType === _string) {
- if (code === 96) { // '`'
- ++tokPos;
- return finishToken(_bquote);
- } else
- if (code === 36 && input.charCodeAt(tokPos + 1) === 123) { // '${'
- tokPos += 2;
- return finishToken(_dollarBraceL);
- }
- }
- // anything else is considered string literal
- return readTmplString();
- }
-
function getTokenFromCode(code) {
switch (code) {
// The interpretation of a dot depends on whether it is followed
@@ -821,15 +807,23 @@
case 44: ++tokPos; return finishToken(_comma);
case 91: ++tokPos; return finishToken(_bracketL);
case 93: ++tokPos; return finishToken(_bracketR);
- case 123: ++tokPos; return finishToken(_braceL);
- case 125: ++tokPos; return finishToken(_braceR);
+ case 123:
+ ++tokPos;
+ if (templates.length) ++templates[templates.length - 1];
+ return finishToken(_braceL);
+ case 125:
+ ++tokPos;
+ if (templates.length && --templates[templates.length - 1] === 0)
+ return readTemplateString(_templateContinued);
+ else
+ return finishToken(_braceR);
case 58: ++tokPos; return finishToken(_colon);
case 63: ++tokPos; return finishToken(_question);
case 96: // '`'
if (options.ecmaVersion >= 6) {
++tokPos;
- return finishToken(_bquote, undefined, false);
+ return readTemplateString(_template);
}
case 48: // '0'
@@ -890,8 +884,6 @@
var code = input.charCodeAt(tokPos);
- if (inTemplate) return getTemplateToken(code);
-
// Identifier or keyword. '\uXXXX' sequences are allowed in
// identifiers, so '\' also dispatches to that.
if (isIdentifierStart(code) || code === 92 /* '\' */) return readWord();
@@ -1074,28 +1066,34 @@
}
}
- function readTmplString() {
- var out = "";
+ function readTemplateString(type) {
+ if (type == _templateContinued) templates.pop();
+ var out = "", start = tokPos;;
for (;;) {
- if (tokPos >= inputLen) raise(tokStart, "Unterminated string constant");
- var ch = input.charCodeAt(tokPos);
- if (ch === 96 || ch === 36 && input.charCodeAt(tokPos + 1) === 123) // '`', '${'
- return finishToken(_string, out);
- if (ch === 92) { // '\'
+ if (tokPos >= inputLen) raise(tokStart, "Unterminated template");
+ var ch = input.charAt(tokPos);
+ if (ch === "`" || ch === "$" && input.charCodeAt(tokPos + 1) === 123) { // '`', '${'
+ var raw = input.slice(start, tokPos);
+ ++tokPos;
+ if (ch == "$") { ++tokPos; templates.push(1); }
+ return finishToken(type, {cooked: out, raw: raw});
+ }
+
+ if (ch === "\\") { // '\'
out += readEscapedChar();
} else {
++tokPos;
- if (newline.test(String.fromCharCode(ch))) {
- if (ch === 13 && input.charCodeAt(tokPos) === 10) {
+ if (newline.test(ch)) {
+ if (ch === "\r" && input.charCodeAt(tokPos) === 10) {
++tokPos;
- ch = 10;
+ ch = "\n";
}
if (options.locations) {
++tokCurLine;
tokLineStart = tokPos;
}
}
- out += String.fromCharCode(ch); // '\'
+ out += ch;
}
}
}
@@ -2010,7 +2008,7 @@
node.callee = base;
node.arguments = parseExprList(_parenR, false);
return parseSubscripts(finishNode(node, "CallExpression"), start, noCalls);
- } else if (tokType === _bquote) {
+ } else if (tokType === _template) {
var node = startNodeAt(start);
node.tag = base;
node.quasi = parseTemplate();
@@ -2125,7 +2123,7 @@
case _new:
return parseNew();
- case _bquote:
+ case _template:
return parseTemplate();
default:
@@ -2149,33 +2147,24 @@
// Parse template expression.
+ function parseTemplateElement() {
+ var elem = startNode();
+ elem.value = tokVal;
+ elem.tail = input.charCodeAt(tokEnd - 1) !== 123; // '{'
+ next();
+ return finishNode(elem, "TemplateElement");
+ }
+
function parseTemplate() {
- var oldInTemplate = inTemplate;
- inTemplate = true;
var node = startNode();
node.expressions = [];
- node.quasis = [];
- next();
- for (;;) {
- var elem = startNode();
- elem.value = {cooked: tokVal, raw: input.slice(tokStart, tokEnd)};
- elem.tail = false;
- next();
- node.quasis.push(finishNode(elem, "TemplateElement"));
- if (tokType === _bquote) { // '`', end of template
- elem.tail = true;
- break;
- }
- inTemplate = false;
- expect(_dollarBraceL);
+ var curElt = parseTemplateElement();
+ node.quasis = [curElt];
+ while (!curElt.tail) {
node.expressions.push(parseExpression());
- inTemplate = true;
- // hack to include previously skipped space
- tokPos = tokEnd;
- expect(_braceR);
+ if (tokType !== _templateContinued) unexpected();
+ node.quasis.push(curElt = parseTemplateElement());
}
- inTemplate = oldInTemplate;
- next();
return finishNode(node, "TemplateLiteral");
}
diff --git a/test/tests-harmony.js b/test/tests-harmony.js
index 9180e29..834979b 100644
--- a/test/tests-harmony.js
+++ b/test/tests-harmony.js
@@ -628,8 +628,8 @@ test("`42`", {
value: {raw: "42", cooked: "42"},
tail: true,
loc: {
- start: {line: 1, column: 1},
- end: {line: 1, column: 3}
+ start: {line: 1, column: 0},
+ end: {line: 1, column:4}
}
}],
expressions: [],
@@ -674,8 +674,8 @@ test("raw`42`", {
value: {raw: "42", cooked: "42"},
tail: true,
loc: {
- start: {line: 1, column: 4},
- end: {line: 1, column: 6}
+ start: {line: 1, column: 3},
+ end: {line: 1, column: 7}
}
}],
expressions: [],
@@ -726,8 +726,8 @@ test("raw`hello ${name}`", {
value: {raw: "hello ", cooked: "hello "},
tail: false,
loc: {
- start: {line: 1, column: 4},
- end: {line: 1, column: 10}
+ start: {line: 1, column: 3},
+ end: {line: 1, column: 12}
}
},
{
@@ -735,8 +735,8 @@ test("raw`hello ${name}`", {
value: {raw: "", cooked: ""},
tail: true,
loc: {
- start: {line: 1, column: 17},
- end: {line: 1, column: 17}
+ start: {line: 1, column: 16},
+ end: {line: 1, column: 18}
}
}
],
@@ -784,8 +784,8 @@ test("`$`", {
value: {raw: "$", cooked: "$"},
tail: true,
loc: {
- start: {line: 1, column: 1},
- end: {line: 1, column: 2}
+ start: {line: 1, column: 0},
+ end: {line: 1, column: 3}
}
}],
expressions: [],
@@ -820,8 +820,8 @@ test("`\\n\\r\\b\\v\\t\\f\\\n\\\r\n`", {
value: {raw: "\\n\\r\\b\\v\\t\\f\\\n\\\r\n", cooked: "\n\r\b\u000b\t\f"},
tail: true,
loc: {
- start: {line: 1, column: 1},
- end: {line: 3, column: 0}
+ start: {line: 1, column: 0},
+ end: {line: 3, column: 1}
}
}],
expressions: [],
@@ -856,8 +856,8 @@ test("`\n\r\n`", {
value: {raw: "\n\r\n", cooked: "\n\n"},
tail: true,
loc: {
- start: {line: 1, column: 1},
- end: {line: 3, column: 0}
+ start: {line: 1, column: 0},
+ end: {line: 3, column: 1}
}
}],
expressions: [],
@@ -892,8 +892,8 @@ test("`\\u{000042}\\u0042\\x42u0\\102\\A`", {
value: {raw: "\\u{000042}\\u0042\\x42u0\\102\\A", cooked: "BBBu0BA"},
tail: true,
loc: {
- start: {line: 1, column: 1},
- end: {line: 1, column: 29}
+ start: {line: 1, column: 0},
+ end: {line: 1, column: 30}
}
}],
expressions: [],
@@ -940,8 +940,8 @@ test("new raw`42`", {
value: {raw: "42", cooked: "42"},
tail: true,
loc: {
- start: {line: 1, column: 8},
- end: {line: 1, column: 10}
+ start: {line: 1, column: 7},
+ end: {line: 1, column: 11}
}
}],
expressions: [],
@@ -976,6 +976,131 @@ test("new raw`42`", {
locations: true
});
+test("`outer${{x: {y: 10}}}bar${`nested${function(){return 1;}}endnest`}end`",{
+ type: "Program",
+ body: [
+ {
+ type: "ExpressionStatement",
+ expression: {
+ type: "TemplateLiteral",
+ expressions: [
+ {
+ type: "ObjectExpression",
+ properties: [
+ {
+ type: "Property",
+ method: false,
+ shorthand: false,
+ computed: false,
+ key: {
+ type: "Identifier",
+ name: "x"
+ },
+ value: {
+ type: "ObjectExpression",
+ properties: [
+ {
+ type: "Property",
+ method: false,
+ shorthand: false,
+ computed: false,
+ key: {
+ type: "Identifier",
+ name: "y"
+ },
+ value: {
+ type: "Literal",
+ value: 10,
+ raw: "10"
+ },
+ kind: "init"
+ }
+ ]
+ },
+ kind: "init"
+ }
+ ]
+ },
+ {
+ type: "TemplateLiteral",
+ expressions: [
+ {
+ type: "FunctionExpression",
+ id: null,
+ params: [],
+ defaults: [],
+ rest: null,
+ generator: false,
+ body: {
+ type: "BlockStatement",
+ body: [
+ {
+ type: "ReturnStatement",
+ argument: {
+ type: "Literal",
+ value: 1,
+ raw: "1"
+ }
+ }
+ ]
+ },
+ expression: false
+ }
+ ],
+ quasis: [
+ {
+ type: "TemplateElement",
+ value: {
+ cooked: "nested",
+ raw: "nested"
+ },
+ tail: false
+ },
+ {
+ type: "TemplateElement",
+ value: {
+ cooked: "endnest",
+ raw: "endnest"
+ },
+ tail: true
+ }
+ ]
+ }
+ ],
+ quasis: [
+ {
+ type: "TemplateElement",
+ value: {
+ cooked: "outer",
+ raw: "outer"
+ },
+ tail: false
+ },
+ {
+ type: "TemplateElement",
+ value: {
+ cooked: "bar",
+ raw: "bar"
+ },
+ tail: false
+ },
+ {
+ type: "TemplateElement",
+ value: {
+ cooked: "end",
+ raw: "end"
+ },
+ tail: true
+ }
+ ]
+ }
+ }
+ ]
+}, {
+ ecmaVersion: 6
+});
+
+
// ES6: Switch Case Declaration
test("switch (answer) { case 42: let t = 42; break; }", {
@@ -13959,7 +14084,7 @@ testFail("class A extends yield B { }", "Unexpected token (1:22)", {ecmaVersion:
testFail("class default", "Unexpected token (1:6)", {ecmaVersion: 6});
-testFail("`test", "Unterminated string constant (1:1)", {ecmaVersion: 6});
+testFail("`test", "Unterminated template (1:0)", {ecmaVersion: 6});
testFail("switch `test`", "Unexpected token (1:7)", {ecmaVersion: 6});
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-javascript/node-acorn-jsx.git
More information about the Pkg-javascript-commits
mailing list