[Pkg-javascript-commits] [node-js-tokens] 01/02: Import Upstream version 2.0.0
Lucas Castro
lucascastro-guest at moszumanska.debian.org
Mon Oct 31 14:19:18 UTC 2016
This is an automated email from the git hooks/post-receive script.
lucascastro-guest pushed a commit to branch master
in repository node-js-tokens.
commit b11771f97a0718ed4f38a5d6f905ce365ff59768
Author: Lucas de Castro Borges <lucas at gnuabordo.com.br>
Date: Mon Oct 31 11:11:02 2016 -0300
Import Upstream version 2.0.0
---
.gitignore | 1 +
.travis.yml | 3 +
LICENSE | 21 ++
changelog.md | 82 +++++
esprima-compare.js | 94 +++++
generate-index.js | 11 +
index.js | 19 +
package.json | 30 ++
readme.md | 217 +++++++++++
regex.coffee | 154 ++++++++
test/fixtures/base64.js | 63 ++++
test/fixtures/base64.json | 63 ++++
test/fixtures/division.js | 37 ++
test/fixtures/division.json | 35 ++
test/fixtures/errors.js | 12 +
test/fixtures/errors.json | 10 +
test/fixtures/regex.js | 41 +++
test/fixtures/regex.json | 43 +++
test/index.js | 868 ++++++++++++++++++++++++++++++++++++++++++++
19 files changed, 1804 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..08b2553
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+node_modules
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..2197832
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,3 @@
+language: node_js
+node_js:
+ - "node"
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..c9a4e1b
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014, 2015, 2016 Simon Lydell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/changelog.md b/changelog.md
new file mode 100644
index 0000000..b789fdd
--- /dev/null
+++ b/changelog.md
@@ -0,0 +1,82 @@
+### Version 2.0.0 (2016-06-19) ###
+
+- Added: Support for ES2016. In other words, support for the `**` exponentiation
+ operator.
+
+These are the breaking changes:
+
+- `'**'.match(jsTokens)` no longer returns `['*', '*']`, but `['**']`.
+- `'**='.match(jsTokens)` no longer returns `['*', '*=']`, but `['**=']`.
+
+
+### Version 1.0.3 (2016-03-27) ###
+
+- Improved: Made the regex ever so slightly smaller.
+- Updated: The readme.
+
+
+### Version 1.0.2 (2015-10-18) ###
+
+- Improved: Limited npm package contents for a smaller download. Thanks to
+ @zertosh!
+
+
+### Version 1.0.1 (2015-06-20) ###
+
+- Fixed: Declared an undeclared variable.
+
+
+### Version 1.0.0 (2015-02-26) ###
+
+- Changed: Merged the 'operator' and 'punctuation' types into 'punctuator'. That
+ type is now equivalent to the Punctuator token in the ECMAScript
+ specification. (Backwards-incompatible change.)
+- Fixed: A `-` followed by a number is now correctly matched as a punctuator
+ followed by a number. It used to be matched as just a number, but there is no
+ such thing as negative number literals. (Possibly backwards-incompatible
+ change.)
+
+
+### Version 0.4.1 (2015-02-21) ###
+
+- Added: Support for the regex `u` flag.
+
+
+### Version 0.4.0 (2015-02-21) ###
+
+- Improved: `jsTokens.matchToToken` performance.
+- Added: Support for octal and binary number literals.
+- Added: Support for template strings.
+
+
+### Version 0.3.1 (2015-01-06) ###
+
+- Fixed: Support for unicode spaces. They used to be allowed in names (which is
+ very confusing), and some unicode newlines were wrongly allowed in strings and
+ regexes.
+
+
+### Version 0.3.0 (2014-12-19) ###
+
+- Changed: The `jsTokens.names` array has been replaced with the
+ `jsTokens.matchToToken` function. The capturing groups of `jsTokens` are no
+ longer part of the public API; instead use said function. See this [gist] for
+ an example. (Backwards-incompatible change.)
+- Changed: The empty string is now considered an “invalid” token, instead an
+ “empty” token (its own group). (Backwards-incompatible change.)
+- Removed: component support. (Backwards-incompatible change.)
+
+[gist]: https://gist.github.com/lydell/be49dbf80c382c473004
+
+
+### Version 0.2.0 (2014-06-19) ###
+
+- Changed: Match ES6 function arrows (`=>`) as an operator, instead of its own
+ category (“functionArrow”), for simplicity. (Backwards-incompatible change.)
+- Added: ES6 splats (`...`) are now matched as an operator (instead of three
+ punctuations). (Backwards-incompatible change.)
+
+
+### Version 0.1.0 (2014-03-08) ###
+
+- Initial release.
diff --git a/esprima-compare.js b/esprima-compare.js
new file mode 100644
index 0000000..e2d23fe
--- /dev/null
+++ b/esprima-compare.js
@@ -0,0 +1,94 @@
+// Copyright 2015 Simon Lydell
+// X11 (“MIT”) Licensed. (See LICENSE.)
+
+var fs = require("fs")
+var esprima = require("esprima")
+var jsTokens = require("./")
+
+
+var typeMap = {
+ Boolean: "name",
+ Identifier: "name",
+ Keyword: "name",
+ Null: "name",
+ Numeric: "number",
+ Punctuator: "punctuator",
+ RegularExpression: "regex",
+ String: "string"
+}
+
+function getEsprimaTokens(code) {
+ var tokens = esprima.tokenize(code, {loc: true})
+ tokens.forEach(function(token) { token.type = typeMap[token.type] })
+ return tokens
+}
+
+
+function jsTokensTokenize(string) {
+ jsTokens.lastIndex = 0
+ if (string === "") return []
+ var tokens = []
+ var match
+ while (match = jsTokens.exec(string)) {
+ tokens.push(jsTokens.matchToToken(match))
+ }
+ return tokens
+}
+
+var exclusionMap = {
+ comment: true,
+ whitespace: true
+}
+
+function getJsTokensTokens(code) {
+ return jsTokensTokenize(code)
+ .filter(function(token) { return !exclusionMap.hasOwnProperty(token.type) })
+}
+
+
+function compare(file) {
+ var code = fs.readFileSync(require.resolve(file)).toString()
+ var esprimaTokens = getEsprimaTokens(code)
+ var jsTokensTokens = getJsTokensTokens(code)
+
+ var length = Math.min(esprimaTokens.length, jsTokensTokens.length)
+ for (var index = 0; index < length; index++) {
+ var esprimaToken = esprimaTokens[index]
+ var jsTokensToken = jsTokensTokens[index]
+ if (
+ esprimaToken.type !== jsTokensToken.type ||
+ esprimaToken.value !== jsTokensToken.value
+ ) {
+ var loc = esprimaToken.loc.start
+ console.error(
+ file + ":" + loc.line + ":" + (loc.column + 1) + ": " +
+ "(token #" + (index + 1) + ")\n" +
+ " esprima: '" + esprimaToken.type + "': " + esprimaToken.value + "\n" +
+ " jsTokens: '" + jsTokensToken.type + "': " + jsTokensToken.value
+ )
+ return false
+ }
+ }
+
+ if (esprimaTokens.length !== jsTokensTokens.length) {
+ console.error(
+ file + ': Number of tokens mismatch.\n' +
+ " esprima: " + (esprimaTokens.length + 1) + "\n" +
+ " jsTokens: " + (jsTokensTokens.length + 1)
+ )
+ return false
+ }
+
+ return true
+}
+
+
+var results = process.argv.slice(2).map(compare)
+
+if (results.every(Boolean)) {
+ console.log(
+ "Comparison succeeded: esprima and jsTokens produced the same tokens!"
+ )
+} else {
+ console.error("Comparison failed.")
+}
diff --git a/generate-index.js b/generate-index.js
new file mode 100644
index 0000000..6963f4b
--- /dev/null
+++ b/generate-index.js
@@ -0,0 +1,11 @@
+// Copyright 2014 Simon Lydell
+// X11 (“MIT”) Licensed. (See LICENSE.)
+
+var fs = require("fs")
+
+require("coffee-script/register")
+var regex = require("./regex.coffee")
+
+var code = fs.readFileSync("index.js").toString()
+code = code.replace(/\/.+\/.+/, regex.toString())
+fs.writeFileSync("index.js", code)
diff --git a/index.js b/index.js
new file mode 100644
index 0000000..2e07040
--- /dev/null
+++ b/index.js
@@ -0,0 +1,19 @@
+// Copyright 2014, 2015, 2016 Simon Lydell
+// X11 (“MIT”) Licensed. (See LICENSE.)
+
+// This regex comes from regex.coffee, and is inserted here by generate-index.js
+// (run `npm run build`).
+module.exports = /((['"])(?:(?!\2|\\).|\\(?:\r\n|[\s\S]))*(\2)?|`(?:[^`\\$]|\\[\s\S]|\$(?!\{)|\$\{(?:[^{}]|\{[^}]*\}?)*\}?)*(`)?)|(\/\/.*)|(\/\*(?:[^*]|\*(?!\/))*(\*\/)?)|(\/(?!\*)(?:\[(?:(?![\]\\]).|\\.)*\]|(?![\/\]\\]).|\\.)+\/(?:(?!\s*(?:\b|[\u0080-\uFFFF$\\'"~({]|[+\-!](?!=)|\.?\d))|[gmiyu]{1,5}\b(?![\u0080-\uFFFF$\\]|\s*(?:[+\-*%&|^<>!=?({]|\/(?![\/*])))))|(0[xX][\da-fA-F]+|0[oO][0-7]+|0[bB][01]+|(?:\d*\.\d+|\d+\.?)(?:[eE][+-]?\d+)?)|((?!\d)(?:(?!\s)[$\w\u0080-\uFFFF]|\\u[\da-fA-F]{ [...]
+
+module.exports.matchToToken = function(match) {
+ var token = {type: "invalid", value: match[0]}
+ if (match[ 1]) token.type = "string" , token.closed = !!(match[3] || match[4])
+ else if (match[ 5]) token.type = "comment"
+ else if (match[ 6]) token.type = "comment", token.closed = !!match[7]
+ else if (match[ 8]) token.type = "regex"
+ else if (match[ 9]) token.type = "number"
+ else if (match[10]) token.type = "name"
+ else if (match[11]) token.type = "punctuator"
+ else if (match[12]) token.type = "whitespace"
+ return token
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..35b0d86
--- /dev/null
+++ b/package.json
@@ -0,0 +1,30 @@
+{
+ "name": "js-tokens",
+ "version": "2.0.0",
+ "author": "Simon Lydell",
+ "license": "MIT",
+ "description": "A regex that tokenizes JavaScript.",
+ "keywords": [
+ "JavaScript",
+ "js",
+ "token",
+ "tokenize",
+ "regex"
+ ],
+ "files": [
+ "index.js"
+ ],
+ "repository": "lydell/js-tokens",
+ "scripts": {
+ "test": "mocha --ui tdd",
+ "esprima-compare": "node esprima-compare ./index.js everything.js/es5.js",
+ "build": "node generate-index.js",
+ "dev": "npm run build && npm test"
+ },
+ "devDependencies": {
+ "coffee-script": "~1.10.0",
+ "esprima": "^2.7.2",
+ "everything.js": "^1.0.3",
+ "mocha": "^2.5.3"
+ }
+}
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..68e470f
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,217 @@
+Overview [](https://travis-ci.org/lydell/js-tokens)
+========
+
+A regex that tokenizes JavaScript.
+
+```js
+var jsTokens = require("js-tokens")
+
+var jsString = "var foo=opts.foo;\n..."
+
+jsString.match(jsTokens)
+// ["var", " ", "foo", "=", "opts", ".", "foo", ";", "\n", ...]
+```
+
+
+Installation
+============
+
+`npm install js-tokens`
+
+```js
+var jsTokens = require("js-tokens")
+```
+
+
+Usage
+=====
+
+### `jsTokens` ###
+
+A regex with the `g` flag that matches JavaScript tokens.
+
+The regex _always_ matches, even invalid JavaScript and the empty string.
+
+The next match is always directly after the previous.
+
+### `var token = jsTokens.matchToToken(match)` ###
+
+Takes a `match` returned by `jsTokens.exec(string)`, and returns a `{type:
+String, value: String}` object. The following types are available:
+
+- string
+- comment
+- regex
+- number
+- name
+- punctuator
+- whitespace
+- invalid
+
+Multi-line comments and strings also have a `closed` property indicating if the
+token was closed or not (see below).
+
+Comments and strings both come in several flavors. To distinguish them, check if
+the token starts with `//`, `/*`, `'`, `"` or `` ` ``.
+
+Names are ECMAScript IdentifierNames, that is, including both identifiers and
+keywords. You may use [is-keyword-js] to tell them apart.
+
+Whitespace includes both line terminators and other whitespace.
+
+For example usage, please see this [gist].
+
+[is-keyword-js]: https://github.com/crissdev/is-keyword-js
+[gist]: https://gist.github.com/lydell/be49dbf80c382c473004
+
+
+ECMAScript support
+==================
+
+The intention is to always support the latest stable ECMAScript version.
+
+If adding support for a newer version requires changes, a new version with a
+major verion bump will be released.
+
+Currently, [ECMAScript 2016] is supported.
+
+[ECMAScript 2016]: http://www.ecma-international.org/ecma-262/7.0/index.html
+
+
+Invalid code handling
+=====================
+
+Unterminated strings are still matched as strings. JavaScript strings cannot
+contain (unescaped) newlines, so unterminated strings simply end at the end of
+the line. Unterminated template strings can contain unescaped newlines, though,
+so they go on to the end of input.
+
+Unterminated multi-line comments are also still matched as comments. They
+simply go on to the end of the input.
+
+Unterminated regex literals are likely matched as division and whatever is
+inside the regex.
+
+Invalid ASCII characters have their own capturing group.
+
+Invalid non-ASCII characters are treated as names, to simplify the matching of
+names (except unicode spaces which are treated as whitespace).
+
+Regex literals may contain invalid regex syntax. They are still matched as
+regex literals. They may also contain repeated regex flags, to keep the regex
+simple.
+
+Strings may contain invalid escape sequences.
+
+
+Limitations
+===========
+
+Tokenizing JavaScript using regexes—in fact, _one single regex_—won’t be
+perfect. But that’s not the point either.
+
+You may compare jsTokens with [esprima] by using `esprima-compare.js`.
+See `npm run esprima-compare`!
+
+[esprima]: http://esprima.org/
+
+### Template string interpolation ###
+
+Template strings are matched as single tokens, from the starting `` ` `` to the
+ending `` ` ``, including interpolations (whose tokens are not matched
+individually).
+
+Matching template string interpolations requires recursive balancing of `{` and
+`}`—something that JavaScript regexes cannot do. Only one level of nesting is
+supported.
+
+### Division and regex literals collision ###
+
+Consider this example:
+
+```js
+var g = 9.82
+var number = bar / 2/g
+
+var regex = / 2/g
+```
+
+A human can easily understand that in the `number` line we’re dealing with
+division, and in the `regex` line we’re dealing with a regex literal. How come?
+Because humans can look at the whole code to put the `/` characters in context.
+A JavaScript regex cannot. It only sees forwards.
+
+When the `jsTokens` regex scans throught the above, it will see the following
+at the end of both the `number` and `regex` rows:
+
+```js
+/ 2/g
+```
+
+It is then impossible to know if that is a regex literal, or part of an
+expression dealing with division.
+
+Here is a similar case:
+
+```js
+foo /= 2/g
+foo(/= 2/g)
+```
+
+The first line divides the `foo` variable with `2/g`. The second line calls the
+`foo` function with the regex literal `/= 2/g`. Again, since `jsTokens` only
+sees forwards, it cannot tell the two cases apart.
+
+There are some cases where we _can_ tell division and regex literals apart,
+though.
+
+First off, we have the simple cases where there’s only one slash in the line:
+
+```js
+var foo = 2/g
+foo /= 2
+```
+
+Regex literals cannot contain newlines, so the above cases are correctly
+identified as division. Things are only problematic when there are more than
+one non-comment slash in a single line.
+
+Secondly, not every character is a valid regex flag.
+
+```js
+var number = bar / 2/e
+```
+
+The above example is also correctly identified as division, because `e` is not a
+valid regex flag. I initially wanted to future-proof by allowing `[a-zA-Z]*`
+(any letter) as flags, but it is not worth it since it increases the amount of
+ambigous cases. So only the standard `g`, `m`, `i`, `y` and `u` flags are
+allowed. This means that the above example will be identified as division as
+long as you don’t rename the `e` variable to some permutation of `gmiyu` 1 to 5
+characters long.
+
+Lastly, we can look _forward_ for information.
+
+- If the token following what looks like a regex literal is not valid after a
+ regex literal, but is valid in a division expression, then the regex literal
+ is treated as division instead. For example, a flagless regex cannot be
+ followed by a string, number or name, but all of those three can be the
+ denominator of a division.
+- Generally, if what looks like a regex literal is followed by an operator, the
+ regex literal is treated as division instead. This is because regexes are
+ seldomly used with operators (such as `+`, `*`, `&&` and `==`), but division
+ could likely be part of such an expression.
+
+Please consult the regex source and the test cases for precise information on
+when regex or division is matched (should you need to know). In short, you
+could sum it up as:
+
+If the end of a statement looks like a regex literal (even if it isn’t), it
+will be treated as one. Otherwise it should work as expected (if you write sane
+code).
+
+
+License
+=======
+
+[The X11 (“MIT”) License](LICENSE).
diff --git a/regex.coffee b/regex.coffee
new file mode 100644
index 0000000..751586c
--- /dev/null
+++ b/regex.coffee
@@ -0,0 +1,154 @@
+# Copyright 2014, 2015, 2016 Simon Lydell
+# X11 (“MIT”) Licensed. (See LICENSE.)
+
+# <http://www.ecma-international.org/ecma-262/7.0/index.html#sec-ecmascript-language-lexical-grammar>
+
+# Don’t worry, you don’t need to know CoffeeScript. It is only used for its
+# readable regex syntax. Everything else is done in JavaScript in index.js.
+
+module.exports = ///
+ ( # <string>
+ ([ ' " ])
+ (?:
+ (?! \2 | \\ ).
+ |
+ \\(?: \r\n | [\s\S] )
+ )*
+ (\2)?
+ |
+ `
+ (?:
+ [^ ` \\ $ ]
+ |
+ \\[\s\S]
+ |
+ \$(?!\{)
+ |
+ \$\{
+ (?:
+ [^{}]
+ |
+ \{ [^}]* \}?
+ )*
+ \}?
+ )*
+ (`)?
+ )
+ |
+ ( # <comment>
+ //.*
+ )
+ |
+ ( # <comment>
+ /\*
+ (?:
+ [^*]
+ |
+ \*(?!/)
+ )*
+ ( \*/ )?
+ )
+ |
+ ( # <regex>
+ /(?!\*)
+ (?:
+ \[
+ (?:
+ (?![ \] \\ ]).
+ |
+ \\.
+ )*
+ \]
+ |
+ (?![ / \] \\ ]).
+ |
+ \\.
+ )+
+ /
+ (?:
+ (?!
+ \s*
+ (?:
+ \b
+ |
+ [ \u0080-\uFFFF $ \\ ' " ~ ( { ]
+ |
+ [ + \- ! ](?!=)
+ |
+ \.?\d
+ )
+ )
+ |
+ [ g m i y u ]{1,5} \b
+ (?!
+ [ \u0080-\uFFFF $ \\ ]
+ |
+ \s*
+ (?:
+ [ + \- * % & | ^ < > ! = ? ( { ]
+ |
+ /(?! [ / * ] )
+ )
+ )
+ )
+ )
+ |
+ ( # <number>
+ 0[xX][ \d a-f A-F ]+
+ |
+ 0[oO][0-7]+
+ |
+ 0[bB][01]+
+ |
+ (?:
+ \d*\.\d+
+ |
+ \d+\.? # Support one trailing dot for integers only.
+ )
+ (?: [eE][+-]?\d+ )?
+ )
+ |
+ ( # <name>
+ # See <http://mathiasbynens.be/notes/javascript-identifiers>.
+ (?!\d)
+ (?:
+ (?!\s)[ $ \w \u0080-\uFFFF ]
+ |
+ \\u[ \d a-f A-F ]{4}
+ |
+ \\u\{[ \d a-f A-F ]{1,6}\}
+ )+
+ )
+ |
+ ( # <punctuator>
+ -- | \+\+
+ |
+ && | \|\|
+ |
+ =>
+ |
+ \.{3}
+ |
+ (?:
+ [ + \- / % & | ^ ]
+ |
+ \*{1,2}
+ |
+ <{1,2} | >{1,3}
+ |
+ !=? | ={1,2}
+ )=?
+ |
+ [ ? ~ . , : ; [ \] ( ) { } ]
+ )
+ |
+ ( # <whitespace>
+ \s+
+ )
+ |
+ ( # <invalid>
+ ^$ # Empty.
+ |
+ [\s\S] # Catch-all rule for anything not matched by the above.
+ )
+///g
diff --git a/test/fixtures/base64.js b/test/fixtures/base64.js
new file mode 100644
index 0000000..22c2bb6
--- /dev/null
+++ b/test/fixtures/base64.js
@@ -0,0 +1,63 @@
+/*
+ * https://github.com/davidchambers/Base64.js
+ */
+;(function () {
+
+ var object = typeof exports != 'undefined' ? exports : this; // #8: web workers
+ var chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';
+
+ function InvalidCharacterError(message) {
+ this.message = message;
+ }
+ InvalidCharacterError.prototype = new Error;
+ InvalidCharacterError.prototype.name = 'InvalidCharacterError';
+
+ // encoder
+ // [https://gist.github.com/999166] by [https://github.com/nignag]
+ object.btoa || (
+ object.btoa = function (input) {
+ for (
+ // initialize result and counter
+ var block, charCode, idx = 0, map = chars, output = '';
+ // if the next input index does not exist:
+ // change the mapping table to "="
+ // check if d has no fractional digits
+ input.charAt(idx | 0) || (map = '=', idx % 1);
+ // "8 - idx % 1 * 8" generates the sequence 2, 4, 6, 8
+ output += map.charAt(63 & block >> 8 - idx % 1 * 8)
+ ) {
+ charCode = input.charCodeAt(idx += 3/4);
+ if (charCode > 0xFF) {
+ throw new InvalidCharacterError("'btoa' failed: The string to be encoded contains characters outside of the Latin1 range.");
+ }
+ block = block << 8 | charCode;
+ }
+ return output;
+ });
+
+ // decoder
+ // [https://gist.github.com/1020396] by [https://github.com/atk]
+ object.atob || (
+ object.atob = function (input) {
+ input = input.replace(/=+$/, '')
+ if (input.length % 4 == 1) {
+ throw new InvalidCharacterError("'atob' failed: The string to be decoded is not correctly encoded.");
+ }
+ for (
+ // initialize result and counters
+ var bc = 0, bs, buffer, idx = 0, output = '';
+ // get next character
+ buffer = input.charAt(idx++);
+ // character found in table? initialize bit storage and add its ascii value;
+ ~buffer && (bs = bc % 4 ? bs * 64 + buffer : buffer,
+ // and if not first of each 4 characters,
+ // convert the first 8 bits to one ascii character
+ bc++ % 4) ? output += String.fromCharCode(255 & bs >> (-2 * bc & 6)) : 0
+ ) {
+ // try to find character in table (0-63, not found => -1)
+ buffer = chars.indexOf(buffer);
+ }
+ return output;
+ });
+
+}());
diff --git a/test/fixtures/base64.json b/test/fixtures/base64.json
new file mode 100644
index 0000000..fd3982a
--- /dev/null
+++ b/test/fixtures/base64.json
@@ -0,0 +1,63 @@
+[
+"/*\n * https://github.com/davidchambers/Base64.js\n */","\n",
+";","(","function"," ","(",")"," ","{","\n\n ",
+
+"var"," ","object"," ","="," ","typeof"," ","exports"," ","!="," ","'undefined'"," ","?"," ","exports"," ",":"," ","this",";"," ","// #8: web workers","\n ",
+"var"," ","chars"," ","="," ","'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='",";","\n\n ",
+
+"function"," ","InvalidCharacterError","(","message",")"," ","{","\n ",
+"this",".","message"," ","="," ","message",";","\n ",
+"}","\n ",
+"InvalidCharacterError",".","prototype"," ","="," ","new"," ","Error",";","\n ",
+"InvalidCharacterError",".","prototype",".","name"," ","="," ","'InvalidCharacterError'",";","\n\n ",
+
+"// encoder","\n ",
+"// [https://gist.github.com/999166] by [https://github.com/nignag]","\n ",
+"object",".","btoa"," ","||"," ","(","\n ",
+"object",".","btoa"," ","="," ","function"," ","(","input",")"," ","{","\n ",
+"for"," ","(","\n ",
+"// initialize result and counter","\n ",
+"var"," ","block",","," ","charCode",","," ","idx"," ","="," ","0",","," ","map"," ","="," ","chars",","," ","output"," ","="," ","''",";","\n ",
+"// if the next input index does not exist:","\n ",
+"// change the mapping table to \"=\"","\n ",
+"// check if d has no fractional digits","\n ",
+"input",".","charAt","(","idx"," ","|"," ","0",")"," ","||"," ","(","map"," ","="," ","'='",","," ","idx"," ","%"," ","1",")",";","\n ",
+"// \"8 - idx % 1 * 8\" generates the sequence 2, 4, 6, 8","\n ",
+"output"," ","+="," ","map",".","charAt","(","63"," ","&"," ","block"," ",">>"," ","8"," ","-"," ","idx"," ","%"," ","1"," ","*"," ","8",")","\n ",
+")"," ","{","\n ",
+"charCode"," ","="," ","input",".","charCodeAt","(","idx"," ","+="," ","3","/","4",")",";","\n ",
+"if"," ","(","charCode"," ",">"," ","0xFF",")"," ","{","\n ",
+"throw"," ","new"," ","InvalidCharacterError","(","\"'btoa' failed: The string to be encoded contains characters outside of the Latin1 range.\"",")",";","\n ",
+"}","\n ",
+"block"," ","="," ","block"," ","<<"," ","8"," ","|"," ","charCode",";","\n ",
+"}","\n ",
+"return"," ","output",";","\n ",
+"}",")",";","\n\n ",
+
+"// decoder","\n ",
+"// [https://gist.github.com/1020396] by [https://github.com/atk]","\n ",
+"object",".","atob"," ","||"," ","(","\n ",
+"object",".","atob"," ","="," ","function"," ","(","input",")"," ","{","\n ",
+"input"," ","="," ","input",".","replace","(","/=+$/",","," ","''",")","\n ",
+"if"," ","(","input",".","length"," ","%"," ","4"," ","=="," ","1",")"," ","{","\n ",
+"throw"," ","new"," ","InvalidCharacterError","(","\"'atob' failed: The string to be decoded is not correctly encoded.\"",")",";","\n ",
+"}","\n ",
+"for"," ","(","\n ",
+"// initialize result and counters","\n ",
+"var"," ","bc"," ","="," ","0",","," ","bs",","," ","buffer",","," ","idx"," ","="," ","0",","," ","output"," ","="," ","''",";","\n ",
+"// get next character","\n ",
+"buffer"," ","="," ","input",".","charAt","(","idx","++",")",";","\n ",
+"// character found in table? initialize bit storage and add its ascii value;","\n ",
+"~","buffer"," ","&&"," ","(","bs"," ","="," ","bc"," ","%"," ","4"," ","?"," ","bs"," ","*"," ","64"," ","+"," ","buffer"," ",":"," ","buffer",",","\n ",
+"// and if not first of each 4 characters,","\n ",
+"// convert the first 8 bits to one ascii character","\n ",
+"bc","++"," ","%"," ","4",")"," ","?"," ","output"," ","+="," ","String",".","fromCharCode","(","255"," ","&"," ","bs"," ",">>"," ","(","-","2"," ","*"," ","bc"," ","&"," ","6",")",")"," ",":"," ","0","\n ",
+")"," ","{","\n ",
+"// try to find character in table (0-63, not found => -1)","\n ",
+"buffer"," ","="," ","chars",".","indexOf","(","buffer",")",";","\n ",
+"}","\n ",
+"return"," ","output",";","\n ",
+"}",")",";","\n\n",
+
+"}","(",")",")",";","\n"
+]
diff --git a/test/fixtures/division.js b/test/fixtures/division.js
new file mode 100644
index 0000000..89294a4
--- /dev/null
+++ b/test/fixtures/division.js
@@ -0,0 +1,37 @@
+foo = 1/a/2
+foo = 1/a/
+ 2
+foo = 1/a/(b+1)
+foo = 1/a/~bar.indexOf(baz)
+foo = 1/a/+str
+foo = 1/a/-1
+foo = 1/a/-bar
+foo = 1/a/--bar
+foo = 1/a/
+ --bar
+foo = 1/a/ ++bar
+
+foo = 1/a/g+1
+foo = 1/a/g - 1
+foo = 1/a/g*1
+foo = 1/a/g/1
+foo = 1/a/g
+ /1
+
+if (1/a/g && foo) {}
+if (1/a/g
+ && foo) {}
+if (1/a/g || foo) {}
+if (1/a/g === 3) {}
+
+foo = 1/a/g ? true : false
+
+nan = 1/a/''
+nan = 1/a/ ""
+
+foo = 1/a/goo
+foo = 1/a/iff
+foo = 1/a/igor
+foo = 1/a/moo
+foo = 1/a/yüm
+foo = 1/a/imgyp
diff --git a/test/fixtures/division.json b/test/fixtures/division.json
new file mode 100644
index 0000000..7e87be9
--- /dev/null
+++ b/test/fixtures/division.json
@@ -0,0 +1,35 @@
+[
+"foo"," ","="," ","1","/","a","/","2","\n",
+"foo"," ","="," ","1","/","a","/","\n ","2","\n",
+"foo"," ","="," ","1","/","a","/","(","b","+","1",")","\n",
+"foo"," ","="," ","1","/","a","/","~","bar",".","indexOf","(","baz",")","\n",
+"foo"," ","="," ","1","/","a","/","+","str","\n",
+"foo"," ","="," ","1","/","a","/","-","1","\n",
+"foo"," ","="," ","1","/","a","/","-","bar","\n",
+"foo"," ","="," ","1","/","a","/","--","bar","\n",
+"foo"," ","="," ","1","/","a","/","\n ","--","bar","\n",
+"foo"," ","="," ","1","/","a","/"," ","++","bar","\n\n",
+
+"foo"," ","="," ","1","/","a","/","g","+","1","\n",
+"foo"," ","="," ","1","/","a","/","g"," ","-"," ","1","\n",
+"foo"," ","="," ","1","/","a","/","g","*","1","\n",
+"foo"," ","="," ","1","/","a","/","g","/","1","\n",
+"foo"," ","="," ","1","/","a","/","g","\n ","/","1","\n\n",
+
+"if"," ","(","1","/","a","/","g"," ","&&"," ","foo",")"," ","{","}","\n",
+"if"," ","(","1","/","a","/","g","\n ","&&"," ","foo",")"," ","{","}","\n",
+"if"," ","(","1","/","a","/","g"," ","||"," ","foo",")"," ","{","}","\n",
+"if"," ","(","1","/","a","/","g"," ","==="," ","3",")"," ","{","}","\n\n",
+
+"foo"," ","="," ","1","/","a","/","g"," ","?"," ","true"," ",":"," ","false","\n\n",
+
+"nan"," ","="," ","1","/","a","/","''","\n",
+"nan"," ","="," ","1","/","a","/"," ","\"\"","\n\n",
+
+"foo"," ","="," ","1","/","a","/","goo","\n",
+"foo"," ","="," ","1","/","a","/","iff","\n",
+"foo"," ","="," ","1","/","a","/","igor","\n",
+"foo"," ","="," ","1","/","a","/","moo","\n",
+"foo"," ","="," ","1","/","a","/","yüm","\n",
+"foo"," ","="," ","1","/","a","/","imgyp","\n"
+]
diff --git a/test/fixtures/errors.js b/test/fixtures/errors.js
new file mode 100644
index 0000000..0ccb6c2
--- /dev/null
+++ b/test/fixtures/errors.js
@@ -0,0 +1,12 @@
+var multiLineString = '\
+While I was writing along\
+I suddenly forgot to
+add a backslash @ the end of a line'
+
+var unterminatedRegex = /3?2:1
+
+# Oops, forgot that this isn’t coffee-script.
+
+/* Let’s try a valid JavaScript comment instead
+
+var string='Pity I forgot to close it, though'
diff --git a/test/fixtures/errors.json b/test/fixtures/errors.json
new file mode 100644
index 0000000..6806781
--- /dev/null
+++ b/test/fixtures/errors.json
@@ -0,0 +1,10 @@
+[
+"var"," ","multiLineString"," ","="," ","'\\\nWhile I was writing along\\\nI suddenly forgot to","\n",
+"add"," ","a"," ","backslash"," ","@"," ","the"," ","end"," ","of"," ","a"," ","line","'","\n\n",
+
+"var"," ","unterminatedRegex"," ","="," ","/","3","?","2",":","1","\n\n",
+
+"#"," ","Oops",","," ","forgot"," ","that"," ","this"," ","isn’t"," ","coffee","-","script",".","\n\n",
+
+"/* Let’s try a valid JavaScript comment instead\n\nvar string='Pity I forgot to close it, though'\n"
+]
diff --git a/test/fixtures/regex.js b/test/fixtures/regex.js
new file mode 100644
index 0000000..83defa4
--- /dev/null
+++ b/test/fixtures/regex.js
@@ -0,0 +1,41 @@
+foo(/a/)
+foo(/a/g)
+foo( /a/ )
+foo( /a/g )
+foo(/a/, bar)
+foo(/a/g, bar)
+foo(/a/, /a/g, /b/g, /b/)
+
+arr = [/a/]
+arr = [/a/g]
+arr = [ /a/ ]
+arr = [ /a/g ]
+
+obj = {re: /a/}
+obj = {re: /a/g}
+obj = { re: /a/ }
+obj = { re: /a/g }
+
+re = foo ? /a/ : RegExp(bar)
+re = foo ? /a/g : RegExp(bar)
+
+/a/.exec(foo)
+/a/g.exec(foo)
+
+foo = (1/2) + /a/.exec(bar)[0]
+foo = (1/2) + /a/g.exec(bar)[0]
+
+re = /a/// comment
+re = /a/g// comment
+re = /a//* comment */
+re = /a/g/* comment */
+
+re = /a/ // comment
+re = /a/g // comment
+re = /a/ /* comment */
+re = /a/g /* comment */
+
+silly = /a/ ? true : false
+if (/a/ == "/a/") {}
+if (/a/ && foo) {}
+if (/a/ || foo) {}
diff --git a/test/fixtures/regex.json b/test/fixtures/regex.json
new file mode 100644
index 0000000..154c641
--- /dev/null
+++ b/test/fixtures/regex.json
@@ -0,0 +1,43 @@
+[
+"foo","(","/a/",")","\n",
+"foo","(","/a/g",")","\n",
+"foo","("," ","/a/"," ",")","\n",
+"foo","("," ","/a/g"," ",")","\n",
+"foo","(","/a/",","," ","bar",")","\n",
+"foo","(","/a/g",","," ","bar",")","\n",
+"foo","(","/a/",","," ","/a/g",","," ","/b/g",","," ","/b/",")","\n\n",
+
+"arr"," ","="," ","[","/a/","]","\n",
+"arr"," ","="," ","[","/a/g","]","\n",
+"arr"," ","="," ","["," ","/a/"," ","]","\n",
+"arr"," ","="," ","["," ","/a/g"," ","]","\n\n",
+
+"obj"," ","="," ","{","re",":"," ","/a/","}","\n",
+"obj"," ","="," ","{","re",":"," ","/a/g","}","\n",
+"obj"," ","="," ","{"," ","re",":"," ","/a/"," ","}","\n",
+"obj"," ","="," ","{"," ","re",":"," ","/a/g"," ","}","\n\n",
+
+"re"," ","="," ","foo"," ","?"," ","/a/"," ",":"," ","RegExp","(","bar",")","\n",
+"re"," ","="," ","foo"," ","?"," ","/a/g"," ",":"," ","RegExp","(","bar",")","\n\n",
+
+"/a/",".","exec","(","foo",")","\n",
+"/a/g",".","exec","(","foo",")","\n\n",
+
+"foo"," ","="," ","(","1","/","2",")"," ","+"," ","/a/",".","exec","(","bar",")","[","0","]","\n",
+"foo"," ","="," ","(","1","/","2",")"," ","+"," ","/a/g",".","exec","(","bar",")","[","0","]","\n\n",
+
+"re"," ","="," ","/a/","// comment","\n",
+"re"," ","="," ","/a/g","// comment","\n",
+"re"," ","="," ","/a/","/* comment */","\n",
+"re"," ","="," ","/a/g","/* comment */","\n\n",
+
+"re"," ","="," ","/a/"," ","// comment","\n",
+"re"," ","="," ","/a/g"," ","// comment","\n",
+"re"," ","="," ","/a/"," ","/* comment */","\n",
+"re"," ","="," ","/a/g"," ","/* comment */","\n\n",
+
+"silly"," ","="," ","/a/"," ","?"," ","true"," ",":"," ","false","\n",
+"if"," ","(","/a/"," ","=="," ","\"/a/\"",")"," ","{","}","\n",
+"if"," ","(","/a/"," ","&&"," ","foo",")"," ","{","}","\n",
+"if"," ","(","/a/"," ","||"," ","foo",")"," ","{","}","\n"
+]
diff --git a/test/index.js b/test/index.js
new file mode 100644
index 0000000..392f5ed
--- /dev/null
+++ b/test/index.js
@@ -0,0 +1,868 @@
+// Copyright 2014, 2015, 2016 Simon Lydell
+// X11 (“MIT”) Licensed. (See LICENSE.)
+
+var fs = require("fs")
+var util = require("util")
+var assert = require("assert")
+var jsTokens = require("../")
+
+
+suite("jsTokens", function() {
+
+ test("is a regex", function() {
+ assert(util.isRegExp(jsTokens))
+ })
+
+})
+
+
+suite("jsTokens.matchToToken", function() {
+
+ test("is a function", function() {
+ assert.equal(typeof jsTokens.matchToToken, "function")
+ })
+
+})
+
+
+suite("tokens", function() {
+
+ function token(name, fn) {
+ suite(name, fn.bind(null, matchHelper.bind(null, name)))
+ }
+
+ function matchHelper(type, string, expected, extra) {
+ extra = extra || {}
+ if (typeof expected === "object") {
+ extra = expected
+ expected = undefined
+ }
+ jsTokens.lastIndex = 0
+ var token = jsTokens.matchToToken(jsTokens.exec(string))
+
+ test(string, function() {
+ if (expected === false) {
+ assert.notEqual(token.type, type)
+ } else {
+ assert.equal(token.type, type)
+ assert.equal(
+ token.value,
+ (typeof expected === "string" ? expected : string)
+ )
+ if ("closed" in extra) {
+ assert.equal(token.closed, extra.closed)
+ } else if (type === "string") {
+ assert.equal(token.closed, true)
+ }
+ }
+ })
+ }
+
+
+ token("whitespace", function(match) {
+
+ match(" ")
+ match(" ")
+ match(" a", " ")
+ match("\t")
+ match("\t\t\t")
+ match("\ta", "\t")
+ match("\n")
+ match("\n\n\n")
+ match("\na", "\n")
+ match("\r")
+ match("\r\r\r")
+ match("\ra", "\r")
+ match(" \t\n\r \r\n")
+ match(" \t\n\r \r\n-1", " \t\n\r \r\n")
+ match("\f")
+ match("\v")
+
+ match("\u00a0")
+ match("\u1680")
+ match("\u180e")
+ match("\u2000")
+ match("\u2001")
+ match("\u2002")
+ match("\u2003")
+ match("\u2004")
+ match("\u2005")
+ match("\u2006")
+ match("\u2007")
+ match("\u2008")
+ match("\u2009")
+ match("\u200a")
+ match("\u2028")
+ match("\u2029")
+ match("\u202f")
+ match("\u205f")
+ match("\u3000")
+
+ })
+
+
+ token("comment", function(match) {
+
+ match("//")
+ match("//comment")
+ match("// comment")
+ match("//comment ")
+ match("///")
+ match("//**//")
+ match("//comment\n", "//comment")
+ match("//comment\r", "//comment")
+ match("//comment\u2028", "//comment")
+ match("//comment\u2029", "//comment")
+ match("//comment\r\n", "//comment")
+ match("//comment \n", "//comment ")
+ match("//comment\t\n", "//comment\t")
+
+ match("/**/", {closed: true})
+ match("/*comment*/", {closed: true})
+ match("/* comment */", {closed: true})
+ match("/***/", {closed: true})
+ match("/*/*/", {closed: true})
+ match("/*\n\r\u2028\u2029 \r\n*/", {closed: true})
+
+ match("/*", {closed: false})
+ match("/*/", {closed: false})
+ match("/*unclosed", {closed: false})
+ match("/*unclosed\nnew Line(this == code ? true : false)", {closed: false})
+
+ })
+
+
+ token("string", function(match) {
+
+ match("''")
+ match('""')
+ match("``")
+ match("'string'")
+ match('"string"')
+ match("`string`")
+ match("'\\''")
+ match('"\\""')
+ match("`\\``")
+ match("'\\\\''", "'\\\\'")
+ match('"\\\\""', '"\\\\"')
+ match("`\\\\``", "`\\\\`")
+ match("'\\\\\\''")
+ match('"\\\\\\""')
+ match("`\\\\\\``")
+ match("'\\u05aF'")
+ match('"\\u05aF"')
+ match("`\\u05aF`")
+ match("'invalid escape sequence is OK: \\u'")
+ match('"invalid escape sequence is OK: \\u"')
+ match("`invalid escape sequence is OK: \\u`")
+ match("'\\\n'")
+ match('"\\\n"')
+ match("`\\\n`")
+ match("'\\\r'")
+ match('"\\\r"')
+ match("`\\\r`")
+ match("'\\\u2028'")
+ match('"\\\u2028"')
+ match("`\\\u2028`")
+ match("'\\\u2029'")
+ match('"\\\u2029"')
+ match("`\\\u2029`")
+ match("'\\\r\n'")
+ match('"\\\r\n"')
+ match("`\\\r\n`")
+ match("'string'code'another string'", "'string'")
+ match('"string"code"another string"', '"string"')
+ match("`string`code`another string`", "`string`")
+ match("'\"'")
+ match("'`'")
+ match('"\'"')
+ match('"`"')
+ match("`'`")
+ match('`"`')
+
+ match("'", {closed: false})
+ match('"', {closed: false})
+ match("`", {closed: false})
+ match("'unclosed", {closed: false})
+ match('"unclosed', {closed: false})
+ match("`unclosed", {closed: false})
+ match("'\n", "'", {closed: false})
+ match('"\n', '"', {closed: false})
+ match("`\n", {closed: false})
+ match("'\r", "'", {closed: false})
+ match('"\r', '"', {closed: false})
+ match("`\r", {closed: false})
+ match("'\u2028", "'", {closed: false})
+ match('"\u2028', '"', {closed: false})
+ match("`\u2028", {closed: false})
+ match("'\u2029", "'", {closed: false})
+ match('"\u2029', '"', {closed: false})
+ match("`\u2029", {closed: false})
+ match("'\r\n", "'", {closed: false})
+ match('"\r\n', '"', {closed: false})
+ match("`\r\n", {closed: false})
+ match("'\\\n", {closed: false})
+ match('"\\\n', {closed: false})
+ match("`\\\n", {closed: false})
+ match("'\\\r", {closed: false})
+ match('"\\\r', {closed: false})
+ match("`\\\r", {closed: false})
+ match("'\\\u2028", {closed: false})
+ match('"\\\u2028', {closed: false})
+ match("`\\\u2028", {closed: false})
+ match("'\\\u2029", {closed: false})
+ match('"\\\u2029', {closed: false})
+ match("`\\\u2029", {closed: false})
+ match("'\\\r\n", {closed: false})
+ match('"\\\r\n', {closed: false})
+ match("`\\\r\n", {closed: false})
+
+ match("'${}'")
+ match('"${}"')
+ match("`${}`")
+ match("'${a}'")
+ match('"${a}"')
+ match("`${a}`")
+ match("'a${b}c'")
+ match('"a${b}c"')
+ match("`a${b}c`")
+ match("'${'a'}'", "'${'")
+ match('"${"a"}"', '"${"')
+ match("`${`a`}`")
+ match("`${`${a}`}`")
+ match("`${fn({a: b})}`")
+ match("`${fn({a: '{'})}`")
+ match("`a${}${a}${ `${b\r}` + `${`c`}` } d $${\n(x=>{return x*2})(4)}$`")
+ match("`\\${{{}}}a`")
+
+ match("`a ${b c`.length", {closed: false})
+ match("`a ${`b${c`} d`.length", {closed: false})
+ match("`a ${ {c:d } e`.length", {closed: false})
+
+ })
+
+
+ token("regex", function(match) {
+
+ match("//", false)
+ match("/a/")
+ match("/\\//")
+ match("/\\\\//", "/\\\\/")
+ match("/\\\\\\//")
+ match("/[/]/")
+ match("/[\\]]/")
+ match("/[\\]/]/")
+ match("/[\\\\]/]/", "/[\\\\]/")
+ match("/[\\\\\\]/]/")
+ match(/\\u05aF/)
+ match("/invalid escape sequence is OK: \\u/")
+ match("/?foo/")
+ match("/*foo/", false)
+
+ match("/a/g")
+ match("/a/m")
+ match("/a/i")
+ match("/a/y")
+ match("/a/u")
+ match("/a/gmiyu")
+ match("/a/myg")
+ match("/a/e", false)
+ match("/a/invalidFlags", false)
+ match("/a/f00", false)
+
+ match("/\n/", false)
+ match("/\r/", false)
+ match("/\u2028/", false)
+ match("/\u2029/", false)
+ match("/\r\n/", false)
+ match("/\\\n/", false)
+ match("/\\\r/", false)
+ match("/\\\u2028/", false)
+ match("/\\\u2029/", false)
+ match("/\\\r\n/", false)
+ match("/[\n]/", false)
+ match("/[\r]/", false)
+ match("/[\u2028]/", false)
+ match("/[\u2029]/", false)
+ match("/[\r\n]/", false)
+ match("/[\\\n]/", false)
+ match("/[\\\r]/", false)
+ match("/[\\\u2028]/", false)
+ match("/[\\\u2029]/", false)
+ match("/[\\\r\n]/", false)
+
+ match("/a/", "/a/")
+ match("/a/g", "/a/g")
+ match("/a/;", "/a/")
+ match("/a/g;", "/a/g")
+ match("/a/ ;", "/a/")
+ match("/a/g ;", "/a/g")
+ match("/a/, b", "/a/")
+ match("/a/g, b", "/a/g")
+ match("/a/ , b", "/a/")
+ match("/a/g , b", "/a/g")
+ match("/a/.exec(b)", "/a/")
+ match("/a/g.exec(b)", "/a/g")
+ match("/a/ .exec(b)", "/a/")
+ match("/a/g .exec(b)", "/a/g")
+ match("/a/['exec'](b)", "/a/")
+ match("/a/g['exec'](b)", "/a/g")
+ match("/a/ ['exec'](b)", "/a/")
+ match("/a/g ['exec'](b)", "/a/g")
+ match("/a/]", "/a/")
+ match("/a/g]", "/a/g")
+ match("/a/ ]", "/a/")
+ match("/a/g ]", "/a/g")
+ match("/a/)", "/a/")
+ match("/a/g)", "/a/g")
+ match("/a/ )", "/a/")
+ match("/a/g )", "/a/g")
+ match("/a/}", "/a/")
+ match("/a/g}", "/a/g")
+ match("/a/ }", "/a/")
+ match("/a/g }", "/a/g")
+
+ match("/a/+=b", "/a/")
+ match("/a/ +=b", "/a/")
+ match("/a/-=b", "/a/")
+ match("/a/ -=b", "/a/")
+ match("/a/*b", "/a/")
+ match("/a/ *b", "/a/")
+ match("/a/ *=b", "/a/")
+ match("/a//b", "/a/")
+ match("/a/ /b", "/a/")
+ match("/a/ /=b", "/a/")
+ match("/a/%b", "/a/")
+ match("/a/ %b", "/a/")
+ match("/a/%=b", "/a/")
+ match("/a/ %=b", "/a/")
+ match("/a/&b", "/a/")
+ match("/a/ &b", "/a/")
+ match("/a/&=b", "/a/")
+ match("/a/ &=b", "/a/")
+ match("/a/&&b", "/a/")
+ match("/a/ &&b", "/a/")
+ match("/a/|b", "/a/")
+ match("/a/ |b", "/a/")
+ match("/a/|=b", "/a/")
+ match("/a/ |=b", "/a/")
+ match("/a/||b", "/a/")
+ match("/a/ ||b", "/a/")
+ match("/a/^b", "/a/")
+ match("/a/ ^b", "/a/")
+ match("/a/^=b", "/a/")
+ match("/a/ ^=b", "/a/")
+ match("/a/<b", "/a/")
+ match("/a/ <b", "/a/")
+ match("/a/<=b", "/a/")
+ match("/a/ <=b", "/a/")
+ match("/a/<<b", "/a/")
+ match("/a/ <<b", "/a/")
+ match("/a/<<=b", "/a/")
+ match("/a/ <<=b", "/a/")
+ match("/a/>b", "/a/")
+ match("/a/ >b", "/a/")
+ match("/a/>=b", "/a/")
+ match("/a/ >=b", "/a/")
+ match("/a/>>b", "/a/")
+ match("/a/ >>b", "/a/")
+ match("/a/>>=b", "/a/")
+ match("/a/ >>=b", "/a/")
+ match("/a/>>>b", "/a/")
+ match("/a/ >>>=b", "/a/")
+ match("/a/>>>=b", "/a/")
+ match("/a/ >>>b", "/a/")
+ match("/a/!=b", "/a/")
+ match("/a/ !=b", "/a/")
+ match("/a/!==b", "/a/")
+ match("/a/ !==b", "/a/")
+ match("/a/=b", "/a/")
+ match("/a/ =b", "/a/")
+ match("/a/==b", "/a/")
+ match("/a/ ==b", "/a/")
+ match("/a/===b", "/a/")
+ match("/a/ ===b", "/a/")
+
+ match("/a/?b:c", "/a/")
+ match("/a/ ? b : c", "/a/")
+ match("/a/:c", "/a/")
+ match("/a/g:c", "/a/g")
+ match("/a/ : c", "/a/")
+ match("/a/g : c", "/a/g")
+
+ match("/a///", "/a/")
+ match("/a/g//", "/a/g")
+ match("/a/ //", "/a/")
+ match("/a/g //", "/a/g")
+ match("/a//**/", "/a/")
+ match("/a/g/**/", "/a/g")
+ match("/a/ /**/", "/a/")
+ match("/a/g /**/", "/a/g")
+
+ match("/a/g''", "/a/g")
+ match("/a/g ''", "/a/g")
+
+ match('/a/g""', "/a/g")
+ match('/a/g ""', "/a/g")
+
+ match("/a//b/", "/a/")
+ match("/a/ /b/", "/a/")
+
+ match("/a/g 0", "/a/g")
+ match("/a/g 0.1", "/a/g")
+ match("/a/g .1", "/a/g")
+ match("/a/g 0x1", "/a/g")
+
+ match("/a/g e", "/a/g")
+ match("/a/g _", "/a/g")
+ match("/a/g $", "/a/g")
+ match("/a/g é", "/a/g")
+ match("/a/g \\u0080", "/a/g")
+
+ })
+
+
+ token("number", function(match) {
+
+ match("1")
+ match("1.")
+ match("1..", "1.")
+ match("0.1")
+ match(".1")
+ match("0.1.", "0.1")
+
+ match("-1", false)
+ match("-1.", false)
+ match("-1..", false)
+ match("-0.1", false)
+ match("-.1", false)
+ match("-0.1.", false)
+ match("-", false)
+
+ match("1e1")
+ match("1.e1")
+ match("1.e1.", "1.e1")
+ match("0.1e1")
+ match(".1e1")
+ match("0.1e1.", "0.1e1")
+
+ match("1e+1")
+ match("1e-1")
+ match("1e0123")
+ match("1e0.123", "1e0")
+ match("1e0x123", "1e0")
+ match("1E1")
+ match("1E+1")
+ match("1E-1")
+ match("1E0123")
+ match("1E0.123", "1E0")
+ match("1E0x123", "1E0")
+ match("1E0o123", "1E0")
+ match("1E0b123", "1E0")
+
+ match("e1", false)
+ match("e+1", false)
+ match("e-1", false)
+ match("E1", false)
+ match("E+1", false)
+ match("E-1", false)
+
+ match("-e1", false)
+ match("-e+1", false)
+ match("-e-1", false)
+ match("-E1", false)
+ match("-E+1", false)
+ match("-E-1", false)
+
+ match("0x1")
+ match("0xa")
+ match("0x015cF")
+ match("0x1e1")
+ match("0x1E1")
+ match("0x1g1", "0x1")
+
+ match("0X1")
+ match("0Xa")
+ match("0X015cF")
+ match("0X1e1")
+ match("0X1E1")
+ match("0X1g1", "0X1")
+
+ match("-0x1", false)
+ match("-0xa", false)
+ match("-0x015cF", false)
+ match("-0x1e1", false)
+ match("-0x1E1", false)
+ match("-0x1g1", false)
+
+ match("0x", "0")
+ match("1x1", "1")
+ match("0x1.", "0x1")
+ match("0x1.1", "0x1")
+ match("0.0x1", "0.0")
+ match(".0x1", ".0")
+
+ match("0o1")
+ match("0oa", "0")
+ match("0o01574")
+ match("0o1e1", "0o1")
+ match("0o1E1", "0o1")
+ match("0o1g1", "0o1")
+
+ match("0O1")
+ match("0Oa", "0")
+ match("0O01574")
+ match("0O1e1", "0O1")
+ match("0O1E1", "0O1")
+ match("0O1g1", "0O1")
+
+ match("-0o1", false)
+ match("-0oa", false)
+ match("-0o01574", false)
+ match("-0o1e1", false)
+ match("-0o1E1", false)
+ match("-0o1g1", false)
+
+ match("0o", "0")
+ match("1o1", "1")
+ match("0o1.", "0o1")
+ match("0o1.1", "0o1")
+ match("0.0o1", "0.0")
+ match(".0o1", ".0")
+
+ match("0b1")
+ match("0ba", "0")
+ match("0b01011")
+ match("0b1e1", "0b1")
+ match("0b1E1", "0b1")
+ match("0b1g1", "0b1")
+
+ match("0B1")
+ match("0Ba", "0")
+ match("0B01011")
+ match("0B1e1", "0B1")
+ match("0B1E1", "0B1")
+ match("0B1g1", "0B1")
+
+ match("-0b1", false)
+ match("-0ba", false)
+ match("-0b01011", false)
+ match("-0b1e1", false)
+ match("-0b1E1", false)
+ match("-0b1g1", false)
+
+ match("0b", "0")
+ match("1b1", "1")
+ match("0b1.", "0b1")
+ match("0b1.1", "0b1")
+ match("0.0b1", "0.0")
+ match(".0b1", ".0")
+
+ })
+
+
+ token("name", function(match) {
+
+ match("$")
+ match("_")
+ match("a")
+ match("z")
+ match("A")
+ match("Z")
+ match("å")
+ match("π")
+ match("0", false)
+ match("0a", false)
+ match("$0")
+ match("_0")
+ match("a0")
+ match("z0")
+ match("A0")
+ match("Z0")
+ match("å0")
+ match("π0")
+ match("a_56åπ")
+ match("Iñtërnâtiônàlizætiøn☃💩") // The last character is Pile of poo.
+
+ match("a\u00a0", "a")
+ match("a\u1680", "a")
+ match("a\u180e", "a")
+ match("a\u2000", "a")
+ match("a\u2001", "a")
+ match("a\u2002", "a")
+ match("a\u2003", "a")
+ match("a\u2004", "a")
+ match("a\u2005", "a")
+ match("a\u2006", "a")
+ match("a\u2007", "a")
+ match("a\u2008", "a")
+ match("a\u2009", "a")
+ match("a\u200a", "a")
+ match("a\u2028", "a")
+ match("a\u2029", "a")
+ match("a\u202f", "a")
+ match("a\u205f", "a")
+ match("a\u3000", "a")
+
+ match("\\u0000")
+ match("\\u15cF")
+ match("\\u15cG", false)
+ match("\\u000", false)
+ match("\\u00000")
+ match("a\\u0000b")
+
+ match("\\u{0}")
+ match("\\u{01}")
+ match("\\u{012}")
+ match("\\u{0123}")
+ match("\\u{01234}")
+ match("\\u{012345}")
+ match("\\u{0123456}", false)
+ match("\\u{15cF}")
+ match("\\u{15cG}", false)
+ match("a\\u{0000}b")
+
+ match("\\x09", false)
+
+ })
+
+
+ token("punctuator", function(match) {
+
+ match("+")
+ match("++")
+ match("+=")
+ match("++=", "++")
+ match("-")
+ match("--")
+ match("-=")
+ match("--=", "--")
+ match("*")
+ match("**")
+ match("*=")
+ match("**=")
+ match("/")
+ match("//", false)
+ match("/=")
+ match("//=", false)
+ match("%")
+ match("%%", "%")
+ match("%=")
+ match("%%=", "%")
+ match("&")
+ match("&&")
+ match("&=")
+ match("&&=", "&&")
+ match("|")
+ match("||")
+ match("|=")
+ match("||=", "||")
+ match("^")
+ match("^^", "^")
+ match("^=")
+ match("^^=", "^")
+ match("<")
+ match("<<")
+ match("<<<", "<<")
+ match("<=")
+ match("<<=")
+ match(">")
+ match(">>")
+ match(">>>")
+ match(">=")
+ match(">>=")
+ match(">>>=")
+ match("!")
+ match("!=")
+ match("!==")
+ match("!===", "!==")
+ match("=")
+ match("==")
+ match("===")
+
+ match("=>")
+ match("==>", "==")
+ match("=>>", "=>")
+
+ match("...")
+ match("..", ".")
+ match(".")
+ match("....", "...")
+
+ match("?")
+ match("~")
+ match(".")
+ match(",")
+ match(":")
+ match(";")
+ match("[")
+ match("]")
+ match("(")
+ match(")")
+ match("{")
+ match("}")
+
+ match("/a/()", "/")
+ match("/a/g()", "/")
+ match("/a/ ()", "/")
+ match("/a/g ()", "/")
+ match("/a/{}", "/")
+ match("/a/g{}", "/")
+ match("/a/ {}", "/")
+ match("/a/g {}", "/")
+
+ match("/a/+b", "/")
+ match("/a/ +b", "/")
+ match("/a/++b", "/")
+ match("/a/ ++b", "/")
+ match("/a/-b", "/")
+ match("/a/ -b", "/")
+ match("/a/--b", "/")
+ match("/a/ --b", "/")
+ match("/a/!b", "/")
+ match("/a/ !b", "/")
+ match("/a/~b", "/")
+ match("/a/ ~b", "/")
+
+ match("/a/g+b", "/")
+ match("/a/g +b", "/")
+ match("/a/g+=b", "/")
+ match("/a/g +=b", "/")
+ match("/a/g++", "/")
+ match("/a/g ++", "/")
+ match("/a/g-b", "/")
+ match("/a/g -b", "/")
+ match("/a/g-=b", "/")
+ match("/a/g -=b", "/")
+ match("/a/g--", "/")
+ match("/a/g --", "/")
+ match("/a/g*b", "/")
+ match("/a/g *b", "/")
+ match("/a/g *=b", "/")
+ match("/a/g/b", "/")
+ match("/a/g /b", "/")
+ match("/a/g /=b", "/")
+ match("/a/g%b", "/")
+ match("/a/g %b", "/")
+ match("/a/g%=b", "/")
+ match("/a/g %=b", "/")
+ match("/a/g&b", "/")
+ match("/a/g &b", "/")
+ match("/a/g&=b", "/")
+ match("/a/g &=b", "/")
+ match("/a/g&&b", "/")
+ match("/a/g &&b", "/")
+ match("/a/g|b", "/")
+ match("/a/g |b", "/")
+ match("/a/g|=b", "/")
+ match("/a/g |=b", "/")
+ match("/a/g||b", "/")
+ match("/a/g ||b", "/")
+ match("/a/g^b", "/")
+ match("/a/g ^b", "/")
+ match("/a/g^=b", "/")
+ match("/a/g ^=b", "/")
+ match("/a/g<b", "/")
+ match("/a/g <b", "/")
+ match("/a/g<=b", "/")
+ match("/a/g <=b", "/")
+ match("/a/g<<b", "/")
+ match("/a/g <<b", "/")
+ match("/a/g<<=b", "/")
+ match("/a/g <<=b", "/")
+ match("/a/g>b", "/")
+ match("/a/g >b", "/")
+ match("/a/g>=b", "/")
+ match("/a/g >=b", "/")
+ match("/a/g>>b", "/")
+ match("/a/g >>b", "/")
+ match("/a/g>>=b", "/")
+ match("/a/g >>=b", "/")
+ match("/a/g>>>b", "/")
+ match("/a/g >>>=b", "/")
+ match("/a/g>>>=b", "/")
+ match("/a/g >>>b", "/")
+ match("/a/g!=b", "/")
+ match("/a/g !=b", "/")
+ match("/a/g!==b", "/")
+ match("/a/g !==b", "/")
+ match("/a/g=b", "/")
+ match("/a/g =b", "/")
+ match("/a/g==b", "/")
+ match("/a/g ==b", "/")
+ match("/a/g===b", "/")
+ match("/a/g ===b", "/")
+
+ match("/a/g?b:c", "/")
+ match("/a/g ? b : c", "/")
+
+ match("/a/''", "/")
+ match("/a/ ''", "/")
+
+ match('/a/""', "/")
+ match('/a/ ""', "/")
+
+ match("/a/g/b/", "/")
+ match("/a/g /b/", "/")
+
+ match("/a/0", "/")
+ match("/a/ 0", "/")
+ match("/a/0.1", "/")
+ match("/a/ 0.1", "/")
+ match("/a/.1", "/")
+ match("/a/ .1", "/")
+
+ match("/a/e", "/")
+ match("/a/ e", "/")
+ match("/a/_", "/")
+ match("/a/ _", "/")
+ match("/a/$", "/")
+ match("/a/ $", "/")
+ match("/a/é", "/")
+ match("/a/ é", "/")
+ match("/a/\\u0080", "/")
+ match("/a/ \\u0080", "/")
+
+ match("/a/ge", "/")
+ match("/a/g_", "/")
+ match("/a/g$", "/")
+ match("/a/gé", "/")
+ match("/a/g0", "/")
+ match("/a/g\\u0080", "/")
+
+ })
+
+
+ token("invalid", function(match) {
+
+ match("")
+ match("@")
+ match("#")
+ match("\\")
+ match("\\xa9", "\\")
+ match("\u0000")
+ match("\u007F")
+
+ })
+
+})
+
+
+suite("tokenization", function() {
+
+ function testFile(file) {
+ var contents = fs.readFileSync("test/fixtures/" + file + ".js").toString()
+ var expected = require("./fixtures/" + file + ".json")
+ var actual = contents.match(jsTokens)
+ test(file + ".js", function() {
+ assert.deepEqual(actual, expected)
+ assert.equal(actual.join(""), contents)
+ })
+ }
+
+ testFile("base64")
+ testFile("errors")
+ testFile("regex")
+ testFile("division")
+
+})
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-javascript/node-js-tokens.git
More information about the Pkg-javascript-commits
mailing list