From 4c8e7c1fbbaeda31701bd2513c14d82823d0a2f4 Mon Sep 17 00:00:00 2001 From: Philippe Elsass Date: Sun, 10 Jun 2018 22:05:50 +0100 Subject: [PATCH] Use Cherow for parsing --- build.hxml | 1 + package-lock.json | 7 ++++++- package.json | 1 + tool/bin/split.js | 27 ++++++++++++++++----------- tool/src/HxSplit.hx | 2 +- tool/src/Parser.hx | 23 ++++++++++++++++++----- tool/src/acorn/Cherow.hx | 26 ++++++++++++++++++++++++++ tool/test/test-suite.js | 7 +++---- 8 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 tool/src/acorn/Cherow.hx diff --git a/build.hxml b/build.hxml index 489a712..2c517ec 100644 --- a/build.hxml +++ b/build.hxml @@ -5,6 +5,7 @@ --next +-D cherow_parser -lib hxnodejs -cp tool/src HxSplit diff --git a/package-lock.json b/package-lock.json index 206ce81..b066769 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "haxe-modular", - "version": "0.10.12", + "version": "0.10.13", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -17,6 +17,11 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz", "integrity": "sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c=" }, + "cherow": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/cherow/-/cherow-1.6.6.tgz", + "integrity": "sha1-3R4M9DeC8CKbPgCrIBvpZxatgcE=" + }, "cli-color": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/cli-color/-/cli-color-0.1.7.tgz", diff --git a/package.json b/package.json index b3e7a02..9ac0f7e 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "dependencies": { "@elsassph/fast-source-map": "^0.3.0", "acorn": "^4.0.3", + "cherow": "^1.6.6", "graphlib": "^2.1.1", "react-deep-force-update": "^2.0.1", "react-proxy": "^2.0.8", diff --git a/tool/bin/split.js b/tool/bin/split.js index 5a7750b..3fe1480 100644 --- a/tool/bin/split.js +++ b/tool/bin/split.js @@ -1029,7 +1029,7 @@ HxOverrides.substr = function(s,pos,len) { var HxSplit = function() { }; HxSplit.__name__ = true; HxSplit.run = $hx_exports["run"] = function(input,output,modules,debugMode,commonjs,debugSourceMap,dump,astHooks) { - var src = js_node_Fs.readFileSync(input).toString(); + var src = js_node_Fs.readFileSync(input,"utf8"); var parser = new Parser(src,debugMode,commonjs); var sourceMap = debugMode ? new SourceMap(input,src) : null; modules = HxSplit.applyAstHooks(parser.mainModule,modules,astHooks,parser.graph); @@ -1148,7 +1148,7 @@ var Parser = function(src,withLocation,commonjs) { this.reservedTypes = { "String" : true, "Math" : true, "Array" : true, "Date" : true, "Number" : true, "Boolean" : true, __map_reserved : true}; this.mainModule = "Main"; var t0 = new Date().getTime(); - this.processInput(src,withLocation); + this.processInputAcorn(src,withLocation); var t1 = new Date().getTime(); console.log("Parsed in: " + (t1 - t0) + "ms"); this.buildGraph(commonjs); @@ -1157,7 +1157,7 @@ var Parser = function(src,withLocation,commonjs) { }; Parser.__name__ = true; Parser.prototype = { - processInput: function(src,withLocation) { + processInputAcorn: function(src,withLocation) { var options = { ecmaVersion : 5, allowReserved : true}; if(withLocation) { options.locations = true; @@ -1165,28 +1165,32 @@ Parser.prototype = { var program = acorn_Acorn.parse(src,options); this.walkProgram(program); } + ,processInputCherow: function(src,withLocation) { + var program = acorn_Cherow.parse(src,{ ranges : true, loc : withLocation}); + this.walkProgram(program); + } ,buildGraph: function(commonjs) { var g = new graphlib_Graph({ directed : true, compound : true}); var cpt = 0; var refs = 0; + var keys = Reflect.fields(this.types); var _g = 0; - var _g1 = Reflect.fields(this.types); - while(_g < _g1.length) { - var t = _g1[_g]; + while(_g < keys.length) { + var t = keys[_g]; ++_g; ++cpt; g.setNode(t,t); } if(!commonjs) { this.types["require"] = []; + keys.push("require"); g.setNode("require","require"); g.setEdge(this.mainModule,"require"); } - var _g2 = 0; - var _g11 = Reflect.fields(this.types); - while(_g2 < _g11.length) { - var t1 = _g11[_g2]; - ++_g2; + var _g1 = 0; + while(_g1 < keys.length) { + var t1 = keys[_g1]; + ++_g1; refs += this.walk(g,t1,this.types[t1]); } console.log("Stats: " + cpt + " types, " + refs + " references"); @@ -1731,6 +1735,7 @@ StringTools.trim = function(s) { }; var acorn_Acorn = require("acorn"); var acorn_Walk = require("acorn/dist/walk"); +var acorn_Cherow = require("cherow"); var graphlib_Graph = require("graphlib").Graph; var haxe_IMap = function() { }; haxe_IMap.__name__ = true; diff --git a/tool/src/HxSplit.hx b/tool/src/HxSplit.hx index a766bda..6258c0d 100644 --- a/tool/src/HxSplit.hx +++ b/tool/src/HxSplit.hx @@ -12,7 +12,7 @@ class HxSplit astHooks:ArrayString->Array>) { // parse input - var src = Fs.readFileSync(input).toString(); + var src = Fs.readFileSync(input, "utf8"); var parser = new Parser(src, debugMode, commonjs); var sourceMap = debugMode ? new SourceMap(input, src) : null; diff --git a/tool/src/Parser.hx b/tool/src/Parser.hx index d64dfba..e9a7d8b 100644 --- a/tool/src/Parser.hx +++ b/tool/src/Parser.hx @@ -1,6 +1,7 @@ +import acorn.Acorn; +import acorn.Cherow; import graphlib.Graph; import haxe.DynamicAccess; -import acorn.Acorn; class Parser { @@ -26,7 +27,11 @@ class Parser public function new(src:String, withLocation:Bool, commonjs:Bool) { var t0 = Date.now().getTime(); - processInput(src, withLocation); + #if cherow_parser + processInputAcorn(src, withLocation); + #else + processInputCherow(src, withLocation); + #end var t1 = Date.now().getTime(); trace('Parsed in: ${t1 - t0}ms'); @@ -35,7 +40,7 @@ class Parser trace('AST processed in: ${t2 - t1}ms'); } - function processInput(src:String, withLocation:Bool) + function processInputAcorn(src:String, withLocation:Bool) { var options:AcornOptions = { ecmaVersion: 5, allowReserved: true }; if (withLocation) options.locations = true; @@ -43,12 +48,19 @@ class Parser walkProgram(program); } + function processInputCherow(src:String, withLocation:Bool) + { + var program = Cherow.parse(src, { ranges: true, loc: withLocation }); + walkProgram(program); + } + function buildGraph(commonjs:Bool) { var g = new Graph({ directed: true, compound:true }); var cpt = 0; var refs = 0; - for (t in types.keys()) { + var keys = types.keys(); + for (t in keys) { cpt++; g.setNode(t, t); } @@ -56,11 +68,12 @@ class Parser if (!commonjs) { // require stub is generated in web entry point types.set('require', []); + keys.push('require'); g.setNode('require', 'require'); g.setEdge(mainModule, 'require'); } - for (t in types.keys()) + for (t in keys) refs += walk(g, t, types.get(t)); trace('Stats: $cpt types, $refs references'); diff --git a/tool/src/acorn/Cherow.hx b/tool/src/acorn/Cherow.hx new file mode 100644 index 0000000..bb9a73b --- /dev/null +++ b/tool/src/acorn/Cherow.hx @@ -0,0 +1,26 @@ +package acorn; + +import acorn.Acorn; + +typedef CherowOptions = { + ?module: Bool, // Enable module syntax + ?loc: Bool, // Attach line/column location information to each node + ?ranges: Bool, // Append start and end offsets to each node + ?impliedStrict: Bool, // Enable strict mode initial enforcement + ?next: Bool, // Enable stage 3 support (ESNext) + ?tolerant: Bool, // Create a top-level error array containing all "skipped" errors + ?source: Bool, // Set to true to record the source file in every node's loc object when the loc option is set. + ?raw: Bool, // Attach raw property to each literal node + ?rawIdentifier: Bool, // Attach raw property to each identifier node +} + +@:jsRequire('cherow') +extern class Cherow { + + /** + * is used to parse a JavaScript program. The input parameter is a string, options can + * be undefined or an object setting some of the options listed below. The return value + * will be an abstract syntax tree object as specified by the ESTree spec. + */ + static public function parse(input:String, options:CherowOptions):AstNode; +} diff --git a/tool/test/test-suite.js b/tool/test/test-suite.js index 9d93de0..5cc74c3 100644 --- a/tool/test/test-suite.js +++ b/tool/test/test-suite.js @@ -115,14 +115,14 @@ function execTest(className, name, params, isNode, callback) { const folder = `tool/test/bin/${name}`; if (useLib[className]) params += ' -D uselib'; var cmd = `haxe tool/test/test-common.hxml -main ${className} -js ${folder}/index.js ${params}`; - //console.log(cmd); + console.log(cmd); exec(cmd, (err, stdout, stderr) => { + console.log(stdout); if (err) { hasFailedCase = 1; console.log(stderr); callback(err); } else { - console.log(stdout); runValidation(name, isNode, callback); } }); @@ -132,12 +132,11 @@ function runValidation(name, isNode, callback) { const result = `tool/test/bin/${name}/index.js.json`; const valid = `tool/test/expect/${name}.json`; exec(`node tool/test/validate.js ${result} ${valid}`, (err, stdout, stderr) => { + console.log(stdout); if (err) { hasFailedCase = 2; - console.log(stdout); callback(err); } else { - console.log(stdout); if (isNode) runOutput(name, callback); else callback(); }