Skip to content
This repository has been archived by the owner on Sep 25, 2018. It is now read-only.

Commit

Permalink
Carve out a code path to defer compilation of generated code...
Browse files Browse the repository at this point in the history 
So that we can generate a big module string which can be generated at
build-time and saved to disk. This is important for two reasons:

1. This allows the browser to avoid using scion compiler at runtime.
Compiled SCXML-to-js modules are much smaller runtime dependency over
the wire than the SCION compiler.

2. Node.js --inspect does not work for code evaluated at runtime using
eval or node vm module. See issue nodejs/node#8369.
So to use chrome dev tools and source maps to debug SCXML, it is
currently necessary to use ahead-of-time compilation to generate a
module file as a workaround.
  • Loading branch information
@jbeard4
jbeard4 committed Apr 22, 2017
1 parent 6186305 commit 3601b12
Show file tree
Hide file tree
Showing 7 changed files with 297 additions and 902 deletions.
951 changes: 171 additions & 780 deletions dist/scxml.js
View file

Large diffs are not rendered by default.

139 changes: 48 additions & 91 deletions lib/compiler/scjson-to-module.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,47 +29,52 @@ function stripAttrNsPrefix(attrName){
return m[1];
}

function parseJsCode(fnBody, action, docUrl, node, isExpression){
//console.log(fnBody, docUrl);
var tokens = esprima.tokenize(fnBody, { loc: true });
var lastTokenEndLine = 1,
lastTokenEndCol = 0;
tokens.forEach(function(token, i){
var numLinesPadding = token.loc.start.line - lastTokenEndLine,
numColsPadding = token.loc.start.column - lastTokenEndCol;
var whitespace = (function(){
var s = [];
for(var i = 0; i < numLinesPadding; i++){
s.push('\n');
}
for(var j = 0; j < numColsPadding ; j++){
s.push(' ');
}
return s.join('');
})();
if(!(isExpression && i==0)){
//skip whitespace
node.add(whitespace);
}
var tokenNode = new SourceNode(
action.$line + token.loc.start.line,
token.loc.start.line === 1 ? action.$column + token.loc.start.column : token.loc.start.column,
docUrl,
token.value
);
//console.log('tokenNode',tokenNode);
node.add(tokenNode);
lastTokenEndLine = token.loc.end.line;
lastTokenEndCol = token.loc.end.column;
});
}

function generateFnDeclaration(fnName,fnBody,action,docUrl,isExpression,parseGeneratedCodeForSourceMap){
if(printTrace) console.log('generateFnDeclaration',fnName,fnBody);

var tokens = esprima.tokenize(fnBody, { loc: true });
var lastTokenEndLine = 1,
lastTokenEndCol = 0;
var node = new SourceNode();
node.add('function ' + fnName + FN_ARGS + '{\n');
if(isExpression){
var returnNode = new SourceNode(action.$line + 1, action.$column, docUrl, 'return ');
node.add(returnNode);
}
if(parseGeneratedCodeForSourceMap){
tokens.forEach(function(token, i){
var numLinesPadding = token.loc.start.line - lastTokenEndLine,
numColsPadding = token.loc.start.column - lastTokenEndCol;
var whitespace = (function(){
var s = [];
for(var i = 0; i < numLinesPadding; i++){
s.push('\n');
}
for(var j = 0; j < numColsPadding ; j++){
s.push(' ');
}
return s.join('');
})();
if(!(isExpression && i==0)){
//skip whitespace
node.add(whitespace);
}
var tokenNode = new SourceNode(
action.$line + token.loc.start.line,
token.loc.start.line === 1 ? action.$column + token.loc.start.column : token.loc.start.column,
docUrl,
token.value
);
//console.log('tokenNode',tokenNode);
node.add(tokenNode);
lastTokenEndLine = token.loc.end.line;
lastTokenEndCol = token.loc.end.column;
});
parseJsCode(fnBody, action, docUrl, node, isExpression);
}else{
node.add(
new SourceNode(
Expand Down Expand Up @@ -194,16 +199,17 @@ function dumpFunctionDeclarations(fnDecAccumulator){

function dumpHeader(strict){
var d = new Date();
var strictStr = (strict ? "'use strict';\n" : "");
return strictStr + '//Generated on ' + d.toLocaleDateString() + ' ' + d.toLocaleTimeString() + ' by the SCION SCXML compiler';
return '//Generated on ' + d.toLocaleDateString() + ' ' + d.toLocaleTimeString() + ' by the SCION SCXML compiler\n' +
(strict ? "'use strict';\n" : "");
}

function generateFactoryFunctionWrapper(o, name, options){

var root = new SourceNode(null,null,null,[
o.headerString + '\n' ,
//o.headerString + '\n' ,
'(function (_x,_sessionid,_ioprocessors,In){\n' ,
' var _name = \'' + name + '\';\n',
new SourceNode(null,null,null,""), //create a dummy node at index 2, which we might use to inject datamodel and scripts
o.sendString,
o.sendIdLocationString,
o.earlyBindingFnDeclaration,
Expand All @@ -218,11 +224,7 @@ function generateFactoryFunctionWrapper(o, name, options){
])
);

var s = root.toStringWithSourceMap();

return s.code + '\n' +
'//# sourceMappingURL=data:application/json;charset=utf-8;base64,' +
new Buffer(s.map.toString()).toString('base64');
return root;
}

ModuleBuilder.prototype.generateModule = function(){
Expand Down Expand Up @@ -420,7 +422,7 @@ ModuleBuilder.prototype.handleExternalActionScript = function(action) {
if (!this.externalActionScripts.has(action.src)) {
this.externalActionScripts.add(action.src);
action.$wrap = function(body) {
return generateFnDeclaration(fnName, body, action);
return generateFnDeclaration(fnName,body,{$line : 0, $column:0},action.src,false,true);
}

this.rootState.rootScripts.push(action);
Expand Down Expand Up @@ -652,6 +654,7 @@ var actionTags = {
var shallowArrayName = getVariableNameForShallowCopy(builder);

var forEachContents =
(!action.index ? 'var ' + index + ';\n' : '') +
'var ' + shallowArrayName + ' = ' + arr + ';\n'+
'if(Array.isArray(' + shallowArrayName + ')){\n' +
' for(' + index + ' = 0; ' + index + ' < ' + shallowArrayName + '.length;' + index + '++){\n' +
Expand Down Expand Up @@ -712,54 +715,8 @@ function generateIdlocationGenerator(sendIdAccumulator){
'}';
}


module.exports = startTraversal;

//for executing directly under node.js
if(require.main === module){
//TODO: clean up command-line interface so that we do not expose unnecessary cruft
var usage = 'Usage: $0 [ FILE | - ]';
var argv = require('optimist').
demand('d').
alias('d', 'doc-url').
usage(usage).
argv;
var input = argv._[0];
var docUrl = argv.d;

if(!input){
console.error(usage);
process.exit(1);
} else if(input === '-'){

//read from stdin or file
process.stdin.setEncoding('utf8');
process.stdin.resume();

var jsonString = '';
process.stdin.on('data',function(s){
jsonString += s;
});

process.stdin.on('end',go.bind(this,docUrl));
} else{
var fs = require('fs');
jsonString = fs.readFileSync(input,'utf8');
go(docUrl);
}
}

function go(docUrl){
if (!docUrl) {
docUrl = '';
}

startTraversal(docUrl, JSON.parse(jsonString), {debug: true}).then(
function resolved(jsModule) {
console.log(jsModule.module);
},
function rejected(err) {
console.error(err);
}
);
}
module.exports = {
startTraversal : startTraversal,
parseJsCode : parseJsCode,
dumpHeader : dumpHeader
};
7 changes: 0 additions & 7 deletions lib/compiler/scxml-to-scjson.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,10 +332,3 @@ function transform(xmlString){
}

module.exports = transform;

//for executing diretly under node.js
if(require.main === module){
//TODO: allow reading from stdin directly
//TODO: use saxjs's support for streaming API.
console.log(JSON.stringify(transform(require('fs').readFileSync(process.argv[2],'utf8')),4,4));
}
70 changes: 58 additions & 12 deletions lib/runtime/document-string-to-model.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ var compilerInternals = require('./compiler-internals'),
platform = require('./platform-bootstrap/node/platform'),
fs = require('fs'),
vm = require('vm'),
SourceNode = require('source-map').SourceNode,
assert = require('assert');

var scxmlToScjson = compilerInternals.scxmlToScjson,
Expand Down Expand Up @@ -55,12 +56,15 @@ function documentStringToModel(url,docString,cb,hostContext){
function fetchScript(scriptInfo, hostContext) {
return new Promise(function(resolve, reject) {

platform.getScriptFromUrl(scriptInfo.src,
platform.getScriptFromUrl(scriptInfo,
function(err, compiledScript, mimeType) {
if (err) {
reject(err);
} else {
scriptInfo.compiled = compiledScript;
if(!hostContext.deferCompilation){
var content = scriptInfo.$wrap ? scriptInfo.$wrap(scriptInfo.content) : scriptInfo.content;
scriptInfo.compiled = new vm.Script(content, {filename: scriptInfo.src});
}
resolve(scriptInfo);
}
},
Expand Down Expand Up @@ -199,16 +203,49 @@ SCModel.prototype.prepare = function(cb, executionContext, hostContext) {
const self = this;
Promise.all(scriptPromises).then(function resolved(scripts) {
try {
if (self.datamodel) {
self.datamodel.runInContext(executionContext);
}
if(hostContext.deferCompilation){
var rootNode = new SourceNode(null, null, null);

for (let i = 0; i < scriptCount; i++) {
self.rootScripts[i].compiled.runInContext(executionContext);
}
rootNode.add(scjsonToModule.dumpHeader(true));

//rootNode.add('module.exports = ');
var injectionNode = self.module.children[2];

if(self.datamodel) {
injectionNode.add(self.datamodel);
injectionNode.add('\n');
}

self.rootScripts.forEach(function(rootScript){
if(rootScript.$wrap){
injectionNode.add(rootScript.$wrap(rootScript.content));
}else{
scjsonToModule.parseJsCode(rootScript.content, {$line : 0, $column : 0}, rootScript.src, injectionNode, false)
}
injectionNode.add('\n');
})

rootNode.add(self.module);

var s = rootNode.toStringWithSourceMap();
var generatedCode = s.code + '\n' +
'//# sourceMappingURL=data:application/json;charset=utf-8;base64,' +
new Buffer(s.map.toString()).toString('base64')
//console.log(generatedCode);
var fn = vm.runInContext(generatedCode, executionContext);
cb(null,fn);
} else {
if (self.datamodel) {
self.datamodel.runInContext(executionContext);
}

for (let i = 0; i < scriptCount; i++) {
self.rootScripts[i].compiled.runInContext(executionContext);
}

let modelFn = self.module.runInContext(executionContext);
cb(undefined, modelFn);
let modelFn = self.module.runInContext(executionContext);
cb(undefined, modelFn);
}
} catch (e) {
cb(e);
}
Expand All @@ -225,12 +262,21 @@ function createModule(url,scJson,hostContext,cb){
}
}

scjsonToModule(url, scJson, hostContext).then(function resolved(rawModule) {
scjsonToModule.startTraversal(url, scJson, hostContext).then(function resolved(rawModule) {
if (platform.debug && rawModule.name) {
fs.writeFileSync('/var/tmp/' + rawModule.name + '.scion', rawModule.module);
}

compileModule(url, rawModule, hostContext, cb);
if(hostContext.deferCompilation){
cb(null, new SCModel(
rawModule.name,
rawModule.datamodel,
rawModule.rootScripts,
rawModule.module
));
} else {
compileModule(url, rawModule, hostContext, cb);
}
}, function rejected(err) {
cb(err);
});
Expand Down
13 changes: 13 additions & 0 deletions lib/runtime/facade.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,3 +84,16 @@ module.exports = {
},
scion : require('scion-core')
};


if(require.main === module){
var file = process.argv[2];
var options = {deferCompilation : true};
pathToModel(file, function(err, model){
if(err) throw err;
model.prepare(function(err, fnModel){
if(err) throw err;
console.log(fnModel.toString());
}, options);
}, options);
}
13 changes: 4 additions & 9 deletions lib/runtime/platform-bootstrap/node/platform.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,21 +40,16 @@ module.exports = {

getResourceFromUrl : get.getResource,

getScriptFromUrl: function(url, cb, context, scriptInfo) {
get.getResource(url, function(err, content) {
getScriptFromUrl: function(script, cb, context, scriptInfo) {
get.getResource(script.src, function(err, content) {
if (err) {
cb(err);
return;
}

if (typeof scriptInfo.$wrap === 'function') {
content = scriptInfo.$wrap(content);
}

var options = Object.assign({filename: url}, scriptInfo);
try {
var script = new vm.Script(content, options);
cb(undefined, script);
script.content = content;
cb(undefined, content);
} catch (e) {
cb(e);
}
Expand Down
6 changes: 3 additions & 3 deletions test/node-test-server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ http.createServer(function (req, res) {
try {
model.prepare(function(err, fnModel) {
if (err) {
console.error('model preparation error: ' + err);
console.error('model preparation error: ' + err, err.stack);
res.writeHead(500, {'Content-Type': 'text/plain'});
res.end(err.message);
return;
Expand All @@ -60,15 +60,15 @@ http.createServer(function (req, res) {

// TODO: timeout should be kicked off before fetch/compilation/preparation
timeouts[sessionToken] = setTimeout(function(){cleanUp(sessionToken);},timeoutMs);
});
}, {deferCompilation : true});
} catch(e) {
console.log(e.stack);
console.log(e);
res.writeHead(500, {'Content-Type': 'text/plain'});
res.end(e.message);
}
}
});
}, null, {deferCompilation : true});

}else if(reqJson.event && (typeof reqJson.sessionToken === "number")){
console.log("sending event to statechart",reqJson.event);
Expand Down

0 comments on commit 3601b12

Please sign in to comment.