Thanks to visit codestin.com
Credit goes to github.com

Skip to content

refactor(exec): move child process to source file #786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 19, 2017
+81 −47
Merged

refactor(exec): move child process to source file #786

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f37b723
refactor(exec): move child process to source file
nfischer Oct 12, 2017
057f442
Disallow requiring exec-child.js
nfischer Oct 19, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions src/exec-child.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
var childProcess = require('child_process');
var fs = require('fs');

// Note: this will break if `paramFilePath` contains special characters ( '\n',
// '\t', etc.). Such characters are possible if $TMP gets modified. We already
// rely on tempdir() to work for other things, so this is an acceptable risk.
var paramFilePath = process.argv[2];

var serializedParams = fs.readFileSync(paramFilePath, 'utf8');
var params = JSON.parse(serializedParams);

var cmd = params.command;
var execOptions = params.execOptions;
var pipe = params.pipe;
var stdoutFile = params.stdoutFile;
var stderrFile = params.stderrFile;
var codeFile = params.codeFile;

var c = childProcess.exec(cmd, execOptions, function (err) {
if (!err) {
fs.writeFileSync(codeFile, '0');
} else if (err.code === undefined) {
fs.writeFileSync(codeFile, '1');
} else {
fs.writeFileSync(codeFile, err.code.toString());
}
});

var stdoutStream = fs.createWriteStream(stdoutFile);
var stderrStream = fs.createWriteStream(stderrFile);

c.stdout.pipe(stdoutStream, { end: false });
c.stderr.pipe(stderrStream, { end: false });
c.stdout.pipe(process.stdout);
c.stderr.pipe(process.stderr);

if (pipe) {
c.stdin.end(pipe);
}

c.stdout.on('end', stdoutStream.end);
c.stderr.on('end', stderrStream.end);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this file is require'd, it will actually do something. To prevent this, wrap the contents like this:

if (require.main === module) {
    // everything goes here
}

This way, the code is only executed when the file is the entrypoint, like when node exec-file.js is run.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems reasonable. Do you think it's worth adding this to similar files (e.g. bin/shjs)?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think this would be a good idea for other executable-only files.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filed #789. I think it would be better to fail loudly in this case (with an exception). Take a look at my alternate suggestion, let me know if there's some issue with this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Failing loudly sounds like a good idea.

76 changes: 29 additions & 47 deletions src/exec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ function execSync(cmd, opts, pipe) {
}

var tempDir = _tempDir();
var stdoutFile = path.resolve(tempDir + '/' + common.randomFileName());
var stderrFile = path.resolve(tempDir + '/' + common.randomFileName());
var codeFile = path.resolve(tempDir + '/' + common.randomFileName());
var scriptFile = path.resolve(tempDir + '/' + common.randomFileName());
var paramsFile = path.resolve(tempDir + '/' + common.randomFileName());
var stderrFile = path.resolve(tempDir + '/' + common.randomFileName());
var stdoutFile = path.resolve(tempDir + '/' + common.randomFileName());

opts = common.extend({
silent: common.config.silent,
Expand All @@ -37,47 +37,29 @@ function execSync(cmd, opts, pipe) {
encoding: 'utf8',
}, opts);

if (fs.existsSync(scriptFile)) common.unlinkSync(scriptFile);
if (fs.existsSync(stdoutFile)) common.unlinkSync(stdoutFile);
if (fs.existsSync(stderrFile)) common.unlinkSync(stderrFile);
if (fs.existsSync(codeFile)) common.unlinkSync(codeFile);

var execCommand = JSON.stringify(common.config.execPath) + ' ' + JSON.stringify(scriptFile);
var script;
if (fs.existsSync(paramsFile)) common.unlinkSync(paramsFile);
if (fs.existsSync(stderrFile)) common.unlinkSync(stderrFile);
if (fs.existsSync(stdoutFile)) common.unlinkSync(stdoutFile);

opts.cwd = path.resolve(opts.cwd);
var optString = JSON.stringify(opts);

script = [
"var child = require('child_process')",
" , fs = require('fs');",
'var childProcess = child.exec(' + JSON.stringify(cmd) + ', ' + optString + ', function(err) {',
' var fname = ' + JSON.stringify(codeFile) + ';',
' if (!err) {',
' fs.writeFileSync(fname, "0");',
' } else if (err.code === undefined) {',
' fs.writeFileSync(fname, "1");',
' } else {',
' fs.writeFileSync(fname, err.code.toString());',
' }',
'});',
'var stdoutStream = fs.createWriteStream(' + JSON.stringify(stdoutFile) + ');',
'var stderrStream = fs.createWriteStream(' + JSON.stringify(stderrFile) + ');',
'childProcess.stdout.pipe(stdoutStream, {end: false});',
'childProcess.stderr.pipe(stderrStream, {end: false});',
'childProcess.stdout.pipe(process.stdout);',
'childProcess.stderr.pipe(process.stderr);',
].join('\n') +
(pipe ? '\nchildProcess.stdin.end(' + JSON.stringify(pipe) + ');\n' : '\n') +
[
'var stdoutEnded = false, stderrEnded = false;',
'function tryClosingStdout(){ if(stdoutEnded){ stdoutStream.end(); } }',
'function tryClosingStderr(){ if(stderrEnded){ stderrStream.end(); } }',
"childProcess.stdout.on('end', function(){ stdoutEnded = true; tryClosingStdout(); });",
"childProcess.stderr.on('end', function(){ stderrEnded = true; tryClosingStderr(); });",
].join('\n');

fs.writeFileSync(scriptFile, script);

var paramsToSerialize = {
command: cmd,
execOptions: opts,
pipe: pipe,
stdoutFile: stdoutFile,
stderrFile: stderrFile,
codeFile: codeFile,
};

fs.writeFileSync(paramsFile, JSON.stringify(paramsToSerialize), 'utf8');
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the advantage of storing the params in a separate file, as opposed to simply passing them in as a argument to the command?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That was my original approach, but there's some issues with passing via CLI argument (at least in shells, via execSync). As an example, we could look at the multi-line string:

has"chars
.txt

JSON.stringify gives us "has\"chars\n.txt". So far, so good.

When this gets passed to the shell, it looks something like:

node exec-child.js "has\"chars\n.txt"

The shell interprets the string and supplies the interpreted value to the node process, so process.argv[2] looks like has"chars\n.txt (that's a one-line string with a literal \ followed by a literal n).

This is where we hit issues. Our string is partially deserialized, so we can't use JSON.parse, but it's still not equivalent to our original string. This is an unfortunate consequence of using both shell and JS--they have different opinions on how to escape strings.

We can consider passing these by CLI argument if we replace execSync() with execFileSync() within exec.js. This is one of my intended future optimizations, but I left it out to limit complexity. Such a change is backwards compatible (we just can't change exec() within exec-child.js).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, that makes sense. We eliminate the need to write the JS file every call, but in return we need to write the arguments to a file. Not the best, but it's an improvement overall.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is still strictly better. The JS file used to contain each argument written inside it (as literals), and sizeof(code) + sizeof(arguments) > sizeof(arguments).

I'll also send some PRs to trim down the number of arguments, which will be a bigger win.


var execCommand = [
JSON.stringify(common.config.execPath),
JSON.stringify(path.join(__dirname, 'exec-child.js')),
JSON.stringify(paramsFile),
].join(' ');

/* istanbul ignore else */
if (opts.silent) {
Expand All @@ -91,10 +73,10 @@ function execSync(cmd, opts, pipe) {
child.execSync(execCommand, opts);
} catch (e) {
// Clean up immediately if we have an exception
try { common.unlinkSync(scriptFile); } catch (e2) {}
try { common.unlinkSync(stdoutFile); } catch (e2) {}
try { common.unlinkSync(stderrFile); } catch (e2) {}
try { common.unlinkSync(codeFile); } catch (e2) {}
try { common.unlinkSync(paramsFile); } catch (e2) {}
try { common.unlinkSync(stderrFile); } catch (e2) {}
try { common.unlinkSync(stdoutFile); } catch (e2) {}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're not using e2 anyway, might as well call it e (it won't overwrite the e from above).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there's an eslint warning against that...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, right.

throw e;
}

Expand All @@ -118,10 +100,10 @@ function execSync(cmd, opts, pipe) {
}

// No biggie if we can't erase the files now -- they're in a temp dir anyway
try { common.unlinkSync(scriptFile); } catch (e) {}
try { common.unlinkSync(stdoutFile); } catch (e) {}
try { common.unlinkSync(stderrFile); } catch (e) {}
try { common.unlinkSync(codeFile); } catch (e) {}
try { common.unlinkSync(paramsFile); } catch (e) {}
try { common.unlinkSync(stderrFile); } catch (e) {}
try { common.unlinkSync(stdoutFile); } catch (e) {}

if (code !== 0) {
common.error('', code, { continue: true });
Expand Down