extract: parallelize extraction

npm · zkat · Jun 29, 2017 · May 13, 2017 · May 13, 2017 · May 13, 2017
commit 8c55cec84d5a797b7712af3c8c3e6ba4b425f658
diff --git a/lib/install/action/extract-worker.js b/lib/install/action/extract-worker.js
@@ -0,0 +1,18 @@
+'use strict'
+
+const BB = require('bluebird')
+
+const extract = require('pacote/extract')
+const npmlog = require('npmlog')
+
+module.exports = (args, cb) => {
+  const parsed = typeof args === 'string' ? JSON.parse(args) : args
+  const spec = parsed[0]
+  const extractTo = parsed[1]
+  const opts = parsed[2]
+  if (!opts.log && opts.loglevel) {
+    opts.log = npmlog
+    opts.log.level = opts.loglevel
+  }
+  BB.resolve(extract(spec, extractTo, opts)).nodeify(cb)
+}
diff --git a/lib/install/action/extract.js b/lib/install/action/extract.js
@@ -10,11 +10,28 @@ const moduleName = require('../../utils/module-name.js')
 const moduleStagingPath = require('../module-staging-path.js')
 const move = BB.promisify(require('../../utils/move.js'))
 const npa = require('npm-package-arg')
+const npm = require('../../npm.js')
 const packageId = require('../../utils/package-id.js')
-const pacote = require('pacote')
 let pacoteOpts
 const path = require('path')
+const localWorker = require('./extract-worker.js')
+const workerFarm = require('worker-farm')
 
+const WORKER_PATH = require.resolve('./extract-worker.js')
+let workers
+
+extract.init = () => {
+  workers = workerFarm({
+    maxConcurrentCallsPerWorker: npm.limit.fetch,
+    maxRetries: 1
+  }, WORKER_PATH)
+  return BB.resolve()
+}
+extract.teardown = () => {
+  workerFarm.end(workers)
+  workers = null
+  return BB.resolve()
+}
 module.exports = extract
 function extract (staging, pkg, log) {
   log.silly('extract', packageId(pkg))
@@ -25,13 +42,32 @@ function extract (staging, pkg, log) {
   const opts = pacoteOpts({
     integrity: pkg.package._integrity
   })
-  return pacote.extract(
+  const args = [
     pkg.package._resolved
     ? npa.resolve(pkg.package.name, pkg.package._resolved)
     : pkg.package._requested,
     extractTo,
     opts
-  ).then(() => {
+  ]
+  return BB.fromNode((cb) => {
+    let launcher = localWorker
+    let msg = args
+    const spec = typeof args[0] === 'string' ? npa(args[0]) : args[0]
+    if (spec.registry || spec.type === 'remote') {
+      // We can't serialize these options
+      opts.loglevel = opts.log.level
+      opts.log = null
+      opts.dirPacker = null
+      // workers will run things in parallel!
+      launcher = workers
+      try {
+        msg = JSON.stringify(msg)
+      } catch (e) {
+        return cb(e)
+      }
+    }
+    launcher(msg, cb)
+  }).then(() => {
     if (pkg.package.bundleDependencies) {
       return readBundled(pkg, staging, extractTo)
     }