Run 2to3 conversions in parallel on multicore machines.

mdboom · mdboom · commit 17fafc025b2d · 2011-03-16T15:22:49.000-04:00
diff --git a/setup.py b/setup.py
@@ -252,13 +252,26 @@ def add_dateutil():
         mod.extra_compile_args.append('-DVERBOSE')
 
 if sys.version_info[0] >= 3:
-    # We need to skip certain files that have already been
-    # converted to Python 3.x
+    import multiprocessing
+    from distutils import util
+    def parallel_refactor(x):
+        from lib2to3.refactor import RefactoringTool, get_fixers_from_package
+        class DistutilsRefactoringTool(RefactoringTool):
+            def ignore(self, msg, *args, **kw):
+                pass
+            log_error = log_message = log_debug = ignore
+        fixer_names = get_fixers_from_package('lib2to3.fixes')
+        r = DistutilsRefactoringTool(fixer_names, options=None)
+        r.refactor([x], write=True)
+
     original_build_py = build_py
     class build_py(original_build_py):
         def run_2to3(self, files):
+            # We need to skip certain files that have already been
+            # converted to Python 3.x
             filtered = [x for x in files if 'py3' not in x]
-            original_build_py.run_2to3(self, filtered)
+            p = multiprocessing.Pool()
+            p.map(parallel_refactor, filtered)
 
 print_raw("pymods %s" % py_modules)
 print_raw("packages %s" % packages)
