Run flake8 incrementally (mypyc/mypyc#103)

JukkaL · web-flow · commit 2e6adcefc0fe · 2018-06-05T17:16:49.000+01:00
Don't lint files that have not changed since the previous clean flake8
run. This speeds up typical lint runtimes by about a second or so on
my laptop. The savings might not be that big right now, but as we add
more code they will add up.

This might break some cross-module checks that flake8 might perform,
but we can disable these if they turn out to be a problem.

Manually removing the .mypyc-flake8-cache.json file will force a full
flake8 run.
diff --git a/.gitignore b/.gitignore
@@ -14,3 +14,4 @@ test_capi
 /.pytest_cache
 /dmypy.json
 /.dmypy.json
+/.mypyc-flake8-cache.json
diff --git a/mypyc/test/test_external.py b/mypyc/test/test_external.py
@@ -53,6 +53,48 @@ def test_self_type_check(self) -> None:
 
     def test_flake8(self) -> None:
         """Use flake8 to lint mypyc."""
-        status = subprocess.call([sys.executable, '-m', 'flake8', 'mypyc'])
+        status = cached_flake8('mypyc')
         if status != 0:
             assert False, "Lint failure"
+
+
+CACHE_FILE = '.mypyc-flake8-cache.json'
+
+
+def cached_flake8(dir: str) -> int:
+    """Run flake8 with cached results from the previous clean run.
+
+    Record the hashes of files after a clean run and don't pass the files to
+    flake8 if they have the same hashes, as they are expected to produce a
+    clean output.
+
+    If flake8 settings or version change, the cache file may need to be manually
+    removed.
+    """
+    import hashlib
+    import glob
+    import json
+
+    if os.path.isfile(CACHE_FILE):
+        with open(CACHE_FILE) as f:
+            cache = json.load(f)
+    else:
+        cache = {}
+    files = glob.glob('%s/**/*.py' % dir, recursive=True)
+    hashes = {}
+    for fn in files:
+        with open(fn, 'rb') as fb:
+            data = fb.read()
+        hashes[fn] = hashlib.sha1(data).hexdigest()
+    filtered = [fn for fn in files
+                if hashes[fn] != cache.get(fn)]
+    if not filtered:
+        # Nothing has changed since the previous clean run -- must be clean.
+        return 0
+    status = subprocess.call([sys.executable, '-m', 'flake8'] + filtered)
+    if status == 0:
+        for fn, sha in hashes.items():
+            cache[fn] = sha
+        with open(CACHE_FILE, 'w') as f:
+            json.dump(cache, f, indent=4)
+    return status
