Python: Improve speed of process-mrva-results.py

RasmusWL · RasmusWL · commit 7d86a8d7f13a · 2023-12-08T11:27:50.000+01:00
Same trick as 'generate-code-scanning-query-list.py'
diff --git a/python/ql/src/meta/ClassHierarchy/process-mrva-results.py b/python/ql/src/meta/ClassHierarchy/process-mrva-results.py
@@ -7,6 +7,8 @@
 import subprocess
 from collections import defaultdict
 import yaml
+import shutil
+import os
 
 VERSION = "process-mrva-results 0.0.1"
 
@@ -18,6 +20,40 @@
 
 # process data
 
+class CodeQL:
+    def __init__(self):
+        pass
+
+    def __enter__(self):
+        self.proc = subprocess.Popen(['codeql', 'execute','cli-server'],
+                      executable=shutil.which('codeql'),
+                      stdin=subprocess.PIPE,
+                      stdout=subprocess.PIPE,
+                      stderr=sys.stderr,
+                      env=os.environ.copy(),
+                     )
+        return self
+    def __exit__(self, type, value, tb):
+        self.proc.stdin.write(b'["shutdown"]\0')
+        self.proc.stdin.close()
+        try:
+            self.proc.wait(5)
+        except:
+            self.proc.kill()
+
+    def command(self, args):
+        data = json.dumps(args)
+        data_bytes = data.encode('utf-8')
+        self.proc.stdin.write(data_bytes)
+        self.proc.stdin.write(b'\0')
+        self.proc.stdin.flush()
+        res = b''
+        while True:
+           b = self.proc.stdout.read(1)
+           if b == b'\0':
+               return res.decode('utf-8')
+           res += b
+
 def wrap_in_template(data):
     return {
         "extensions": [
@@ -46,15 +82,16 @@ def parse_from_file(path: Path) -> set:
 
 
 def gather_from_bqrs_results():
-    for f in glob.glob(f"{sys.argv[1]}/**/results.bqrs", recursive=True):
-        print(f"Processing {f}")
+    with CodeQL() as codeql:
+        for f in glob.glob(f"{sys.argv[1]}/**/results.bqrs", recursive=True):
+            print(f"Processing {f}")
 
-        json_data = subprocess.check_output(["codeql", "bqrs", "decode", "--format=json", f])
-        select = json.loads(json_data)
+            json_data = codeql.command(["bqrs", "decode", "--format=json", f])
+            select = json.loads(json_data)
 
-        for t in select["#select"]["tuples"]:
-            pkg = t[1]
-            package_data[pkg].add(tuple(t))
+            for t in select["#select"]["tuples"]:
+                pkg = t[1]
+                package_data[pkg].add(tuple(t))
 
 def gather_from_existing():
     for f in glob.glob(f"{mad_path}/auto-*.model.yml", recursive=True):
