[CI] Rewrite get_all_files to use Path.glob

malfet · malfet · commit d909b5d2ff6f · 2022-05-22T12:48:08.000-07:00
Rather than call `find` for hard-code list of folder,
search for all `.py` files in folders ending with `_source` excluding
those in `data folder
diff --git a/.jenkins/get_files_to_run.py b/.jenkins/get_files_to_run.py
@@ -1,29 +1,21 @@
 from typing import List
-from subprocess import run
 import json
 import os
 from pathlib import Path
 from remove_runnable_code import remove_runnable_code
 
 
 # Calculate repo base dir
-REPO_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+REPO_BASE_DIR = Path(__file__).absolute().parent.parent
 
-def get_all_files(encoding="utf-8") -> List[str]:
-    sources = [
-        "beginner_source",
-        "intermediate_source",
-        "advanced_source",
-        "recipes_source",
-        "prototype_source",
-    ]
-    cmd = ["find"] + sources + ["-name", "*.py", "-not", "-path", "*/data/*"]
 
-    return run(cmd, capture_output=True, cwd=REPO_BASE_DIR).stdout.decode(encoding).splitlines()
+def get_all_files() -> List[str]:
+    sources = [x.relative_to(REPO_BASE_DIR) for x in REPO_BASE_DIR.glob("*_source/**/*.py") if 'data' not in x.parts]
+    return [str(x) for x in sources]
 
 
 def calculate_shards(all_files, num_shards=20):
-    with open(os.path.join(REPO_BASE_DIR, ".jenkins", "metadata.json")) as fp:
+    with (REPO_BASE_DIR / ".jenkins" / "metadata.json").open() as fp:
         metadata = json.load(fp)
     sharded_files = [(0.0, []) for _ in range(num_shards)]
 
