ai2cm · elynnwu · Nov 4, 2021 · Oct 25, 2021 · Oct 25, 2021 · Oct 25, 2021
diff --git a/Makefile b/Makefile
@@ -6,6 +6,7 @@ CWD=$(shell pwd)
 PULL ?=True
 CONTAINER_ENGINE ?=docker
 RUN_FLAGS ?=--rm
+CHECK_CHANGED_SCRIPT=$(CWD)/changed_from_main.py
 
 
 build:
@@ -18,16 +19,18 @@ dev:
 		$(FV3GFS_IMAGE) bash
 
 test_util:
-	$(MAKE) -C fv3gfs-util test
+	if [ $(shell $(CHECK_CHANGED_SCRIPT) fv3gfs-util) != false ]; then \
+		$(MAKE) -C fv3gfs-util test; \
+	fi
 
 savepoint_tests:
-	$(MAKE) -C fv3core savepoint_tests
+	$(MAKE) -C fv3core $@
 
 savepoint_tests_mpi:
-	$(MAKE) -C fv3core savepoint_tests_mpi
+	$(MAKE) -C fv3core $@
 
-dependencies.svg:
-	dot -Tsvg dependencies.dot -o $@
+dependencies.svg: dependencies.dot
+	dot -Tsvg $< -o $@
 
 constraints.txt: fv3core/requirements.txt fv3core/requirements/requirements_wrapper.txt fv3core/requirements/requirements_lint.txt fv3gfs-util/requirements.txt fv3gfs-physics/requirements.txt
 	pip-compile $^ --output-file constraints.txt
diff --git a/README.md b/README.md
@@ -11,7 +11,7 @@ Currently, we support tests in the dynamical core, physics, and util.
 
 This git repository is laid out as a mono-repo, containing multiple independent projects. Because of this, it is important not to introduce unintended dependencies between projects. The graph below indicates a project depends on another by an arrow pointing from the parent project to its dependency. For example, the tests for fv3core should be able to run with only files contained under the fv3core and util projects, and should not access any files in the driver or physics packages. Only the top-level tests in Pace are allowed to read all files.
 
-![Graph of interdependencies of Pace modules](./dependencies.svg)
+![Graph of interdependencies of Pace modules, generated from dependences.dot](./dependencies.svg)
 
 ### Dynamical core tests
 

diff --git a/changed_from_main.py b/changed_from_main.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+This script determines whether one of the projects in the repo or any of its
+dependencies are different from the version on the `main` branch.
+
+This is useful for running tests only on projects that have changed.
+
+This script should depend only on Python 3.6+ standard libraries.
+"""
+import argparse
+import re
+import os
+import subprocess
+from typing import Any, Dict, Set
+
+
+DIRNAME = os.path.dirname(os.path.abspath(__file__))
+DEPENDENCIES_DOTFILE = os.path.join(DIRNAME, "dependencies.dot")
+
+DEFINITION_PATTERN = re.compile(r"\s*([a-zA-Z0-9]+) \[.*label=\"(.*)\".*\]")
+DEPENDENCY_PATTERN = re.compile(r"\s*([a-zA-Z0-9]+) -> ([a-zA-Z0-9]+)")
+
+
+def get_dependencies() -> Dict[str, Set[str]]:
+    name_to_subdir = {}
+    subdir_dependencies = {}
+    with open(DEPENDENCIES_DOTFILE, "r") as f:
+        dotfile_text = f.read()
+    for groups in DEFINITION_PATTERN.findall(dotfile_text):
+        name_to_subdir[groups[0]] = groups[1]
+        subdir_dependencies[groups[1]] = set()
+    for groups in DEPENDENCY_PATTERN.findall(dotfile_text):
+        name = groups[0]
+        dependency_name = groups[1]
+        if name in name_to_subdir:
+            subdir = name_to_subdir[name]
+            dependency_subdir = name_to_subdir[dependency_name]
+            subdir_dependencies[subdir].add(dependency_subdir)
+    add_nested_dependencies(subdir_dependencies)
+    return subdir_dependencies
+
+
+def add_nested_dependencies(dependency_map: Dict[str, Set[str]]) -> Dict[str, Set[str]]:
+    """
+    Given a dictionary mapping keys to dependent keys which may or may
+    not contain dependencies of dependencies, update it in-place so that
+    dependencies include all sub-dependencies.
+
+    Assumes the dependencies contain no cycles.
+    """
+    # path can be at most as long as the total number of items
+    for _ in range(len(dependency_map)):
+        for dependencies in dependency_map.values():
+            for dependent_key in dependencies:
+                dependencies.update(dependency_map[dependent_key])
+
+
+def parse_args(subdir_dependencies: Dict[str, Any]):
+    parser = argparse.ArgumentParser(
+        description=(
+            "Determines whether one of the projects in the repo or any of its "
+            "dependencies are different from the version on the `main` branch. "
+            "Prints \"false\" if the subdirectory and its dependencies are "
+            "unchanged, or \"true\" if they have changed."
+        )
+    )
+    parser.add_argument(
+        "project_name", type=str, help="subdirectory name of project to check", choices=subdir_dependencies.keys()
+    )
+    return parser.parse_args()
+
+
+def unstaged_files(dirname) -> bool:
+    result = subprocess.check_output(
+        ["git", "ls-files", "--other", "--directory", "--exclude-standard", dirname]
+    )
+    return len(result) > 0
+
+
+def staged_files_changed(dirname) -> bool:
+    result = subprocess.check_output(
+        ["git", "diff", "main", dirname]
+    )
+    return len(result) > 0
+
+
+def changed(dirname) -> bool:
+    return unstaged_files(dirname) or staged_files_changed(dirname)
+
+
+if __name__ == '__main__':
+    subdir_dependencies = get_dependencies()
+    args = parse_args(subdir_dependencies)
+    if changed(args.project_name):
+        print("true")
+    else:
+        for dependency_subdir in subdir_dependencies[args.project_name]:
+            if changed(dependency_subdir):
+                print("true")
+                break
+        else:
+            print("false")
diff --git a/dependencies.dot b/dependencies.dot
@@ -1,14 +1,20 @@
+# this dotfile is used as a reference source for project dependencies
+# each folder entry must have a "label" equal to its directory name
+#
+# If you update this file, please re-generate the svg with `make dependencies.svg`
+# and commit it to the repository
+
 digraph {
-  Pace [shape=box]
-  fv3core [shape=oval]
-  driver [shape=oval]
-  physics [shape=oval]
-  util [shape=oval]
+  pace [shape=box]
+  fv3core [shape=oval, label="fv3core"]
+  driver [shape=oval, label="driver"]
+  physics [shape=oval, label="fv3gfs-physics"]
+  util [shape=oval, label="fv3gfs-util"]
 
-  Pace -> fv3core
-  Pace -> physics
-  Pace -> util
-  Pace -> driver
+  pace -> fv3core
+  pace -> physics
+  pace -> util
+  pace -> driver
   driver -> fv3core
   driver -> physics
   driver -> util

diff --git a/dependencies.svg b/dependencies.svg