Improve path handling for modules

alexrudy · alexrudy · commit 167be85bab76 · 2019-07-29T09:15:11.000-07:00
diff --git a/doc/sphinxext/missing_references.py b/doc/sphinxext/missing_references.py
@@ -18,13 +18,14 @@
 from collections import defaultdict
 import json
 import logging
-import os.path
-import posixpath
+from pathlib import Path, PosixPath
 
 from docutils.utils import get_source_line
 from docutils import nodes
 from sphinx.util import logging as sphinx_logging
 
+import matplotlib
+
 logger = sphinx_logging.getLogger(__name__)
 
 
@@ -42,11 +43,11 @@ def _record_reference(self, record):
                 isinstance(getattr(record, 'location', None), nodes.Node)):
             return
 
-        if not hasattr(self.app.env, "missing_reference_warnings"):
-            self.app.env.missing_reference_warnings = defaultdict(set)
+        if not hasattr(self.app.env, "missing_references_warnings"):
+            self.app.env.missing_references_warnings = defaultdict(set)
 
         record_missing_reference(self.app,
-            self.app.env.missing_reference_warnings,
+            self.app.env.missing_references_warnings,
             record.location)
 
     def filter(self, record):
@@ -60,9 +61,9 @@ def record_missing_reference(app, record, node):
     target = node["reftarget"]
     location = get_location(node, app)
 
-    dtype = "{}:{}".format(domain, typ)
+    domain_type = "{}:{}".format(domain, typ)
 
-    record[(dtype, target)].add(location)
+    record[(domain_type, target)].add(location)
 
 
 def record_missing_reference_handler(app, env, node, contnode):
@@ -75,10 +76,10 @@ def record_missing_reference_handler(app, env, node, contnode):
         # no-op when we are disabled.
         return
 
-    if not hasattr(env, "missing_reference_events"):
-        env.missing_reference_events = defaultdict(set)
+    if not hasattr(env, "missing_references_events"):
+        env.missing_references_events = defaultdict(set)
 
-    record_missing_reference(app, env.missing_reference_events, node)
+    record_missing_reference(app, env.missing_references_events, node)
 
 
 def get_location(node, app):
@@ -97,56 +98,92 @@ def get_location(node, app):
 
     if path:
 
-        basepath = os.path.abspath(os.path.join(app.confdir, ".."))
-        path = os.path.relpath(path, start=basepath)
+        # We locate references relative to the parent of the doc
+        # directory, which for matplotlib, will be the root of the
+        # matplotlib repo. When matplotlib is not an editable install
+        # wierd things will happen, but we can't totally recover from
+        # that.
+        basepath = Path(app.srcdir).parent.resolve()
+
+        fullpath = Path(path).resolve()
+
+        try:
+            path = fullpath.relative_to(basepath)
+        except ValueError:
+            # Sometimes docs directly contain e.g. docstrings
+            # from installed modules, and we record those as
+            # <external> so as to be independent of where the
+            # module was installed
+            path = Path("<external>") / fullpath.name
 
-        if path.startswith(os.path.pardir):
-            path = posixpath.join("<external>", os.path.basename(path))
+        # Ensure that all reported paths are POSIX so that docs
+        # on windows result in the same warnings in the JSON file.
+        path = path.as_posix()
 
     else:
         path = "<unknown>"
 
-    line = str(line) if line else ""
+    if not line:
+        line = ""
+
     return f"{path}:{line}"
 
 
-def save_missing_references_handler(app, exc):
-    """
-    At the end of the sphinx build, either save the missing references to a
-    JSON file. Also ensure that all lines of the existing JSON file are still
-    necessary.
-    """
-    if not app.config.missing_references_enabled:
-        # no-op when we are disabled.
+def _warn_unused_missing_references(app):
+    if not app.config.missing_references_warn_unused_ignores:
         return
 
-    json_path = os.path.join(app.confdir,
-                             app.config.missing_references_filename)
+    # We can only warn if we are building from a source install
+    # otherwise, we just have to skip this step.
+    basepath = Path(matplotlib.__file__).parent.parent.parent.resolve()
+    srcpath = Path(app.srcdir).parent.resolve()
 
-    references_warnings = getattr(app.env, 'missing_reference_warnings', {})
+    if basepath != srcpath:
+        return
 
-    # This is a dictionary of {(dtype,target): locations}
+    # This is a dictionary of {(domain_type, target): locations}
     references_ignored = getattr(app.env,
         'missing_references_ignored_references', {})
-    references_events = getattr(app.env, 'missing_reference_events', {})
+    references_events = getattr(app.env, 'missing_references_events', {})
 
     # Warn about any reference which is no longer missing.
-    for (dtype, target), locations in references_ignored.items():
+    for (domain_type, target), locations in references_ignored.items():
         missing_reference_locations = references_events.get(
-                (dtype, target), [])
+                (domain_type, target), [])
 
-        # For each ignored reference location, ensure a missing reference was
-        # observed. If it wasn't observed, issue a warning.
+        # For each ignored reference location, ensure a missing reference
+        # was observed. If it wasn't observed, issue a warning.
         for ignored_reference_location in locations:
-            if ignored_reference_location not in missing_reference_locations:
-                msg = (f"Reference {dtype} {target} for "
-                       f"{ignored_reference_location} can be removed"
-                       f" from {app.config.missing_references_filename}."
+            if (ignored_reference_location not in
+                    missing_reference_locations):
+                msg = (f"Reference {domain_type} {target} for "
+                    f"{ignored_reference_location} can be removed"
+                    f" from {app.config.missing_references_filename}."
                         "It is no longer a missing reference in the docs.")
                 logger.warning(msg,
                     location=ignored_reference_location,
                     type='ref',
-                    subtype=dtype)
+                    subtype=domain_type)
+
+
+def save_missing_references_handler(app, exc):
+    """
+    At the end of the sphinx build, check that all lines of the existing JSON
+    file are still necessary.
+
+    If the configuration value ``missing_references_write_json`` is set
+    then write a new JSON file containing missing references.
+    """
+    if not app.config.missing_references_enabled:
+        # no-op when we are disabled.
+        return
+
+    _warn_unused_missing_references(app)
+
+    json_path = (Path(app.confdir) /
+                 app.config.missing_references_filename)
+
+    references_warnings = getattr(app.env, 'missing_references_warnings', {})
 
     if app.config.missing_references_write_json:
         _write_missing_references_json(references_warnings, json_path)
@@ -156,15 +193,15 @@ def _write_missing_references_json(records, json_path):
     """
     Convert ignored references to a format which we can write as JSON
 
-    Convert from ``{(dtype, target): locaitons}`` to
-    ``{dtype: {target: locations}}`` since JSON can't serialize tuples.
+    Convert from ``{(domain_type, target): locaitons}`` to
+    ``{domain_type: {target: locations}}`` since JSON can't serialize tuples.
     """
     transformed_records = defaultdict(dict)
 
-    for (dtype, target), paths in records.items():
-        transformed_records[dtype][target] = sorted(paths)
+    for (domain_type, target), paths in records.items():
+        transformed_records[domain_type][target] = sorted(paths)
 
-    with open(json_path, "w") as stream:
+    with json_path.open("w") as stream:
         json.dump(transformed_records, stream, indent=2)
 
 
@@ -173,24 +210,24 @@ def _read_missing_references_json(json_path):
     Convert from the JSON file to the form used internally by this
     extension.
 
-    The JSON file is stored as ``{dtype: {target: [locations,]}}`` since JSON
-    can't store dictionary keys which are tuples. We convert this back to
-    ``{(dtype,target):[locations]}`` for internal use.
+    The JSON file is stored as ``{domain_type: {target: [locations,]}}``
+    since JSON can't store dictionary keys which are tuples. We convert
+    this back to ``{(domain_type, target):[locations]}`` for internal use.
 
     """
-    with open(json_path, "r") as stream:
+    with json_path.open("r") as stream:
         data = json.load(stream)
 
     ignored_references = {}
-    for dtype, targets in data.items():
+    for domain_type, targets in data.items():
         for target, locations in targets.items():
-            ignored_references[(dtype, target)] = locations
+            ignored_references[(domain_type, target)] = locations
     return ignored_references
 
 
 def prepare_missing_references_handler(app):
     """
-    Handler called to initalize this extension once the configuration
+    Handler called to initialize this extension once the configuration
     is ready.
 
     Reads the missing references file and populates ``nitpick_ignore`` if
@@ -203,8 +240,8 @@ def prepare_missing_references_handler(app):
     sphinx_logger = logging.getLogger('sphinx')
     missing_reference_filter = MissingReferenceFilter(app)
     for handler in sphinx_logger.handlers[:]:
-        if (isinstance(handler, sphinx_logging.WarningStreamHandler) and
-            missing_reference_filter not in handler.filters):
+        if (isinstance(handler, sphinx_logging.WarningStreamHandler)
+                and missing_reference_filter not in handler.filters):
 
             # This *must* be the first filter, because subsequent filters
             # throw away the node information and then we can't identify
@@ -213,16 +250,16 @@ def prepare_missing_references_handler(app):
 
     app.env.missing_references_ignored_references = {}
 
-    json_path = os.path.join(app.confdir,
-                             app.config.missing_references_filename)
-    if not os.path.exists(json_path):
+    json_path = (Path(app.confdir) /
+                    app.config.missing_references_filename)
+    if not json_path.exists():
         return
 
     ignored_references = _read_missing_references_json(json_path)
 
     app.env.missing_references_ignored_references = ignored_references
 
-    # If we are going to re-write the JSON file, then don't supress missing
+    # If we are going to re-write the JSON file, then don't suppress missing
     # reference warnings. We want to record a full list of missing references
     # for use later. Otherwise, add all known missing references to
     # ``nitpick_ignore```
@@ -233,6 +270,7 @@ def prepare_missing_references_handler(app):
 def setup(app):
     app.add_config_value("missing_references_enabled", True, "env")
     app.add_config_value("missing_references_write_json", False, "env")
+    app.add_config_value("missing_references_warn_unused_ignores", True, "env")
     app.add_config_value("missing_references_filename",
                          "missing-references.json", "env")
 
