Better way to treat interpolation errors

epcim · epcim · commit 6898740d5937 · 2018-03-14T16:25:40.000+01:00
diff --git a/README-extentions.rst b/README-extentions.rst
@@ -123,6 +123,43 @@ The ``${beta:${alpha:two}}`` construct first resolves the ``${alpha:two}`` refer
 the reference ``${beta:a}`` to the value 99.
 
 
+Return missing references
+-------------------------
+
+Similarly as new "saltclass" it may return missed references instead of throw InterpolationError.
+
+This is supposed to be feature, as
+- you can interpolate them in another ext_pillar engine.
+- your pillars will not fail.
+
+Be aware that if this option is enabled, it may resolve in rendering an "reference string" as value in the system critical
+files without notice.
+
+Default value is False to keep backward compatible behavior.
+
+.. code-block:: yaml
+
+  return_missing_reference: False
+
+
+Print summary of missed references
+----------------------------------
+
+Instead of fail on first interpolation error it will print all missed references at once.
+Also will print warnings if reference is used before it is defined in class hierarchy.
+
+.. code-block:: yaml
+  reclass --nodeinfo mynode
+  [WARNING] Reference '_param:kkk' undefined. Possibly used too early and defined later in class hierarchy.
+  [WARNING] Reference '_param:yyy' undefined. Possibly used too early and defined later in class hierarchy.
+  [WARNING] Reference '_param:kkk' undefined. Possibly used too early and defined later in class hierarchy.
+  [WARNING] Reference '_param:kkk' undefined. Possibly used too early and defined later in class hierarchy.
+  None
+
+  -> dontpanic
+     Cannot resolve ${_param:kkk}, in mykey1:path:to:fail
+     Cannot resolve ${_param:kkk}, in mykey2:another:path:to:fail
+
 
 Inventory Queries
 -----------------
diff --git a/reclass/core.py b/reclass/core.py
@@ -187,7 +187,13 @@ def _nodeinfo(self, nodename, inventory):
             node.initialise_interpolation()
             if node.parameters.has_inv_query() and inventory is None:
                 inventory = self._get_inventory(node.parameters.needs_all_envs(), node.environment, node.parameters.get_inv_queries())
+
+            return_missing_reference = self._settings.return_missing_reference
+            self._settings.return_missing_reference = True
             node.interpolate(inventory)
+            self._settings.return_missing_reference = return_missing_reference
+            if not return_missing_reference:
+                node.check_failed_interpolation()
             return node
         except InterpolationError as e:
             e.nodename = nodename
diff --git a/reclass/datatypes/entity.py b/reclass/datatypes/entity.py
@@ -10,6 +10,8 @@
 from applications import Applications
 from exports import Exports
 from parameters import Parameters
+from reclass.errors import ResolveError, InterpolationError, ResolveErrorList
+from reclass.defaults import REFERENCE_SENTINELS, EXPORT_SENTINELS
 
 class Entity(object):
     '''
@@ -99,6 +101,14 @@ def interpolate_exports(self):
     def interpolate_single_export(self, references):
         self._exports.interpolate_single_from_external(self._parameters, references)
 
+    def check_failed_interpolation(self):
+        p = [ ResolveError(value[len(REFERENCE_SENTINELS[0]):-len(REFERENCE_SENTINELS[1])], uri=path) for path,value in self._parameters.check_failed_interpolation() ]
+        if len(p) > 0:
+            raise ResolveErrorList(p)
+        e = [ ResolveError(value[len(REFERENCE_SENTINELS[0]):-len(REFERENCE_SENTINELS[1])], uri=path) for path,value in self._exports.check_failed_interpolation() ]
+        if len(e) > 0:
+            raise ResolveErrorList(e)
+
     def __eq__(self, other):
         return isinstance(other, type(self)) \
                 and self._applications == other._applications \
diff --git a/reclass/datatypes/parameters.py b/reclass/datatypes/parameters.py
@@ -10,11 +10,13 @@
 import copy
 import sys
 import types
+import re
 from collections import namedtuple
 from reclass.utils.dictpath import DictPath
 from reclass.values.value import Value
 from reclass.values.valuelist import ValueList
 from reclass.errors import InfiniteRecursionError, ResolveError, InterpolationError, ParseError, BadReferencesError
+from reclass.defaults import REFERENCE_SENTINELS, EXPORT_SENTINELS, REFERENCE_PATTERN
 
 class Parameters(object):
     '''
@@ -47,6 +49,8 @@ def __init__(self, mapping, settings, uri):
         self._unrendered = None
         self._escapes_handled = {}
         self._inv_queries = []
+        self._ref_missed = []
+        self._ref_pattern = '.*'+ re.escape(REFERENCE_SENTINELS[0]) + REFERENCE_PATTERN + re.escape(REFERENCE_SENTINELS[1])
         self._needs_all_envs = False
         self._keep_overrides = False
         if mapping is not None:
@@ -282,6 +286,8 @@ def _interpolate_render_value(self, path, value, inventory):
             self._render_simple_dict(new, path)
         elif isinstance(new, list):
             self._render_simple_list(new, path)
+        if re.match(self._ref_pattern, str(new)):
+           self._ref_missed.append(path)
         return new
 
     def _interpolate_references(self, path, value, inventory):
@@ -318,3 +324,12 @@ def _interpolate_references(self, path, value, inventory):
                 value.assembleRefs(self._base)
                 if old == len(value.get_references()):
                     raise BadReferencesError(value.get_references(), str(path), value.uri())
+
+    def check_failed_interpolation(self):
+        missing = []
+        for p in self._ref_missed:
+            v = str(p.get_value(self._base))
+            if re.match(self._ref_pattern, v):
+               missing.append((str(p), v))
+        return missing
+
diff --git a/reclass/defaults.py b/reclass/defaults.py
@@ -21,6 +21,8 @@
 OPT_IGNORE_CLASS_NOTFOUND_REGEXP = ['.*']
 OPT_IGNORE_CLASS_NOTFOUND_WARNING = True
 
+OPT_RETURN_MISSING_REFERENCE = False
+
 OPT_ALLOW_SCALAR_OVER_DICT = False
 OPT_ALLOW_SCALAR_OVER_LIST = False
 OPT_ALLOW_LIST_OVER_SCALAR = False
@@ -41,6 +43,7 @@
 PARAMETER_INTERPOLATION_DELIMITER = ':'
 PARAMETER_DICT_KEY_OVERRIDE_PREFIX = '~'
 ESCAPE_CHARACTER = '\\'
+REFERENCE_PATTERN = '[-_{}.\w]+'.format(PARAMETER_INTERPOLATION_DELIMITER)
 
 AUTOMATIC_RECLASS_PARAMETERS = True
 DEFAULT_ENVIRONMENT = 'base'
diff --git a/reclass/errors.py b/reclass/errors.py
@@ -178,10 +178,28 @@ def __init__(self, reference, uri=None, context=None):
         super(ResolveError, self).__init__(msg=None)
         self.reference = reference
 
+        if uri:
+            self.uri = uri
+
+        if context:
+            self.context = context
+
     def _get_error_message(self):
         msg = 'Cannot resolve {0}'.format(self.reference.join(REFERENCE_SENTINELS)) + self._add_context_and_uri()
         return [ msg ]
 
+class ResolveErrorList(InterpolationError):
+
+    def __init__(self, resolve_errors=[]):
+        super(ResolveErrorList, self).__init__(msg=None)
+        self.resolve_errors = resolve_errors
+
+    def _get_error_message(self):
+        msgs = []
+        for e in self.resolve_errors:
+            msgs.append('Cannot resolve {0}'.format(e.reference.join(REFERENCE_SENTINELS)) + e._add_context_and_uri())
+        return msgs
+
 
 class InvQueryError(InterpolationError):
 
diff --git a/reclass/settings.py b/reclass/settings.py
@@ -26,6 +26,10 @@ def __init__(self, options={}):
 
         self.ignore_class_notfound_warning = options.get('ignore_class_notfound_warning', OPT_IGNORE_CLASS_NOTFOUND_WARNING)
 
+        # passing missing reference is a feature (for example in saltclass),
+        # allow additional processing on next ext_pillar.
+        self.return_missing_reference = options.get('return_missing_reference', OPT_RETURN_MISSING_REFERENCE)
+
 
         self.ref_parser = reclass.values.parser_funcs.get_ref_parser(self.escape_character, self.reference_sentinels, self.export_sentinels)
         self.simple_ref_parser = reclass.values.parser_funcs.get_simple_ref_parser(self.escape_character, self.reference_sentinels, self.export_sentinels)
diff --git a/reclass/values/refitem.py b/reclass/values/refitem.py
@@ -8,6 +8,8 @@
 from reclass.settings import Settings
 from reclass.utils.dictpath import DictPath
 from reclass.errors import ResolveError
+from reclass.defaults import REFERENCE_SENTINELS, EXPORT_SENTINELS
+import sys
 
 class RefItem(Item):
 
@@ -52,7 +54,11 @@ def _resolve(self, ref, context):
         try:
             return path.get_value(context)
         except (KeyError, TypeError) as e:
-            raise ResolveError(ref)
+            print >>sys.stderr, "[WARNING] Reference '%s' undefined. Possibly used too early and defined later in class hierarchy." % (ref)
+            if self._settings.return_missing_reference:
+                return "%s" % ref.join(REFERENCE_SENTINELS)
+            else:
+                raise ResolveError(ref)
 
     def render(self, context, inventory):
         if len(self._items) == 1:
