saltstack · dmurphy18 · Jul 23, 2024 · Aug 15, 2024
@@ -0,0 +1 @@
+Ensure sync from _grains occurs before attempting pillar compilation in case custom grain used in pillar file and salt-minion in masterless mode
@@ -46,18 +46,30 @@
 MAX_FILENAME_LENGTH = 255
 
 
-def get_file_client(opts, pillar=False):
+def get_file_client(opts, pillar=False, force_local=False):
     """
     Read in the ``file_client`` option and return the correct type of file
     server
     """
-    client = opts.get("file_client", "remote")
+    print(
+        f"DGM get_file_client entry, opts '{opts}', pillar '{pillar}', force_local '{force_local}'",
+        flush=True,
+    )
+    if force_local:
+        client = "local"
+    else:
+        client = opts.get("file_client", "remote")
 
     if pillar and client == "local":
         client = "pillar"
-    return {"remote": RemoteClient, "local": FSClient, "pillar": PillarClient}.get(
+    # DGM return {"remote": RemoteClient, "local": FSClient, "pillar": PillarClient}.get(
+    # DGM     client, RemoteClient
+    # DGM )(opts)
+    dgm_ret = {"remote": RemoteClient, "local": FSClient, "pillar": PillarClient}.get(
         client, RemoteClient
     )(opts)
+    print(f"DGM get_file_client exit, ret '{dgm_ret}'", flush=True)
+    return dgm_ret
 
 
 def decode_dict_keys_to_str(src):

@@ -113,6 +113,26 @@
 # 6. Handle publications
 
 
+def _sync_grains(opts):
+    # if local client (masterless minion), need sync of custom grains
+    # as they may be used in pillar compilation
+    # in addition, with masterless minion some opts may not be filled
+    # at this point of syncing,for example sometimes does not contain
+    # extmod_whitelist and extmod_blacklist hence set those to defaults,
+    # empty dict, if not part of opts, as ref'd in
+    # salt.utils.extmod sync function
+    dgm_fc_opts = opts.get("file_client", "remote")
+    print(f"DGM _sync_grains entry file_client settings '{dgm_fc_opts}'", flush=True)
+    if "local" == opts.get("file_client", "remote"):
+        if opts.get("extmod_whitelist", None) is None:
+            opts["extmod_whitelist"] = {}
+
+        if opts.get("extmod_blacklist", None) is None:
+            opts["extmod_blacklist"] = {}
+
+        salt.utils.extmods.sync(opts, "grains", force_local=True)
+
+
 def resolve_dns(opts, fallback=True):
     """
     Resolves the master_ip and master_uri options
@@ -925,6 +945,7 @@ def __init__(self, opts, context=None):
         # Late setup of the opts grains, so we can log from the grains module
         import salt.loader
 
+        _sync_grains(opts)
         opts["grains"] = salt.loader.grains(opts)
         super().__init__(opts)
 

@@ -39,10 +39,16 @@ def sync(
     saltenv=None,
     extmod_whitelist=None,
     extmod_blacklist=None,
+    force_local=False,
 ):
     """
     Sync custom modules into the extension_modules directory
     """
+    print(
+        f"DGM syc entry, opts '{opts}', form '{form}', saltenv '{saltenv}', extmod_whitelist '{extmod_whitelist}', extmod_blacklist '{extmod_blacklist}', force_local '{force_local}'",
+        flush=True,
+    )
+
     if saltenv is None:
         saltenv = ["base"]
 
@@ -82,7 +88,9 @@ def sync(
                         "Cannot create cache module directory %s. Check permissions.",
                         mod_dir,
                     )
-            with salt.fileclient.get_file_client(opts) as fileclient:
+            with salt.fileclient.get_file_client(
+                opts, pillar=False, force_local=force_local
+            ) as fileclient:
                 for sub_env in saltenv:
                     log.info("Syncing %s for environment '%s'", form, sub_env)
                     cache = []
@@ -153,4 +161,6 @@ def sync(
                         shutil.rmtree(emptydir, ignore_errors=True)
         except Exception as exc:  # pylint: disable=broad-except
             log.error("Failed to sync %s module: %s", form, exc)
+
+    print(f"DGM syc exit, ret '{ret}', touched '{touched}',  opts '{opts}'", flush=True)
     return ret, touched
@@ -531,3 +531,93 @@ def test_cve_2024_37088(salt_master_alt, salt_call_alt, caplog):
         assert ret.returncode == 1
         assert ret.data is None
         assert "Got a bad pillar from master, type str, expecting dict" in caplog.text
+
+
+def test_state_highstate_custom_grains_masterless_mode(
+    salt_master, salt_minion_factory
+):
+    """
+    This test ensure that custom grains in salt://_grains are loaded before pillar compilation
+    to ensure that any use of custom grains in pillar files are available when in masterless mode,
+    this implies that a sync of grains occurs before loading the regular
+    /etc/salt/grains or configuration file grains, as well as the usual grains.
+    Note: cannot use salt_minion and salt_call_cli, since these will be loaded before
+    the pillar and custom_grains files are written, hence using salt_minion_factory.
+    """
+    pillar_top_sls = """
+    base:
+      '*':
+        - defaults
+        """
+
+    pillar_defaults_sls = """
+    mypillar: "{{ grains['custom_grain'] }}"
+    """
+
+    salt_top_sls = """
+    base:
+      '*':
+        - test
+        """
+
+    salt_test_sls = """
+    "donothing":
+      test.nop: []
+    """
+
+    salt_custom_grains_py = """
+    def main():
+        return {'custom_grain': 'test_value'}
+    """
+
+    assert salt_master.is_running()
+    with salt_minion_factory.started():
+        salt_minion = salt_minion_factory
+        salt_call_cli = salt_minion_factory.salt_call_cli()
+        with salt_minion.pillar_tree.base.temp_file(
+            "top.sls", pillar_top_sls
+        ), salt_minion.pillar_tree.base.temp_file(
+            "defaults.sls", pillar_defaults_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "top.sls", salt_top_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "test.sls", salt_test_sls
+        ), salt_minion.state_tree.base.temp_file(
+            "_grains/custom_grain.py", salt_custom_grains_py
+        ):
+            ## need to try masterless mode
+            opts = salt_minion.config.copy()
+            opts["file_client"] = "local"
+
+            ret = salt_call_cli.run("--local", "state.highstate")
+            assert ret.returncode == 0
+            ret = salt_call_cli.run("pillar.items")
+            print(
+                f"DGM test_state_highstate_custom_grains_masterless_mode, ret '{ret}'",
+                flush=True,
+            )
+            assert ret.returncode == 0
+            assert ret.data
+            pillar_items = ret.data
+            assert "mypillar" in pillar_items
+            assert pillar_items["mypillar"] == "test_value"
+
+            ## need to try with master mode
+            ret = salt_call_cli.run("state.highstate")
+            assert ret.returncode == 0
+            ret = salt_call_cli.run("pillar.items")
+            assert ret.returncode == 0
+            assert ret.data
+            pillar_items = ret.data
+            assert "mypillar" not in pillar_items
+
+
+def test_salt_call_versions(salt_call_cli, caplog):
+    """
+    Call test.versions without '--local' to test grains
+    are sync'd without any missing keys in opts
+    """
+    with caplog.at_level(logging.DEBUG):
+        ret = salt_call_cli.run("test.versions")
+        assert ret.returncode == 0
+        assert "Failed to sync grains module: 'master_uri'" not in caplog.messages
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Ensure sync from _grains occurs before attempting pillar compilation in case custom grain used in pillar file and salt-minion in masterless mode