From 43a072dd17ac46254fb3ff91c930b6e0508c921f Mon Sep 17 00:00:00 2001 From: OpenShift Cherrypick Robot Date: Sat, 6 Jun 2020 17:03:17 +0200 Subject: [PATCH 1/3] [release-0.11] Recursive diff (#370) * Fix strategic patch merge of env variables The key is `env`, not `envVars` * Add a Kubernetes specific recursive dict diff Understands strategic patch keys and can therefore better diff list elements * Remove unused dictdiffer dependency Co-authored-by: Will Thames --- openshift/dynamic/apply.py | 62 ++++++++++++++++++++++++++++++++-- python-openshift.spec | 4 +-- requirements.txt | 1 - test-requirements.txt | 1 - test/unit/test_diff.py | 69 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 test/unit/test_diff.py diff --git a/openshift/dynamic/apply.py b/openshift/dynamic/apply.py index 3b0ef325..3b4fb81d 100644 --- a/openshift/dynamic/apply.py +++ b/openshift/dynamic/apply.py @@ -15,15 +15,15 @@ 'imagePullSecrets': 'name', 'containers.volumeMounts': 'mountPath', 'containers.volumeDevices': 'devicePath', - 'containers.envVars': 'name', + 'containers.env': 'name', 'containers.ports': 'containerPort', 'initContainers.volumeMounts': 'mountPath', 'initContainers.volumeDevices': 'devicePath', - 'initContainers.envVars': 'name', + 'initContainers.env': 'name', 'initContainers.ports': 'containerPort', 'ephemeralContainers.volumeMounts': 'mountPath', 'ephemeralContainers.volumeDevices': 'devicePath', - 'ephemeralContainers.envVars': 'name', + 'ephemeralContainers.env': 'name', 'ephemeralContainers.ports': 'containerPort', } @@ -183,6 +183,62 @@ def list_merge(last_applied, actual, desired, position): return desired +def recursive_list_diff(list1, list2, position=None): + result = (list(), list()) + if position in STRATEGIC_MERGE_PATCH_KEYS: + patch_merge_key = STRATEGIC_MERGE_PATCH_KEYS[position] + dict1 = list_to_dict(list1, patch_merge_key, position) + dict2 = list_to_dict(list2, patch_merge_key, position) + dict1_keys = set(dict1.keys()) + dict2_keys = set(dict2.keys()) + for key in dict1_keys - dict2_keys: + result[0].append(dict1[key]) + for key in dict2_keys - dict1_keys: + result[1].append(dict2[key]) + for key in dict1_keys & dict2_keys: + diff = recursive_diff(dict1[key], dict2[key], position) + if diff: + # reinsert patch merge key to relate changes in other keys to + # a specific list element + diff[0].update({patch_merge_key: dict1[key][patch_merge_key]}) + diff[1].update({patch_merge_key: dict2[key][patch_merge_key]}) + result[0].append(diff[0]) + result[1].append(diff[1]) + if result[0] or result[1]: + return result + elif list1 != list2: + return (list1, list2) + return None + + +def recursive_diff(dict1, dict2, position=None): + if not position: + if 'kind' in dict1 and dict1.get('kind') == dict2.get('kind'): + position = dict1['kind'] + left = dict((k, v) for (k, v) in dict1.items() if k not in dict2) + right = dict((k, v) for (k, v) in dict2.items() if k not in dict1) + for k in (set(dict1.keys()) & set(dict2.keys())): + if position: + this_position = "%s.%s" % (position, k) + if isinstance(dict1[k], dict) and isinstance(dict2[k], dict): + result = recursive_diff(dict1[k], dict2[k], this_position) + if result: + left[k] = result[0] + right[k] = result[1] + elif isinstance(dict1[k], list) and isinstance(dict2[k], list): + result = recursive_list_diff(dict1[k], dict2[k], this_position) + if result: + left[k] = result[0] + right[k] = result[1] + elif dict1[k] != dict2[k]: + left[k] = dict1[k] + right[k] = dict2[k] + if left or right: + return left, right + else: + return None + + def get_deletions(last_applied, desired): patch = {} for k, last_applied_value in last_applied.items(): diff --git a/python-openshift.spec b/python-openshift.spec index cdc15eeb..17f2e93c 100644 --- a/python-openshift.spec +++ b/python-openshift.spec @@ -35,7 +35,6 @@ BuildRequires: python-setuptools BuildRequires: git Requires: python2 -Requires: python2-dictdiffer Requires: python2-kubernetes Requires: python2-string_utils Requires: python-requests @@ -55,7 +54,6 @@ BuildRequires: %{py3}-setuptools BuildRequires: git Requires: %{py3} -Requires: %{py3}-dictdiffer Requires: %{py3}-kubernetes Requires: %{py3}-string_utils Requires: %{py3}-requests @@ -92,7 +90,7 @@ Python client for the OpenShift API #the requirements are also done in an non-backwards compatible way %if 0%{?rhel} sed -i -e "s/find_packages(include='openshift.*')/['openshift', 'openshift.dynamic', 'openshift.helper']/g" setup.py -sed -i -e '30s/^/REQUIRES = [\n "dictdiffer",\n "jinja2",\n "kubernetes",\n "setuptools",\n "six",\n "ruamel.yaml",\n "python-string-utils",\n]\n/g' setup.py +sed -i -e '30s/^/REQUIRES = [\n "jinja2",\n "kubernetes",\n "setuptools",\n "six",\n "ruamel.yaml",\n "python-string-utils",\n]\n/g' setup.py sed -i -e "s/extract_requirements('requirements.txt')/REQUIRES/g" setup.py #sed -i -e '14,21d' setup.py %endif diff --git a/requirements.txt b/requirements.txt index 9cef5704..5fa256fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -dictdiffer jinja2 kubernetes ~= 11.0.0 python-string-utils diff --git a/test-requirements.txt b/test-requirements.txt index 0f7331e6..993602ad 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,4 +5,3 @@ pytest pytest-bdd pytest-cov PyYAML -dictdiffer diff --git a/test/unit/test_diff.py b/test/unit/test_diff.py new file mode 100644 index 00000000..a9b7c3fa --- /dev/null +++ b/test/unit/test_diff.py @@ -0,0 +1,69 @@ +from openshift.dynamic.apply import recursive_diff + +tests = [ + dict( + before = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8080, name="http")]) + ), + after = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8080, name="http")]) + ), + expected = None + ), + dict( + before = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8080, name="http")]) + ), + after = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8081, name="http")]) + ), + expected = ( + dict(spec=dict(ports=[dict(port=8080, name="http")])), + dict(spec=dict(ports=[dict(port=8081, name="http")])) + ) + ), + dict( + before = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8080, name="http"), dict(port=8081, name="https")]) + ), + after = dict( + kind="Service", + metadata=dict(name="foo"), + spec=dict(ports=[dict(port=8081, name="https"), dict(port=8080, name="http")]) + ), + expected = None + ), + + dict( + before = dict( + kind="Pod", + metadata=dict(name="foo"), + spec=dict(containers=[dict(name="busybox", image="busybox", + env=[dict(name="hello", value="world"), + dict(name="another", value="next")])]) + ), + after = dict( + kind="Pod", + metadata=dict(name="foo"), + spec=dict(containers=[dict(name="busybox", image="busybox", + env=[dict(name="hello", value="everyone")])]) + ), + expected=(dict(spec=dict(containers=[dict(name="busybox", env=[dict(name="another", value="next"), dict(name="hello", value="world")])])), + dict(spec=dict(containers=[dict(name="busybox", env=[dict(name="hello", value="everyone")])]))) + ), + ] + + +def test_diff(): + for test in tests: + assert(recursive_diff(test['before'], test['after']) == test['expected']) From e2e6989e71d6bb6ed578906a39f074ac3b1e2347 Mon Sep 17 00:00:00 2001 From: OpenShift Cherrypick Robot Date: Sat, 6 Jun 2020 17:03:24 +0200 Subject: [PATCH 2/3] Handle list merges where items have been added elsewhere (#371) For example, a Pod has its service account token mounted as a volume - we don't want to later try and remove it. Other examples are likely available Co-authored-by: Will Thames --- openshift/dynamic/apply.py | 3 +++ test/unit/test_apply.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/openshift/dynamic/apply.py b/openshift/dynamic/apply.py index 3b4fb81d..ed42be11 100644 --- a/openshift/dynamic/apply.py +++ b/openshift/dynamic/apply.py @@ -178,6 +178,9 @@ def list_merge(last_applied, actual, desired, position): else: patch = merge(last_applied_dict[key], desired_dict[key], actual_dict[key], position) result.append(dict_merge(actual_dict[key], patch)) + for key in actual_dict: + if key not in desired_dict and key not in last_applied_dict: + result.append(actual_dict[key]) return result else: return desired diff --git a/test/unit/test_apply.py b/test/unit/test_apply.py index d8d74bb0..907d08d2 100644 --- a/test/unit/test_apply.py +++ b/test/unit/test_apply.py @@ -53,6 +53,7 @@ ), expected = dict(metadata=dict(annotations=None), data=dict(two=None, three="3")) ), + dict( last_applied = dict( kind="Service", @@ -165,6 +166,43 @@ expected=dict(spec=dict(containers=[dict(name="busybox", image="busybox", resources=dict(requests=dict(cpu="50m", memory="50Mi"), limits=dict(cpu=None, memory="50Mi")))])) ), + dict( + desired = dict(kind='Pod', + spec=dict(containers=[ + dict(name='hello', + volumeMounts=[dict(name="test", mountPath="/test")]) + ], + volumes=[ + dict(name="test", configMap=dict(name="test")), + ])), + last_applied = dict(kind='Pod', + spec=dict(containers=[ + dict(name='hello', + volumeMounts=[dict(name="test", mountPath="/test")]) + ], + volumes=[ + dict(name="test", configMap=dict(name="test")), + ])), + actual = dict(kind='Pod', + spec=dict(containers=[ + dict(name='hello', + volumeMounts=[dict(name="test", mountPath="/test"), + dict(mountPath="/var/run/secrets/kubernetes.io/serviceaccount", name="default-token-xyz")]) + ], + volumes=[ + dict(name="test", configMap=dict(name="test")), + dict(name="default-token-xyz", secret=dict(secretName="default-token-xyz")), + ])), + expected = dict(spec=dict(containers=[ + dict(name='hello', + volumeMounts=[dict(name="test", mountPath="/test"), + dict(mountPath="/var/run/secrets/kubernetes.io/serviceaccount", name="default-token-xyz")]) + ], + volumes=[ + dict(name="test", configMap=dict(name="test")), + dict(name="default-token-xyz", secret=dict(secretName="default-token-xyz")), + ])), + ), # This next one is based on a real world case where definition was mostly # str type and everything else was mostly unicode type (don't ask me how) From f5c66d2ffa60edfac829a6cc7c94661de0b46b7d Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Sat, 6 Jun 2020 11:07:56 -0400 Subject: [PATCH 3/3] Bump version --- openshift/__init__.py | 2 +- python-openshift.spec | 2 +- scripts/constants.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openshift/__init__.py b/openshift/__init__.py index 0333c745..ae4e3874 100644 --- a/openshift/__init__.py +++ b/openshift/__init__.py @@ -14,5 +14,5 @@ # Do not edit these constants. They will be updated automatically # by scripts/update-version.sh. -__version__ = "0.11.1" +__version__ = "0.11.2" __k8s_client_version__ = "11.0.0" diff --git a/python-openshift.spec b/python-openshift.spec index 17f2e93c..43e148c1 100644 --- a/python-openshift.spec +++ b/python-openshift.spec @@ -16,7 +16,7 @@ %endif Name: python-%{library} -Version: 0.11.1 +Version: 0.11.2 Release: 1%{?dist} Summary: Python client for the OpenShift API License: ASL 2.0 diff --git a/scripts/constants.py b/scripts/constants.py index 426ea96b..9c3c3b5c 100644 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -23,7 +23,7 @@ # client version for packaging and releasing. It can # be different than SPEC_VERSION. -CLIENT_VERSION = "0.11.1" +CLIENT_VERSION = "0.11.2" KUBERNETES_CLIENT_VERSION = "11.0.0" # Name of the release package diff --git a/setup.py b/setup.py index f49ca27c..4c7d8dc8 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # Do not edit these constants. They will be updated automatically # by scripts/update-client.sh. -CLIENT_VERSION = "0.11.1" +CLIENT_VERSION = "0.11.2" PACKAGE_NAME = "openshift" DEVELOPMENT_STATUS = "3 - Alpha"