rtdip · GBBBAS · May 12, 2023 · May 11, 2023 · May 11, 2023 · May 11, 2023
diff --git a/.github/workflows/sonarcloud_reusable.yml b/.github/workflows/sonarcloud_reusable.yml
@@ -47,7 +47,9 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.11"]
+        python-version: ["3.10"]
+        pyspark: ["3.3.2"]
+        delta-spark: ["2.3.0"]
     runs-on: ${{ matrix.os }} 
     steps:
       - uses: actions/checkout@v3
@@ -76,13 +78,15 @@ jobs:
         with:
           environment-file: environment.yml
           extra-specs: |
-            python=${{ matrix.python-version }}           
+            python=${{ matrix.python-version }}
+            pyspark=${{ matrix.pyspark }}
+            delta-spark=${{ matrix.delta-spark }}          
           cache-env: true
 
       - name: Test
         run: |
           mkdir -p coverage-reports
-          coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests && tests_ok=true
+          coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests
           coverage xml --omit "venv/**,maintenance/**,xunit-reports/**" -i -o coverage-reports/coverage-unittests.xml
           echo Coverage `coverage report --omit "venv/**" | grep TOTAL | tr -s ' ' | cut -d" " -f4`
 

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -26,6 +26,14 @@ jobs:
         matrix:
           os: [ubuntu-latest]
           python-version: ["3.8", "3.9", "3.10"]
+          pyspark: ["3.3.0", "3.3.1", "3.3.2"]
+          include:
+            - pyspark: "3.3.0"
+              delta-spark: "2.2.0"
+            - pyspark: "3.3.1"
+              delta-spark: "2.3.0"
+            - pyspark: "3.3.2"
+              delta-spark: "2.3.0"
       runs-on: ${{ matrix.os }}
       steps:
         - uses: actions/checkout@v3
@@ -52,12 +60,14 @@ jobs:
           with:
             environment-file: environment.yml
             extra-specs: |
-              python=${{ matrix.python-version }}           
+              python=${{ matrix.python-version }}
+              pyspark=${{ matrix.pyspark }}
+              delta-spark=${{ matrix.delta-spark }}     
             cache-env: true
 
         - name: Test
           run: |
             mkdir -p coverage-reports-previous
-            coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests && tests_ok=true
+            coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests
             coverage xml --omit "venv/**,maintenance/**,xunit-reports/**" -i -o coverage-reports-previous/coverage-unittests.xml
             echo Coverage `coverage report --omit "venv/**" | grep TOTAL | tr -s ' ' | cut -d" " -f4`
diff --git a/environment.yml b/environment.yml
@@ -18,7 +18,7 @@ channels:
     - conda-forge
     - defaults
 dependencies:
-    - python==3.10.9
+    - python>=3.8,<3.11
     - mkdocs-material==9.1.8
     - mkdocs-material-extensions==1.1.1
     - jinja2==3.0.3
@@ -40,8 +40,8 @@ dependencies:
     - fastapi==0.95.1
     - httpx==0.24.0
     - trio==0.22.0
-    - pyspark==3.3.2
-    - delta-spark==2.3.0
+    - pyspark>=3.3.0,<3.4.0
+    - delta-spark>=2.2.0,<2.4.0
     - openjdk==11.0.15    
     - python-dotenv==1.0.0
     - mkdocstrings==0.21.2
@@ -50,6 +50,7 @@ dependencies:
     - pygments==2.15.1
     - pymdown-extensions==9.11
     - databricks-sql-connector==2.5.1
+    - semver==3.0.0
     - pip:
         - dependency-injector==4.41.0
         - azure-functions==1.14.0

diff --git a/setup.py b/setup.py
@@ -34,12 +34,14 @@
   "pyodbc==4.0.39",
   "pandas==1.5.2",
   "jinja2==3.0.3",
-  "jinjasql==0.1.8"
+  "jinjasql==0.1.8",
+  "importlib_metadata>=1.0.0"
+  "semver==3.0.0"
 ]
 
 PYSPARK_PACKAGES = [
-  "pyspark==3.3.2",
-  "delta-spark==2.3.0",
+  "pyspark>=3.3.0,<3.4.0",
+  "delta-spark>=2.2.0,<2.4.0",
 ]
 
 PIPELINE_PACKAGES = [

diff --git a/src/sdk/python/rtdip_sdk/_sdk_utils/__init__.py b/src/sdk/python/rtdip_sdk/_sdk_utils/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/src/sdk/python/rtdip_sdk/_sdk_utils/compare_versions.py b/src/sdk/python/rtdip_sdk/_sdk_utils/compare_versions.py
@@ -0,0 +1,23 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from importlib_metadata import version
+from semver.version import Version
+
+def _package_version_meets_minimum(package_name: str, minimum_version: str) -> bool:
+    package_version = Version.parse(version(package_name))
+    version_result =  Version.compare(package_version, minimum_version)
+    if version_result < 0:
+        raise AssertionError("Package {} with version {} does not meet minimum version requirement {}".format(package_name, str(package_version), minimum_version))
+    return True
diff --git a/src/sdk/python/rtdip_sdk/functions/metadata.py b/src/sdk/python/rtdip_sdk/functions/metadata.py
@@ -35,7 +35,7 @@ def get(connection: object, parameters_dict: dict) -> pd.DataFrame:
         region (str): Region
         asset (str): Asset 
         data_security_level (str): Level of data security
-        tag_names (list): (Optional) Either pass a list of tagname/tagnames ["tag_1", "tag_2"] or leave the list blank [] or leave the parameter out completely
+        tag_names (optional, list): Either pass a list of tagname/tagnames ["tag_1", "tag_2"] or leave the list blank [] or leave the parameter out completely
 
     Returns:
         DataFrame: A dataframe of metadata.

diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/models.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/models.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from enum import Enum
-from typing import Optional
+from typing import List, Optional
 from pydantic import BaseModel
 
 class SystemType(Enum):
@@ -51,9 +51,9 @@ class PythonWheelLibrary(BaseModel):
     path: str
 
 class Libraries(BaseModel):
-    maven_libraries: list[MavenLibrary] = []
-    pypi_libraries: list[PyPiLibrary] = []
-    pythonwheel_libraries: list[PythonWheelLibrary] = []
+    maven_libraries: List[MavenLibrary] = []
+    pypi_libraries: List[PyPiLibrary] = []
+    pythonwheel_libraries: List[PythonWheelLibrary] = []
 
     def add_maven_library(self, maven_library: MavenLibrary):
         self.maven_libraries.append(maven_library)

diff --git a/src/sdk/python/rtdip_sdk/pipelines/destinations/spark/delta_merge.py b/src/sdk/python/rtdip_sdk/pipelines/destinations/spark/delta_merge.py
@@ -14,7 +14,7 @@
 
 import logging
 import time
-from typing import Optional, Union
+from typing import List, Optional, Union
 from pydantic import BaseModel
 from pyspark.sql.functions import broadcast
 from pyspark.sql import DataFrame, SparkSession
@@ -24,6 +24,7 @@
 from ..interfaces import DestinationInterface
 from ..._pipeline_utils.models import Libraries, SystemType
 from ..._pipeline_utils.constants import DEFAULT_PACKAGES
+from ...._sdk_utils.compare_versions import _package_version_meets_minimum
 
 class DeltaMergeConditionValues(BaseModel):
     condition: Optional[str]
@@ -57,11 +58,11 @@ class SparkDeltaMergeDestination(DestinationInterface):
     table_name: str
     options: dict
     merge_condition: str
-    when_matched_update_list: list[DeltaMergeConditionValues]
-    when_matched_delete_list = list[DeltaMergeCondition]
-    when_not_matched_insert_list: list[DeltaMergeConditionValues]
-    when_not_matched_by_source_update_list: list[DeltaMergeConditionValues]
-    when_not_matched_by_source_delete_list: list[DeltaMergeCondition]
+    when_matched_update_list: List[DeltaMergeConditionValues]
+    when_matched_delete_list: List[DeltaMergeCondition]
+    when_not_matched_insert_list: List[DeltaMergeConditionValues]
+    when_not_matched_by_source_update_list: List[DeltaMergeConditionValues]
+    when_not_matched_by_source_delete_list: List[DeltaMergeCondition]
     try_broadcast_join: bool
     trigger: str
     query_name: str
@@ -72,11 +73,11 @@ def __init__(self,
                  table_name:str, 
                  options: dict,
                  merge_condition: str, 
-                 when_matched_update_list: list[DeltaMergeConditionValues] = None, 
-                 when_matched_delete_list: list[DeltaMergeCondition] = None,
-                 when_not_matched_insert_list: list[DeltaMergeConditionValues] = None,
-                 when_not_matched_by_source_update_list: list[DeltaMergeConditionValues] = None,
-                 when_not_matched_by_source_delete_list: list[DeltaMergeCondition] = None,
+                 when_matched_update_list: List[DeltaMergeConditionValues] = None, 
+                 when_matched_delete_list: List[DeltaMergeCondition] = None,
+                 when_not_matched_insert_list: List[DeltaMergeConditionValues] = None,
+                 when_not_matched_by_source_update_list: List[DeltaMergeConditionValues] = None,
+                 when_not_matched_by_source_delete_list: List[DeltaMergeCondition] = None,
                  try_broadcast_join: bool = True,
                  trigger="10 seconds",
                  query_name: str ="DeltaMergeDestination") -> None:
@@ -88,7 +89,11 @@ def __init__(self,
         self.when_matched_update_list = [] if when_matched_update_list == None else when_matched_update_list
         self.when_matched_delete_list = [] if when_matched_delete_list == None else when_matched_delete_list
         self.when_not_matched_insert_list = [] if when_not_matched_insert_list == None else when_not_matched_insert_list
+        if isinstance(when_not_matched_by_source_update_list, list) and len(when_not_matched_by_source_update_list) > 0:
+            _package_version_meets_minimum("delta-spark", "2.3.0")
         self.when_not_matched_by_source_update_list = [] if when_not_matched_by_source_update_list == None else when_not_matched_by_source_update_list
+        if isinstance(when_not_matched_by_source_delete_list, list) and len(when_not_matched_by_source_delete_list) > 0:
+            _package_version_meets_minimum("delta-spark", "2.3.0")        
         self.when_not_matched_by_source_delete_list = [] if when_not_matched_by_source_delete_list == None else when_not_matched_by_source_delete_list
         self.try_broadcast_join = try_broadcast_join
         self.trigger = trigger

diff --git a/src/sdk/python/rtdip_sdk/pipelines/execute/job.py b/src/sdk/python/rtdip_sdk/pipelines/execute/job.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List
 from dependency_injector import containers, providers
 from .container import Clients, Configs
 from .models import PipelineJob, PipelineTask, PipelineStep
@@ -52,7 +53,7 @@ def _get_secret_provider_attributes(self, pipeline_secret: PipelineSecret) -> pr
         secret_provider.add_kwargs(vault=pipeline_secret.vault, key=pipeline_secret.key)
         return secret_provider
 
-    def _tasks_order(self, task_list: list[PipelineTask]):
+    def _tasks_order(self, task_list: List[PipelineTask]):
         '''
         Orders tasks within a job
         '''
@@ -71,7 +72,7 @@ def _tasks_order(self, task_list: list[PipelineTask]):
                             break
         return ordered_task_list
 
-    def _steps_order(self, step_list: list[PipelineStep]):
+    def _steps_order(self, step_list: List[PipelineStep]):
         '''
         Orders steps within a task
         '''        
@@ -90,7 +91,7 @@ def _steps_order(self, step_list: list[PipelineStep]):
                         break
         return ordered_step_list
 
-    def _task_setup_dependency_injection(self, step_list: list[PipelineStep]):
+    def _task_setup_dependency_injection(self, step_list: List[PipelineStep]):
         '''
         Determines the dependencies to be injected into each component 
         '''

diff --git a/src/sdk/python/rtdip_sdk/pipelines/execute/models.py b/src/sdk/python/rtdip_sdk/pipelines/execute/models.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional, Type, Union, Dict
+from typing import List, Optional, Type, Union, Dict
 import re
 from pydantic import BaseConfig, BaseModel, validator
 from abc import ABCMeta
@@ -37,10 +37,10 @@ def validate_name(name: str) -> str:
 class PipelineStep(BaseModel):
     name: str
     description: str
-    depends_on_step: Optional[list[str]]
+    depends_on_step: Optional[List[str]]
     component: Union[Type[SourceInterface], Type[TransformerInterface], Type[DestinationInterface], Type[UtilitiesInterface]]
     component_parameters: Optional[dict]
-    provide_output_to_step: Optional[list[str]]
+    provide_output_to_step: Optional[List[str]]
 
     class Config:
         json_encoders = {
@@ -56,8 +56,8 @@ class Config:
 class PipelineTask(BaseModel):
     name: str
     description: str
-    depends_on_task: Optional[list[str]]
-    step_list: list[PipelineStep]
+    depends_on_task: Optional[List[str]]
+    step_list: List[PipelineStep]
     batch_task: Optional[bool]
 
     class Config:
@@ -74,7 +74,7 @@ class PipelineJob(BaseModel):
     name: str
     description: str
     version: str
-    task_list: list[PipelineTask]
+    task_list: List[PipelineTask]
 
     class Config:
         json_encoders = {

diff --git a/src/sdk/python/rtdip_sdk/pipelines/utilities/aws/s3_bucket_policy.py b/src/sdk/python/rtdip_sdk/pipelines/utilities/aws/s3_bucket_policy.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import logging
-from typing import Dict
+from typing import Dict, List
 
 from pydantic import BaseModel
 
@@ -45,10 +45,10 @@ class S3BucketPolicyUtility(UtilitiesInterface):
     sid: str
     effect: str
     principal: str
-    action: list[str]
-    resource: list[str]
+    action: List[str]
+    resource: List[str]
 
-    def __init__(self, bucket_name: str, aws_access_key_id: str, aws_secret_access_key: str, aws_session_token: str, sid: str, principal: str, effect: str, action: list[str], resource: list[str]) -> None:
+    def __init__(self, bucket_name: str, aws_access_key_id: str, aws_secret_access_key: str, aws_session_token: str, sid: str, principal: str, effect: str, action: List[str], resource: List[str]) -> None:
         self.bucket_name = bucket_name
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key

diff --git a/src/sdk/python/rtdip_sdk/pipelines/utilities/spark/configuration.py b/src/sdk/python/rtdip_sdk/pipelines/utilities/spark/configuration.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+from typing import List
 from pyspark.sql import SparkSession
 from pyspark.sql.types import StructField
 from py4j.protocol import Py4JJavaError
@@ -30,8 +31,8 @@ class SparkConfigurationUtility(UtilitiesInterface):
     ''' 
     spark: SparkSession
     config: dict
-    columns: list[StructField]
-    partitioned_by: list[str]
+    columns: List[StructField]
+    partitioned_by: List[str]
     location: str
     properties: dict
     comment: str

diff --git a/src/sdk/python/rtdip_sdk/pipelines/utilities/spark/delta_table_create.py b/src/sdk/python/rtdip_sdk/pipelines/utilities/spark/delta_table_create.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import logging
-from typing import Optional
+from typing import List, Optional
 from pydantic import BaseModel
 from pyspark.sql import SparkSession
 from pyspark.sql.types import StructField
@@ -67,13 +67,13 @@ class DeltaTableCreateUtility(UtilitiesInterface):
     ''' 
     spark: SparkSession
     table_name: str
-    columns: list[DeltaTableColumn]
-    partitioned_by: list[str]
+    columns: List[DeltaTableColumn]
+    partitioned_by: List[str]
     location: str
     properties: dict
     comment: str
 
-    def __init__(self, spark: SparkSession, table_name: str, columns: list[StructField], partitioned_by: list[str] = None, location: str = None, properties: dict = None, comment: str = None) -> None:
+    def __init__(self, spark: SparkSession, table_name: str, columns: List[StructField], partitioned_by: List[str] = None, location: str = None, properties: dict = None, comment: str = None) -> None:
         self.spark = spark
         self.table_name = table_name
         self.columns = columns