rtdip · GBBBAS · Mar 29, 2023 · Mar 28, 2023 · Mar 28, 2023 · Mar 29, 2023
diff --git a/.gitignore b/.gitignore
@@ -136,4 +136,7 @@ spark-warehouse/
 spark-checkpoints/
 
 # Delta Sharing
-config.share
+config.share
+
+# DBX package
+src/sdk/python/rtdip_sdk/pipelines/deploy/dbx/.dbx/project.json
diff --git a/docs/sdk/code-reference/pipelines/converters/pipeline_job_json.md b/docs/sdk/code-reference/pipelines/converters/pipeline_job_json.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.converters.pipeline_job_json
diff --git a/docs/sdk/code-reference/pipelines/deploy/databricks_dbx.md b/docs/sdk/code-reference/pipelines/deploy/databricks_dbx.md
@@ -0,0 +1 @@
+::: src.sdk.python.rtdip_sdk.pipelines.deploy.databricks
diff --git a/environment.yml b/environment.yml
@@ -52,7 +52,7 @@ dependencies:
           - dependency-injector==4.41.0
           - azure-functions==1.12.0
           - databricks-sql-connector==2.4.0
-          - dbx==0.8.9
+          - dbx==0.8.10
           - pygithub==1.58.0
           - strawberry-graphql[fastapi]==0.159.0
           - nest_asyncio==1.5.6
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -72,7 +72,9 @@ extra:
 plugins:
     - search
     - autorefs
-    - mkdocstrings
+    - mkdocstrings:
+        watch:
+            - src/sdk/python/rtdip_sdk    
     - macros:
         module_name: docs/macros
 
@@ -139,8 +141,12 @@ nav:
                       - Utilities:
                             - Spark:
                                  - Delta Table Create: sdk/code-reference/pipelines/utilities/spark/delta_table_create.md
+                      - Converters:
+                            - Json: sdk/code-reference/pipelines/converters/pipeline_job_json.md    
                       - Secrets:
                             - Databricks: sdk/code-reference/pipelines/secrets/databricks.md
+                      - Deploy:
+                            - Databricks DBX: sdk/code-reference/pipelines/deploy/databricks_dbx.md                          
     - API:
           - Overview: api/overview.md
           - Authentication: api/authentication.md

diff --git a/setup.py b/setup.py
@@ -45,7 +45,7 @@
 
 PIPELINE_PACKAGES = [
   "dependency-injector==4.41.0",
-  "dbx==0.8.9",
+  "dbx==0.8.10",
   "pydantic==1.10.6"
 ]
 

diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py
@@ -1,5 +1,6 @@
-
-
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #

diff --git a/src/sdk/python/rtdip_sdk/pipelines/converters/__init__.py b/src/sdk/python/rtdip_sdk/pipelines/converters/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .pipeline_job_json import *
diff --git a/src/sdk/python/rtdip_sdk/pipelines/converters/interfaces.py b/src/sdk/python/rtdip_sdk/pipelines/converters/interfaces.py
@@ -0,0 +1,21 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+class ConverterInterface(ABC):
+
+    @abstractmethod
+    def convert(self):
+        pass  
diff --git a/src/sdk/python/rtdip_sdk/pipelines/converters/pipeline_job_json.py b/src/sdk/python/rtdip_sdk/pipelines/converters/pipeline_job_json.py
@@ -0,0 +1,89 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import json 
+
+from .interfaces import ConverterInterface
+from ..secrets.models import PipelineSecret
+from ..execute.models import PipelineJob
+from ..sources import * # NOSONAR
+from ..transformers import * # NOSONAR
+from ..destinations import * # NOSONAR
+from ..utilities import * # NOSONAR
+from ..secrets import * # NOSONAR
+
+class PipelineJobFromJson(ConverterInterface):
+    '''
+    Converts a json string into a Pipeline Job
+
+    Args:
+        pipeline_json (str): Json representing PipelineJob information, including tasks and related steps 
+    '''
+    pipeline_json: str
+
+    def __init__(self, pipeline_json: str):
+        self.pipeline_json = pipeline_json
+
+    def _try_convert_to_pipeline_secret(self, value):
+        try:
+            if "pipeline_secret" in value:
+                value["pipeline_secret"]["type"] = getattr(sys.modules[__name__], value["pipeline_secret"]["type"])
+            return PipelineSecret.parse_obj(value["pipeline_secret"])
+        except: # NOSONAR
+            return value
+
+    def convert(self) -> PipelineJob:
+        '''
+        Converts a json string to a Pipeline Job
+        '''
+        pipeline_job_dict = json.loads(self.pipeline_json)
+
+        # convert string component to class
+        for task in pipeline_job_dict["task_list"]:
+            for step in task["step_list"]:
+                step["component"] = getattr(sys.modules[__name__], step["component"])
+                for param_key, param_value in step["component_parameters"].items():
+                    step["component_parameters"][param_key] = self._try_convert_to_pipeline_secret(param_value)
+                    if not isinstance(step["component_parameters"][param_key], PipelineSecret) and isinstance(param_value, dict):
+                        for key, value in param_value.items():
+                            step["component_parameters"][param_key][key] = self._try_convert_to_pipeline_secret(value) 
+
+        return PipelineJob(**pipeline_job_dict)
+
+class PipelineJobToJson(ConverterInterface):
+    '''
+    Converts a Pipeline Job into a json string
+
+    Args:
+        pipeline_job (PipelineJob): A Pipeline Job consisting of tasks and steps
+    '''
+    pipeline_job: PipelineJob
+
+    def __init__(self, pipeline_job: PipelineJob):
+        self.pipeline_job = pipeline_job
+
+    def convert(self):
+        '''
+        Converts a Pipeline Job to a json string
+        '''
+        # required because pydantic does not use encoders in subclasses
+        for task in self.pipeline_job.task_list:
+            step_dict_list = []
+            for step in task.step_list:
+                step_dict_list.append(json.loads(step.json(models_as_dict=False, exclude_none=True)))
+            task.step_list = step_dict_list
+
+        pipeline_job_json = self.pipeline_job.json(exclude_none=True)
+        return pipeline_job_json
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		::: src.sdk.python.rtdip_sdk.pipelines.converters.pipeline_job_json
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		::: src.sdk.python.rtdip_sdk.pipelines.deploy.databricks