rtdip · GBBBAS · Mar 21, 2023 · Mar 21, 2023 · Mar 21, 2023
diff --git a/setup.py b/setup.py
@@ -76,7 +76,7 @@
     packages=find_packages(where="src/sdk/python"),
     python_requires=">=3.8, <=3.10",
     install_requires=INSTALL_REQUIRES,
-    extra_requires=EXTRAS_DEPENDENCIES,
+    extras_require=EXTRAS_DEPENDENCIES,
     setup_requires=["pytest-runner","setuptools_scm"],
     tests_require=["pytest"],
     test_suite="tests",

diff --git a/src/sdk/python/rtdip_sdk/pipelines/deploy/dbx/rtdip/tasks/pipeline_task.py b/src/sdk/python/rtdip_sdk/pipelines/deploy/dbx/rtdip/tasks/pipeline_task.py
@@ -13,22 +13,21 @@
 # limitations under the License.
 
 import sys
-import json
 from rtdip.tasks.common import Task
-from rtdip_sdk.pipelines.execute.models import PipelineJob
+from rtdip_sdk.pipelines.execute import *
+from rtdip_sdk.pipelines.sources import *
+from rtdip_sdk.pipelines.transformers import *
+from rtdip_sdk.pipelines.destinations import *
+from rtdip_sdk.pipelines.utilities import *
 
 class RTDIPPipelineTask(Task):
     def launch(self):
         self.logger.info("Launching RTDIP Pipeline Task")
         self.logger.info("Job to execute {}".format(sys.argv[0]))
-        pipeline_job = json.loads(sys.argv[0], object_hook= lambda d: PipelineJob(**d))
-        # pipeline_job = json.loads(sys.argv[0])
-        # for task in pipeline_job.task_list:
-        #     for step in task.step_list:
-        #         step.component = type(step.component)
-        # pipeline = PipelineJobExecute(pipeline_job)
-        # pipeline.run()
-        self.logger.info("Sample ETL task finished!")
+        pipeline_job = PipelineJobFromJson(sys.argv[0])
+        pipeline_job_execute = PipelineJobExecute(pipeline_job)
+        pipeline_job_execute.run()
+        self.logger.info("RTDIP Pipeline Task finished!")
 
 # if you're using python_wheel_task, you'll need the entrypoint function to be used in setup.py
 def entrypoint():  # pragma: no cover

diff --git a/src/sdk/python/rtdip_sdk/pipelines/execute/__init__.py b/src/sdk/python/rtdip_sdk/pipelines/execute/__init__.py
@@ -12,4 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .models import *
+from .models import *
+from .job import *
diff --git a/src/sdk/python/rtdip_sdk/pipelines/execute/job.py b/src/sdk/python/rtdip_sdk/pipelines/execute/job.py
@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import sys
+import json
 from dependency_injector import containers, providers
 
 from .container import Clients, Configs
@@ -164,4 +165,26 @@ def run(self) -> bool:
                         task_results[step] = result
 
         return True
-
+
+class PipelineJobFromJson():
+    '''
+    Converts a json string into a Pipeline Jobs
+
+    Args:
+        pipeline_json: Json representing PipelineJob information, including tasks and related steps 
+    '''
+    pipeline_json: str
+
+    def init(self, pipeline_json: str):
+        self.pipeline_json = pipeline_json
+
+    def convert(self) -> PipelineJob:
+        pipeline_job_dict = json.loads(self.pipeline_json)
+
+        # convert string component to class
+        for task in pipeline_job_dict["task_list"]:
+            for step in task["step_list"]:
+                step["component"] = getattr(sys.modules[__name__], step["component"])
+
+        return PipelineJob(**pipeline_job_dict)
+