rtdip · cching95 · Mar 8, 2023 · Mar 8, 2023
diff --git a/docs/blog/rtdip_ingestion_pipelines.md b/docs/blog/rtdip_ingestion_pipelines.md
@@ -193,4 +193,4 @@ Edge components are designed to provide a lightweight, low latency, low resource
 
 ## Conclusion
 
-This is a very high level overview of the framework and the components that will be developed. As the framework is open source, the lists defined above and timelines can change depending on circumstances and resource availability. Its an exciting year for 2023 for the Real Time Data Ingestion Platform. Check back in regularly for updates and new features! If you would like to contribute, please visit our repository onn Github and connect with us on our Slack channel on the LF Energy Foundation Slack workspace.
+This is a very high level overview of the framework and the components that will be developed. As the framework is open source, the lists defined above and timelines can change depending on circumstances and resource availability. Its an exciting year for 2023 for the Real Time Data Ingestion Platform. Check back in regularly for updates and new features! If you would like to contribute, please visit our repository on Github and connect with us on our Slack channel on the LF Energy Foundation Slack workspace.
diff --git a/environment.yml b/environment.yml
@@ -39,19 +39,20 @@ dependencies:
     - httpx==0.23.1
     - trio==0.22.0
     - pyspark==3.3.2
-    - delta-spark>=2.2.0
+    - delta-spark==2.2.0
     - openjdk==11.0.15    
-    - python-dotenv>=0.21.1
+    - python-dotenv==1.0.0
     - mkdocstrings==0.20.0
     - mkdocstrings-python==0.8.3
     - mkdocs-macros-plugin==0.7.0
-    - pygments>=2.14
-    - pymdown-extensions>=9.9.2
+    - pygments==2.14
+    - pymdown-extensions==9.10
+    - airflow[databricks]==2.5.1
     - pip:
-          - dependency-injector>=4.41.0
+          - dependency-injector==4.41.0
           - azure-functions==1.12.0
-          - dbx==0.8.9
           - databricks-sql-connector==2.4.0
+          - dbx==0.8.9
           - pygithub==1.58.0
           - strawberry-graphql[fastapi]==0.159.0
           - nest_asyncio==1.5.6
diff --git a/setup.py b/setup.py
@@ -28,10 +28,33 @@
 
 long_description = (here / "PYPI-README.md").read_text()
 
+INSTALL_REQUIRES = [
+  "databricks-sql-connector==2.4.0",
+  "azure-identity==1.11.0",
+  "pyodbc==4.0.34",
+  "pandas==1.5.2",
+  "jinja2==3.0.3",
+  "jinjasql==0.1.8"
+]
+
+PIPELINE_PACKAGES = [
+  "pyspark==3.3.2",
+  "delta-spark==2.2.0",
+  "openjdk==11.0.15",   
+  "python-dotenv==0.21.1",
+  "dependency-injector==4.41.0",
+  "dbx==0.8.9",
+  "airflow==2.5.1"
+]
+
+EXTRAS_DEPENDENCIES: dict[str, list[str]] = {
+  "pipelines": PIPELINE_PACKAGES
+}
+
 setup(
-    name='rtdip-sdk',
+    name="rtdip-sdk",
     long_description=long_description,
-    long_description_content_type='text/markdown',
+    long_description_content_type="text/markdown",
     url="https://github.com/rtdip/core", 
     classifiers=[
       "License :: OSI Approved :: Apache Software License",
@@ -46,11 +69,12 @@
         "Documentation": "https://www.rtdip.io/"
     },    
     version=sic(os.environ["RTDIP_SDK_NEXT_VER"]),
-    package_dir={'': 'src/sdk/python'},
-    packages=find_packages(where='src/sdk/python'),
-    python_requires='>=3.8, <4',
-    install_requires=['databricks-sql-connector','azure-identity','azure-storage-file-datalake','pyodbc','pandas','jinja2==3.0.3','jinjasql==0.1.8'],
-    setup_requires=['pytest-runner','setuptools_scm'],
-    tests_require=['pytest'],
-    test_suite='tests',
+    package_dir={"": "src/sdk/python"},
+    packages=find_packages(where="src/sdk/python"),
+    python_requires=">=3.8, <=3.10",
+    install_requires=INSTALL_REQUIRES,
+    extra_requires=EXTRAS_DEPENDENCIES,
+    setup_requires=["pytest-runner","setuptools_scm"],
+    tests_require=["pytest"],
+    test_suite="tests",
 )
Original file line number	Diff line number	Diff line change
Expand Up		@@ -193,4 +193,4 @@ Edge components are designed to provide a lightweight, low latency, low resource

		## Conclusion

		This is a very high level overview of the framework and the components that will be developed. As the framework is open source, the lists defined above and timelines can change depending on circumstances and resource availability. Its an exciting year for 2023 for the Real Time Data Ingestion Platform. Check back in regularly for updates and new features! If you would like to contribute, please visit our repository onn Github and connect with us on our Slack channel on the LF Energy Foundation Slack workspace.
		This is a very high level overview of the framework and the components that will be developed. As the framework is open source, the lists defined above and timelines can change depending on circumstances and resource availability. Its an exciting year for 2023 for the Real Time Data Ingestion Platform. Check back in regularly for updates and new features! If you would like to contribute, please visit our repository on Github and connect with us on our Slack channel on the LF Energy Foundation Slack workspace.