diff --git a/docs/blog/rtdip_ingestion_pipelines.md b/docs/blog/rtdip_ingestion_pipelines.md index 61056e3fe..209ca650d 100644 --- a/docs/blog/rtdip_ingestion_pipelines.md +++ b/docs/blog/rtdip_ingestion_pipelines.md @@ -193,4 +193,4 @@ Edge components are designed to provide a lightweight, low latency, low resource ## Conclusion -This is a very high level overview of the framework and the components that will be developed. As the framework is open source, the lists defined above and timelines can change depending on circumstances and resource availability. Its an exciting year for 2023 for the Real Time Data Ingestion Platform. Check back in regularly for updates and new features! If you would like to contribute, please visit our repository onn Github and connect with us on our Slack channel on the LF Energy Foundation Slack workspace. \ No newline at end of file +This is a very high level overview of the framework and the components that will be developed. As the framework is open source, the lists defined above and timelines can change depending on circumstances and resource availability. Its an exciting year for 2023 for the Real Time Data Ingestion Platform. Check back in regularly for updates and new features! If you would like to contribute, please visit our repository on Github and connect with us on our Slack channel on the LF Energy Foundation Slack workspace. \ No newline at end of file diff --git a/environment.yml b/environment.yml index b2b16a8f3..f3da436d7 100644 --- a/environment.yml +++ b/environment.yml @@ -39,19 +39,20 @@ dependencies: - httpx==0.23.1 - trio==0.22.0 - pyspark==3.3.2 - - delta-spark>=2.2.0 + - delta-spark==2.2.0 - openjdk==11.0.15 - - python-dotenv>=0.21.1 + - python-dotenv==1.0.0 - mkdocstrings==0.20.0 - mkdocstrings-python==0.8.3 - mkdocs-macros-plugin==0.7.0 - - pygments>=2.14 - - pymdown-extensions>=9.9.2 + - pygments==2.14 + - pymdown-extensions==9.10 + - airflow[databricks]==2.5.1 - pip: - - dependency-injector>=4.41.0 + - dependency-injector==4.41.0 - azure-functions==1.12.0 - - dbx==0.8.9 - databricks-sql-connector==2.4.0 + - dbx==0.8.9 - pygithub==1.58.0 - strawberry-graphql[fastapi]==0.159.0 - nest_asyncio==1.5.6 \ No newline at end of file diff --git a/setup.py b/setup.py index 6eb98e665..ce370b793 100644 --- a/setup.py +++ b/setup.py @@ -28,10 +28,33 @@ long_description = (here / "PYPI-README.md").read_text() +INSTALL_REQUIRES = [ + "databricks-sql-connector==2.4.0", + "azure-identity==1.11.0", + "pyodbc==4.0.34", + "pandas==1.5.2", + "jinja2==3.0.3", + "jinjasql==0.1.8" +] + +PIPELINE_PACKAGES = [ + "pyspark==3.3.2", + "delta-spark==2.2.0", + "openjdk==11.0.15", + "python-dotenv==0.21.1", + "dependency-injector==4.41.0", + "dbx==0.8.9", + "airflow==2.5.1" +] + +EXTRAS_DEPENDENCIES: dict[str, list[str]] = { + "pipelines": PIPELINE_PACKAGES +} + setup( - name='rtdip-sdk', + name="rtdip-sdk", long_description=long_description, - long_description_content_type='text/markdown', + long_description_content_type="text/markdown", url="https://github.com/rtdip/core", classifiers=[ "License :: OSI Approved :: Apache Software License", @@ -46,11 +69,12 @@ "Documentation": "https://www.rtdip.io/" }, version=sic(os.environ["RTDIP_SDK_NEXT_VER"]), - package_dir={'': 'src/sdk/python'}, - packages=find_packages(where='src/sdk/python'), - python_requires='>=3.8, <4', - install_requires=['databricks-sql-connector','azure-identity','azure-storage-file-datalake','pyodbc','pandas','jinja2==3.0.3','jinjasql==0.1.8'], - setup_requires=['pytest-runner','setuptools_scm'], - tests_require=['pytest'], - test_suite='tests', + package_dir={"": "src/sdk/python"}, + packages=find_packages(where="src/sdk/python"), + python_requires=">=3.8, <=3.10", + install_requires=INSTALL_REQUIRES, + extra_requires=EXTRAS_DEPENDENCIES, + setup_requires=["pytest-runner","setuptools_scm"], + tests_require=["pytest"], + test_suite="tests", ) \ No newline at end of file