diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..89d2226ee --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,37 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' +--- + +**Describe the bug** + +A clear and concise description of what the bug is. + +**To Reproduce** + +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** + +A clear and concise description of what you expected to happen. + +**Screenshots** + +If applicable, add screenshots to help explain your problem. + +**Installation Setup (please complete the following information):** + + - OS: [e.g. iOS] + - Python Version: [e.g. 3.8, 3.6] + - SDK Version: [e.g. 1.0] + +**Additional context** + +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..f0291e059 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,24 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** + +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** + +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** + +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** + +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/general_inquiry.md b/.github/ISSUE_TEMPLATE/general_inquiry.md new file mode 100644 index 000000000..8f0a250bd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general_inquiry.md @@ -0,0 +1,20 @@ +--- +name: General Inquiry +about: General inquiry about usage +title: '' +labels: '' +assignees: '' + +--- + +**Question Summary** + +A clear and concise summary of question or goal that you have in mind. + +**Detailed background** + +Additional details required to frame the context of your question. + +**Screenshots** + +If applicable, add screenshots to help explain your question. diff --git a/.github/ISSUE_TEMPLATE/security_vulnerability.md b/.github/ISSUE_TEMPLATE/security_vulnerability.md new file mode 100644 index 000000000..794c2cc12 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/security_vulnerability.md @@ -0,0 +1,39 @@ +--- +name: Security Vulnerabilities/Risks +about: Report on security vulnerabilities or risks +title: '' +labels: '' +assignees: '' +--- + +**Describe the Security Vulnerability** + +A clear and concise description of what the security vulnerability/risk is. + +**To Reproduce** + +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected Security Policy/Actions** + +A clear and concise description of what you expected to happen and any possible recommendation. + +**Screenshots** + +If applicable, add screenshots to help explain your problem. + +**Installation Setup (please complete the following information):** + + - OS: [e.g. iOS] + - Python Version: [e.g. 3.8, 3.6] + - SDK Version: [e.g. 22] + +**Additional context** + +Add any other context about the problem here. + + diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 000000000..3a5ca35e7 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,41 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +changelog: + exclude: + labels: + - dependencies + - duplicate + - help wanted + - invalid + - wontfix + - question + authors: + - dependabot[bot] + categories: + - title: Major Release + labels: + - breaking-change + - title: New Features + labels: + - enhancement + - title: Bug Fixes + labels: + - bug + - title: Documentation + labels: + - documentation + - title: Other Changes + labels: + - "*" \ No newline at end of file diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml new file mode 100644 index 000000000..f1b45a598 --- /dev/null +++ b/.github/workflows/develop.yml @@ -0,0 +1,130 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Develop' + +on: + # Trigger the workflow on push to develop + push: + branches: + - develop + +jobs: + job_run_unit_tests_and_sonarqube: + uses: rtdip/core/.github/workflows/test.yml@develop + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + job_build_python_package_and_docker_container: + runs-on: ubuntu-latest + needs: job_run_unit_tests_and_sonarqube + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install twine + pip install build + pip install requests + pip install semver==3.0.0.dev3 + - name: Determine Next Test PyPI Version + id: next_ver + run: | + import requests + import semver + import os + from packaging.version import Version as PyPIVersion + + def get_semver_version(pypi_url: str, package: str, include_prereleases=False) -> semver.Version: + response = requests.get(f'{pypi_url}/pypi/{package}/json') + if response.status_code != 200: + pypi_latest_version = "0.0.0" + else: + if include_prereleases == True: + pypi_latest_version = list(response.json()["releases"].keys())[-1] + else: + pypi_latest_version = response.json()['info']['version'] + + pypi_ver = PyPIVersion(pypi_latest_version) + + pre=None + if pypi_ver.is_prerelease: + pre = "".join(str(i) for i in pypi_ver.pre) + pypi_ver = semver.Version(*pypi_ver.release, pre) + return pypi_ver + + package = 'rtdip-sdk' + + pypi_ver = get_semver_version("https://pypi.org", package) + print("Current PyPi version: " + str(pypi_ver)) + + next_ver = pypi_ver.bump_patch() + + test_pypi_ver = get_semver_version("https://test.pypi.org", package) + print("Current TestPyPi version: " + str(test_pypi_ver)) + + if next_ver == "0.0.1": + next_ver = test_pypi_ver + elif test_pypi_ver.major == next_ver.major and test_pypi_ver.minor == next_ver.minor and test_pypi_ver.patch == next_ver.patch and test_pypi_ver.prerelease != None: + next_ver = next_ver.replace(prerelease=test_pypi_ver.prerelease) + + next_ver = next_ver.bump_prerelease() + print("Next version: " + str(next_ver)) + print(f'::set-output name=rtdip_sdk_next_ver::{str(next_ver)}') + shell: python + - name: Build Wheel + run: | + python -m build --wheel + env: + RTDIP_SDK_NEXT_VER: ${{ steps.next_ver.outputs.rtdip_sdk_next_ver }} + - name: Upload Python wheel as artifact + uses: actions/upload-artifact@v2 + with: + name: rtdip_sdk_whl + path: ./dist/*.whl + - name: Publish distribution πŸ“¦ to Test PyPI + run: | + twine upload --repository testpypi --username __token__ --password ${{ secrets.TEST_PYPI_API_TOKEN }} --verbose dist/* + + # Deploy to Docker Hub + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: | + rtdip/prerelease + tags: | + type=semver,pattern={{version}},prefix=api-azure-,value=${{ steps.next_ver.outputs.rtdip_sdk_next_ver }} + + - name: Build and push Docker images + uses: docker/build-push-action@v3 + with: + context: . + file: ./src/api/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..43e8bfff8 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Main' + +on: + # Trigger the workflow on push to main + push: + branches: + - main + +jobs: + job_run_unit_tests_and_sonarqube: + uses: rtdip/core/.github/workflows/test.yml@main + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 000000000..6e2c791c3 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,27 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'PR' + +on: + # Trigger the workflow on pull request + pull_request: + branches: [main, develop] + types: [opened, synchronize, reopened] + +jobs: + job_run_unit_tests_and_sonarqube: + uses: rtdip/core/.github/workflows/test.yml@develop + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..bef7993e1 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,127 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Release' + +on: + # Trigger the workflow on release published + release: + types: [published] + +jobs: + job_run_unit_tests_and_sonarqube: + uses: rtdip/core/.github/workflows/test.yml@main + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + job_build_python_whl: + runs-on: ubuntu-latest + needs: job_run_unit_tests_and_sonarqube + steps: + - uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install twine + pip install build + python -m build --wheel + env: + RTDIP_SDK_NEXT_VER: ${{ github.ref_name }} + - name: Upload Python wheel as artifact + uses: actions/upload-artifact@v2 + with: + name: rtdip_sdk_whl + path: ./dist/*.whl + - name: Publish distribution πŸ“¦ to PyPI + run: | + twine upload --username __token__ --password ${{ secrets.PYPI_API_TOKEN }} --verbose dist/* + + push_to_registries: + name: Push Docker image to multiple registries + needs: job_build_python_whl + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: | + rtdip/api + ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern={{version}},prefix=azure-,value=${{ github.refname }} + + - name: Build and push Docker images + uses: docker/build-push-action@v3 + with: + context: . + file: ./src/api/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + job_deploy_mkdocs_github_pages: + runs-on: ubuntu-latest + needs: job_build_python_whl + env: + PYTHONPATH: home/runner/work/core/ + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: 3.8 + + - name: Install Boost + run: | + sudo apt update + sudo apt install -y libboost-all-dev + + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + + - name: Install dependencies + run: | + conda env update --file environment.yml --name base + + - name: Deploy + run: | + mkdocs gh-deploy --force --remote-branch gh-pages-main + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..ce56ce907 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,69 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Reusable Test Workflow' + +on: + workflow_call: + secrets: + SONAR_TOKEN: + required: true + +jobs: + job_test_python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: 3.8 + + - name: Install Boost + run: | + sudo apt update + sudo apt install -y libboost-all-dev + + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Install dependencies + run: | + conda env update --file environment.yml --name base + + - name: Test + run: | + python setup.py pytest + mkdir -p coverage-reports + coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests && tests_ok=true + coverage xml --omit "venv/**,maintenance/**,xunit-reports/**" -i -o coverage-reports/coverage-unittests.xml + echo Coverage `coverage report --omit "venv/**" | grep TOTAL | tr -s ' ' | cut -d" " -f4` + env: + RTDIP_SDK_NEXT_VER: "0.0.1" + - name: Override Coverage Source Path for Sonar + run: | + sed -i "s/\/home\/runner\/work\/core\/core<\/source>/\/github\/workspace<\/source>/g" /home/runner/work/core/core/coverage-reports/coverage-unittests.xml + + - name: SonarCloud Scan + uses: SonarSource/sonarcloud-github-action@master + with: + args: > + -Dsonar.organization=rtdip + -Dsonar.projectKey=rtdip_core + -Dsonar.python.coverage.reportPaths=coverage-reports/coverage-unittests.xml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..a6f2ce05d --- /dev/null +++ b/.gitignore @@ -0,0 +1,131 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +*.DS_Store diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..61401874b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,17 @@ +{ + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.analysis.extraPaths": ["${workspaceFolder}"], + "python.envFile": "${workspaceFolder}/.env", + "terminal.integrated.env.osx":{ + "PYTHONPATH": "${workspaceFolder}:${env:PYTHONPATH}" + }, + "terminal.integrated.env.windows":{ + "PYTHONPATH": "${workspaceFolder};${env:PYTHONPATH}" + }, + "git.alwaysSignOff": true, + +} \ No newline at end of file diff --git a/CODEOWNERS.md b/CODEOWNERS.md new file mode 100644 index 000000000..d22b4a4e5 --- /dev/null +++ b/CODEOWNERS.md @@ -0,0 +1,9 @@ + +| Name | GitHub ID | +| -------------- | ----------------:| +| Bryce Bartmann | GBBBAS | +| Chloe Ching | cching95 | +| Ewan Watson | ewatson95 | +| Osman Ahmed | osmanahmed-shell | +| Amber Rigg | Amber-Rigg | +| Noora Kubati | NooraKubati | diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..3b9a27012 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,202 @@ +# Code of Conduct + +Effective: September 14, 2017 + +LF Projects, LLC (β€œLF Projects”) supports and hosts open source and open +standards projects (each a β€œProject”) and undertakes such other activities as is +consistent with its mission and purpose. + +## Introduction + +The purposes of LF Projects, LLC (β€œLF Projects”) are to: + +* support the collaborative development, availability and adoption of open + source software, hardware and networking and other technologies and the +collaborative development, availability and adoption of open protocols and +standards (individually and collectively, β€œOpen Technology”); +* host various projects pursuing the development of Open Technology and other + technical assets, materials and processes (each such project, which itself may +include any number of projects, a β€œProject”); +* provide enablement and support to Projects to assist their development + activities; and +* undertake such other lawful activity as permitted by law and as consistent + with the mission, purpose and tax status of LFP, Inc., a Delaware non-profit +non-stock corporation and the sole member of LF Projects. + +LF Projects hosts communities where participants choose to work together, and in +that process experience differences in language, location, nationality, and +experience. In such a diverse environment, misunderstandings and disagreements +happen, which in most cases can be resolved informally. In rare cases, however, +behavior can intimidate, harass, or otherwise disrupt one or more people in the +community, which LF Projects will not tolerate. + +A **Code of Conduct (β€œCode”)** is useful to define accepted and acceptable +behaviors and to promote high standards of professional practice. It also +provides a benchmark for self-evaluation and acts as a vehicle for better +identity of the organization. + +LF Projects is a Delaware series limited liability company. Projects of LF +Projects are formed as separate series of LF Projects (each, a β€œSeries”). +References to β€œProjects” within this Policy include the applicable Series for +each Project. + +This Code applies to any participant of any Project – including without +limitation developers, participants in meetings, teleconferences, mailing lists, +conferences or functions, and contributors. Note that this Code complements +rather than replaces legal rights and obligations pertaining to any particular +situation. In addition, with the approval of LF Projects, Projects are free to +adopt their own code of conduct in place of the Code. + +## Statement of Intent + +LF Projects is committed to maintain a positive, professional work environment. +This commitment calls for workplaces where participants at all levels behave +according to the rules of the following code. A foundational concept of this +code is that we all share responsibility for our work environment. + +## Code + +1. Treat each other with respect, professionalism, fairness, and sensitivity to +our many differences and strengths, including in situations of high pressure and +urgency. +2. Never harass or bully anyone verbally, physically, sexually, +or in digital or written form. +3. Never discriminate on the basis of personal characteristics or group +membership. +4. Communicate constructively and avoid demeaning or insulting behavior or +language. +5. Seek, accept, and offer objective work criticism, and acknowledge properly +the contributions of others. +6. Be honest about your own qualifications, and about any circumstances that +might lead to conflicts of interest. +7. Respect the privacy of others and the confidentiality of data you access. +8. With respect to cultural differences, be conservative in what you do and +liberal in what you accept from others, but not to the point of accepting +disrespectful, unprofessional or unfair or unwelcome behavior or advances. +9. Promote the rules of this Code and take action (especially if you are in a +leadership position) to bring the discussion back to a more civil level whenever +inappropriate behaviors are observed. +10. Stay on topic: Make sure that you are posting to the correct channel and +avoid off-topic discussions. Remember when you update an issue or respond to an +email you are potentially sending to a large number of people. +11. Step down considerately: participants in every project come and go, and LF +Projects is no different. When you leave or disengage from the project, in whole +or in part, we ask that you do so in a way that minimizes disruption to the +project. This means you should tell people you are leaving and take the proper +steps to ensure that others can pick up where you left off. + +## Glossary + +### Demeaning behavior + +is acting in a way that reduces another person’s dignity, sense of self-worth or +respect within the community. + +### Discrimination + +is the prejudicial treatment of an individual based on criteria such as: +physical appearance, race, ethnic origin, genetic differences, national or +social origin, name, religion, gender, sexual orientation, family or health +situation, pregnancy, disability, age, education, wealth, domicile, political +view, morals, employment, or union activity. + +### Insulting behavior + +is treating another person with scorn or disrespect. + +### Acknowledgement + +is a record of the origin(s) and author(s) of a contribution. + +### Harassment + +is any conduct, verbal, physical, digital, written, or otherwise, that has the +intent or effect of interfering with an individual, or that creates an +intimidating, hostile, or offensive environment. + +### Leadership position + +includes group Chairs, project maintainers, staff members, and Board members. + +### Participant + +includes the following persons: + +* Developers +* Representatives of corporate participants +* Anyone from the Public partaking in the LF Projects work environment (e.g. + contribute code, comment on our code or specs, email us, attend our +conferences, functions, etc) + +### Respect + +is the genuine consideration you have for someone (if only because of their +status as participant in LF Projects, like yourself), and that you show by +treating them in a polite and kind way. + +### Sexual harassment + +includes visual displays of degrading sexual images, sexually suggestive +conduct, offensive remarks of a sexual nature, requests for sexual favors, +unwelcome physical contact, and sexual assault. + +### Unwelcome behavior + +Hard to define? Some questions to ask yourself are: + +* how would I feel if I were in the position of the recipient? +* would my spouse, parent, child, sibling or friend like to be treated this way? +* would I like an account of my behavior published in the organization’s + newsletter? +* could my behavior offend or hurt other members of the work group? +* could someone misinterpret my behavior as intentionally harmful or harassing? +* would I treat my boss or a person I admire at work like that ? + +*Summary*: if you are unsure whether something might be welcome or unwelcome, don’t do it. + +### Unwelcome sexual advance + +includes requests for sexual favors, and other verbal, digital, written, or +physical conduct of a sexual nature, where: + +* submission to such conduct is made either explicitly or implicitly a term or + condition of an individual’s employment, +* submission to or rejection of such conduct by an individual is used as a basis + for employment decisions affecting the individual, +* such conduct has the purpose or effect of unreasonably interfering with an + individual’s work performance or creating an intimidating hostile or offensive +working environment. + +### Workplace Bullying + +is a tendency of individuals or groups to use persistent aggressive or +unreasonable behavior (e.g. verbal or written abuse, offensive conduct or any +interference which undermines or impedes work) against a co-worker or any +professional relations. + +### Work Environment + +is the set of all available means of collaboration, including, but not limited +to messages to mailing lists, private correspondence, Web pages, chat channels, +phone and video teleconferences, and any kind of face-to-face meetings or +discussions. + +## Incident Procedure + +To report incidents or to appeal reports of incidents, send email to the Manager of LF Projects, Mike Dolan (manager@lfprojects.org). + +Please include any available relevant information, including links to any +publicly accessible material relating to the matter. Every effort will be taken +to ensure a safe and collegial environment in which to collaborate on matters +relating to the Project. In order to protect the community, the Project reserves +the right to take appropriate action, potentially including the removal of an +individual from any and all participation in the project. The Project will work +towards an equitable resolution in the event of a misunderstanding. + +## Credits + +This code is based on the [W3C’s Code of Ethics and Professional +Conduct](https://www.w3.org/Consortium/cepc) with some additions from the [Cloud +Foundry](https://www.cloudfoundry.org/)’s Code of Conduct and the Hyperledger +Project Code of Conduct. It has been modified from the Linux Foundation Project +Code of Conduct to include incident managers local to the OpenBMC project. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..de262050e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,109 @@ +# How to contribute + +This Real Time Data Ingestion Platform (RTDIP) welcomes contributions and suggestions. If you have a suggestion that would improve RTDIP you can simply open an issue with the relevant title. A few contribution examples can be: + +* File a bug report +* Suggest a new feature +* General enquiries +* Security Vulnerabilities + +## Prerequisites: + +- Read RTDIP's [Code Of Conduct](https://github.com/rtdip/core/blob/develop/CODE_OF_CONDUCT.md) before making any contributions or suggestions. +- Install [Visual Studio Code](https://code.visualstudio.com/) and make sure you have the latest updates. +- Install [Python](https://www.python.org/downloads/). The Python version requirement is specfied in the setup.py file. +- Install [Conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) +- Clone the github repository + ```sh + git clone https://github.com/rtdip/core.git + ``` +## Issues Guidelines + +To contribute to Real Time Data Ingestion Platform, open an issue by following the steps below: + +1) Locate to the [Issues](https://github.com/rtdip/core/issues) tab on the github repository. +2) Select `New Issue` +3) Fill out the details and press `Submit New Issue`. + +Template examples for bug reporting and feature requests can be found [here](https://github.com/rtdip/core/tree/develop/.github/ISSUE_TEMPLATE). + +The title of your Issue should be a short description of your `bug` or `suggested new feature` followed by a comment for more details. + * *Examples of good Issue titles* + * Added deprecationDetails config to MetricsHistogram + * Fixed MetricsHistogram chart title tests + * Added MR job to export data to redshift + * *Examples of poor Issue titles* + * Fixed tests + * tests + * feature/00001 + +## Branch Naming Convention + +- Feature branches ```feature/``` e.g. ```feature/12345``` - ***Issue Number*** is the number of the issue item or work item in the GitHub issues page or project board. The issue number should be 5 digits, ***For example: if your issue number is 7 then your feature branch will be feature/00007***. +- Hotfix branches ```hotfix/``` e.g. ```hotfix/52679``` - ***Issue Number*** is the number of Bug/Issue Item in the GitHub project board. Hotfix branches are used for patch releases. +The standard process for working off the `develop` branch should go as follows: +1. `git fetch` in your current local branch to get latest remote branches list +2. `git checkout develop` to switch to `develop` branch +3. `git pull` to get the latest of `develop` branch +4. `git checkout -b branch_name` to create local branch for your task/issue + * This is consistent with the Git Flow approach and with Azure DevOps integration requirements +5. Go through the motions of development and committing and pushing as before. Once completed, create a pull request into `develop` branch +6. When pull request is completed, repeat from the top to start a new local branch/new dev work item + +## Testing + + * Always write tests for any parts of the code base you have touched. These tests should cover all possible edge cases (malformed input, guarding against nulls, infinite loops etc) + +## How To Start Contributing/Developing + +Before you start developing code, you will need to ensure you are following this respository's guidelines and standards. RTDIP SDK resides in the `src/sdk/python` folder. + +1) The `src/sdk` folder is the home of all our source code. If you would like to contribute to the code base this would be our entry point. From here you can navigate to the relevant folder or create new one where needed. + +> **_NOTE:_** Our folder naming standards are all lowercase with words split by "_". + +2) Write your code. + +3) Write unit tests for each function. All unit tests are located in the `test/sdk` folder. The folder structure for `test` should be identical to the `src` folder. All RTDIP SDK tests are written using `pytest`. + +4) All functions will need to be documented via docstrings. An example is shown below. + + ``` + ```including a description of your function + + Args: any arguments your function will need + + Attributes: if needed + + Returns: what the function returns``` + + ``` +5) Add function documentation to the `docs/sdk/code-reference` folder using the following example: + + ``` + # Title of your function + ::: src.sdk.{sdk_language_folder_location}.{sdk_folder_location}.{function_file}.{function_method} + ``` + +> **_NOTE:_** You will need to change the parameters respectively and remember to create docs for each function you create. RTDIP SDK uses mkdocstrings for code documentation. For more information on mkdocstrings, [see here](https://mkdocstrings.github.io/). + +6) If you would like your documentation to be visible on our [RTDIP Documentation](https://www.rtdip.io/) site then you will need to add references to [mkdocs.yml](mkdocs.yml) under the **nav:** section. + +7) Finally, you are ready to publish your changes. Create a PR on [Github - Pull Request](https://github.com/rtdip/core/pulls) and ensure to add reviewers from the [Codeowners List](CODEOWNERS.md) to review your code. RTDIP has built in workflows with sonarqube enabled to avoid pushing untested code. Reviewers will need to wait for all testing to pass before approving a PR and Squash Merging to develop. If everything passes your code should merge to develop successfully. + +> **_NOTE:_** Please ensure you read the [Release Guidelines](RELEASE.md) before publishing your contributions. + +8) **VERY IMPORTANT STEP!** To publish a new version of this python package you **MUST** create a tag and release in Github using the following versioning convention: + + Bump versioning standards: + * Patch - if you are patching code then you should only increment the last number 0.0. 1 . + + * Feature - if you are adding a new feature to develop you should increment the second number 0. 1 .0. + + * Production - if you are deploying a production ready version you should increment the first number 1 .0.0 + +**Always** ensure you have followed and checked all the steps to create a new release by following the [Release Guide](RELEASE.md). + +RTDIP has built in pipeline workflows to automatically deploy the python package to develop or production based on your Pull Request. + +Congratulations! Your clean and tested code should now be visible on Real Time Data Ingestion Platform. Thank you for your contribution. Any feedback on your experience will be greatly appreciated. \ No newline at end of file diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 000000000..216ee3c5d --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,68 @@ +# Overview + +This project aims to be governed in a transparent, accessible way for the benefit of the community. All participation in this project is open and not bound to corporate affilation. Participants are bound to the project's [Code of Conduct](https://github.com/rtdip/core/blob/develop/CODE_OF_CONDUCT.md). + +# Project roles + +## Contributor + +The contributor role is the starting role for anyone participating in the project and wishing to contribute code. + +# Process for becoming a contributor + +* Review the [Contribution Guidelines](https://github.com/rtdip/core/blob/develop/CONTRIBUTING.md) to ensure your contribution is inline with the project's coding and styling guidelines. +* Submit your code as a PR with the appropriate DCO signoff +* Have your submission approved by the committer(s) and merged into the codebase. + +## Codeowner + +The codeowners role enables the contributor to commit code directly to the repository, but also comes with the responsibility of being a responsible leader in the community. + +### Process for becoming a codeowner + +* Show your experience with the codebase through contributions and engagement on the community channels. +* Request to become a codeowner. To do this, create a new pull request that adds your name and details to the [Codeowners File](https://github.com/rtdip/core/blob/develop/CODEOWNERS.md) file and request existing codeowners to approve. +* After the majority of codeowners approve you, merge in the PR. Be sure to tag the whomever is managing the GitHub permissions to update the codeowners team in GitHub. + +### Codeowners responsibilities + +* Monitor email aliases (if any). +* Monitor Slack (delayed response is perfectly acceptable). +* Triage GitHub issues and perform pull request reviews for other codeowners and the community. +* Make sure that ongoing PRs are moving forward at the right pace or closing them. +* In general continue to be willing to spend at least 25% of ones time working on the project (~1.25 business days per week). + +### When does a codeowner lose codeowner status + +If a codeowner is no longer interested or cannot perform the codeowner duties listed above, they +should volunteer to be moved to emeritus status. In extreme cases this can also occur by a vote of +the codeowners per the voting process below. + +## Lead + +The project codeowners will elect a lead ( and optionally a co-lead ) which will be the primary point of contact for the project and representative to the TAC upon becoming an Active stage project. The lead(s) will be responsible for the overall project health and direction, coordination of activities, and working with other projects and codeowners as needed for the continuted growth of the project. + +# Release Process + +Project releases will occur on a scheduled basis as agreed to by the codeowners. + +# Conflict resolution and voting + +In general, we prefer that technical issues and codeowner membership are amicably worked out +between the persons involved. If a dispute cannot be decided independently, the codeowners can be +called in to decide an issue. If the codeowners themselves cannot decide an issue, the issue will +be resolved by voting. The voting process is a simple majority in which each codeowner receives one vote. + +# Communication + +This project, just like all of open source, is a global community. In addition to the [Code of Conduct](https://github.com/rtdip/core/blob/develop/CODE_OF_CONDUCT.md), this project will: + +* Keep all communucation on open channels ( mailing list, forums, chat ). +* Be respectful of time and language differences between community members ( such as scheduling meetings, email/issue responsiveness, etc ). +* Ensure tools are able to be used by community members regardless of their region. + +If you have concerns about communication challenges for this project, please contact the codeowners. + +[Code of Conduct]: CODE_OF_CONDUCT.md +[Codeowners File]: CODEOWNERS.csv +[Contribution Guidelines]: CONTRIBUTING.md \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 000000000..2bb9ad240 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/PYPI-README.md b/PYPI-README.md new file mode 100644 index 000000000..52c44e190 --- /dev/null +++ b/PYPI-README.md @@ -0,0 +1,20 @@ +# What is RTDIP SDK? + +​​**Real Time Data Ingestion Platform (RTDIP) SDK** is a software development kit built to easily access some of RTDIP's transformation functions. + +The RTDIP SDK will give the end user the power to use some of the convenience methods for frequency conversions and resampling of Pi data all through a self-service platform. RTDIP is offering a flexible product with the ability to authenticate and connect to Databricks SQL Warehouses given the end users preferences. RTDIP have taken the initiative to cut out the middle man and instead wrap these commonly requested methods in a simple python module so that you can instead focus on the data. + +See [RTDIP Documentation](https://www.rtdip.io/) for more information on how to use the SDK. + +# Licensing + +Distributed under the Apache 2.0 License. See [LICENSE.md](https://github.com/rtdip/core/blob/develop/LICENSE.md) for more information. + +# Need help? +* For reference documentation, pre-requisites, getting started, tutorials and examples visit [RTDIP Documentation](https://www.rtdip.io/). +* File an issue via [Github Issues](https://github.com/rtdip/core/issues). +* Check previous questions and answers or ask new ones on our slack channel [**#real-time-data-ingestion-platform**](https://join.slack.com/t/lfenergy/shared_invite/zt-1ilkyecnq-8TDP6pzZXnmx1o0Lc~kMcA) + + +# Community +* Chat with other community members by joining the **#real-time-data-ingestion-platform** Slack channel. [Click here to join our slack community](https://join.slack.com/t/lfenergy/shared_invite/zt-1ilkyecnq-8TDP6pzZXnmx1o0Lc~kMcA) \ No newline at end of file diff --git a/README.md b/README.md index e2720ea0a..4d8f282b7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,158 @@ -# core -Easy access to high volume, historical and real time process data for analytics applications, engineers, and data scientists wherever they are. +# Real Time Data Ingestion Platform (RTDIP) + +This repository contains Real Time Data Ingestion Platform SDK functions and documentation. This README will be a developer guide to understand the repository. + +## What is RTDIP SDK? + +​​**Real Time Data Ingestion Platform (RTDIP) SDK** is a software development kit built to easily access some of RTDIP's transformation functions. + +The RTDIP SDK will give the end user the power to use some of the convenience methods for frequency conversions and resampling of Pi data all through a self-service platform. RTDIP is offering a flexible product with the ability to authenticate and connect to Databricks SQL Warehouses given the end users preferences. RTDIP have taken the initiative to cut out the middle man and instead wrap these commonly requested methods in a simple python module so that you can instead focus on the data. + +See [RTDIP Documentation](https://www.rtdip.io/) for more information on how to use the SDK. + +| Branch | Status | +|--------|--------| +| main | [![Main](https://github.com/rtdip/core/actions/workflows/main.yml/badge.svg?branch=develop)](https://github.com/rtdip/core/actions/workflows/main.yml) | +| develop | [![Develop](https://github.com/rtdip/core/actions/workflows/develop.yml/badge.svg)](https://github.com/rtdip/core/actions/workflows/develop.yml) | +| feature | [![.github/workflows/pr.yml](https://github.com/rtdip/core/actions/workflows/pr.yml/badge.svg)](https://github.com/rtdip/core/actions/workflows/pr.yml) | + +# Repository Guidelines +The structure of this repository is shown below in the tree diagram. + + β”œβ”€β”€ .devcontainer + β”œβ”€β”€ .github + β”‚ β”œβ”€β”€ workflows + β”œβ”€β”€ docs + β”‚ β”œβ”€β”€ api + β”‚ β”œβ”€β”€ assets + β”‚ β”œβ”€β”€ blog + β”‚ β”œβ”€β”€ getting-started + β”‚ β”œβ”€β”€ images + β”‚ β”œβ”€β”€ integration + β”‚ β”œβ”€β”€ releases + β”‚ β”œβ”€β”€ roadmap + β”‚ β”œβ”€β”€ sdk + β”‚ β”œβ”€β”€ index.md + β”œβ”€β”€ src + β”‚ β”œβ”€β”€ api + β”‚ β”‚ β”œβ”€β”€ assets + β”‚ β”‚ β”œβ”€β”€ auth + β”‚ β”‚ β”œβ”€β”€ FastAPIApp + β”‚ β”‚ β”œβ”€β”€ v1 + β”‚ β”œβ”€β”€ apps + β”‚ β”‚ β”œβ”€β”€ docs + β”‚ β”œβ”€β”€ sdk + β”‚ β”‚ β”œβ”€β”€ python + β”‚ β”‚ β”‚ β”œβ”€β”€ rtdip-sdk + β”‚ β”‚ β”‚ β”‚ β”œβ”€β”€ authentication + β”‚ β”‚ β”‚ β”‚ β”œβ”€β”€ functions + β”‚ β”‚ β”‚ β”‚ β”œβ”€β”€ odbc + β”œβ”€β”€ tests + β”‚ β”œβ”€β”€ api + β”‚ β”‚ β”œβ”€β”€ auth + β”‚ β”‚ β”œβ”€β”€ v1 + β”‚ β”œβ”€β”€ apps + β”‚ β”‚ β”œβ”€β”€ docs + β”‚ β”œβ”€β”€ sdk + β”‚ β”‚ β”œβ”€β”€ python + β”‚ β”‚ β”‚ β”œβ”€β”€ rtdip-sdk + β”‚ β”‚ β”‚ β”‚ β”œβ”€β”€ authentication + β”‚ β”‚ β”‚ β”‚ β”œβ”€β”€ functions + β”‚ β”‚ β”‚ β”‚ β”œβ”€β”€ odbc + β”œβ”€β”€ CODE_OF_CONDUCT.md + β”œβ”€β”€ CODEOWNERS.md + β”œβ”€β”€ CONTRIBUTING.md + β”œβ”€β”€ GOVERNANCE.md + β”œβ”€β”€ LICENSE.md + β”œβ”€β”€ RELEASE.md + β”œβ”€β”€ SUPPORT.md + β”œβ”€β”€ PYPI-README.md + β”œβ”€β”€ environment.yml + β”œβ”€β”€ mkdocs.yml + β”œβ”€β”€ setup.py + └── .gitignore + +## Folder Structure + +| Folder Name | Description | +|--------------------|----------------------------------------------------------------------| +|`.github/workflows` | yml files for Github Action workflows | +|`.devcontainer` | Setup for VS Code Remote Containers and Github Spaces | +|`docs` | Documentation in markdown, organised by subject name at folder level | +|`src` | Main projects and all souce code, organised by language and sdk name | +|`tests` | Test projects and unit tests, organised by language and sdk name | + +## File Structure + +| File Name | Description | +|------------------|-----------------------------------------------------------------------------------------| +|`mkdocs.yml` | yml file to host all documentation on mkdocs | +|`setup.py` | Set up requirements for python package deployment | +|`environment.yml` | yml file to create an environment with all the dependencies for developers | +|`CODE_OF_CONDUCT` | code of conduct | +|`CODEOWNERS` | codeowners | +|`CONTRIBUTING.yml`| contributing | +|`GOVERNANCE.yml` | governance | +|`LICENSE.yml` | license | +|`RELEASE.yml` | releases | +|`SUPPORT.yml` | support | +|`PYPI-README.yml` | PyPi read me documentation | +|`README.yml` | RTDIP read me documentation | +|`.gitignore` | Informs Git which files to ignore when committing your project to the GitHub repository | + +# Developer Guide + +## Getting Started + +1) To get started with developing for this project, clone the repository. +``` + git clone https://github.com/rtdip/core.git +``` +2) Open the respository in VS Code, Visual Studio or your preferered code editor. + +3) Create a new environment using the following command: +``` + conda env create -f environment.yml +``` + +> **_NOTE:_** You will need to have conda, python and pip installed to use the command above. + +4) Activate your newly set up environment using the following command: +``` + conda activate rtdip-sdk +``` +You are now ready to start developing your own functions. Please remember to follow RTDIP's development lifecycle to maintain clarity and efficiency for a fully robust self serving platform. + +## RTDIP Development Lifecycle + +1) Develop + +2) Write unit tests + +3) Document + +4) Publish + +> **_NOTE:_** Ensure you have read the [Release Guidelines](RELEASE.md) before publishing your code. + +# Contribution +This project welcomes contributions and suggestions. If you have a suggestion that would make this better you can simply open an issue with a relevant title. Don't forget to give the project a star! Thanks again! + +For more details on contributing to this repository, see the [Contibuting Guide](https://github.com/rtdip/core/blob/develop/CONTRIBUTING.md). Please read this projects [Code of Conduct](https://github.com/rtdip/core/blob/develop/CODE_OF_CONDUCT.md) before contributing. + +## Documentation +This repository has been configured with support documentation for Real Time Data Ingestion Platform (RTDIP) to make it easier to get started. RTDIP's documentation is created using mkdocs and is hosted on Github Pages. + +* Documentation can be found on [RTDIP Documentation](https://www.rtdip.io/) + +# Licensing + +Distributed under the Apache License Version 2.0. See [LICENSE.md](https://github.com/rtdip/core/blob/develop/LICENSE.md) for more information. + +# Need help? +* For reference documentation, pre-requisites, getting started, tutorials and examples visit [RTDIP Documentation](https://www.rtdip.io/). +* File an issue via [Github Issues](https://github.com/rtdip/core/issues). +* Check previous questions and answers or ask new ones on our slack channel [**#real-time-data-ingestion-platform**](https://join.slack.com/t/lfenergy/shared_invite/zt-1ilkyecnq-8TDP6pzZXnmx1o0Lc~kMcA) + +### Community +* Chat with other community members by joining the **#real-time-data-ingestion-platform** Slack channel. [Click here to join our slack community](https://join.slack.com/t/lfenergy/shared_invite/zt-1ilkyecnq-8TDP6pzZXnmx1o0Lc~kMcA) \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..10d59f22d --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,92 @@ +# RELEASES + +## Release policy + +We strive to improve the repo and fix known issues. + + +RTDIP is released with semantic versioning. + +Given a version number MAJOR.MINOR.PATCH, increment the: + +* MAJOR version when you make incompatible API changes: example vX.0.0 + +* MINOR version when you add functionality in a backwards compatible manner: example v0.X.0 + +* PATCH version when you make backwards compatible bug fixes: example v0.0.X + +## Checklist + +This checklict guides you through preparing, testing and documenting a release. + +``` +Please ensure to SQUASH MERGE to DEVELOP and MERGE to MAIN +``` + +### Squash Merging a Pull Request into Develop + +- [ ] Update your current branch with your work + - [ ] Decide which commits you want to push to your branch. + - [ ] Update dependencies. + - [ ] Ensure tests have been ran and passed. + - [ ] Check whether documentation has been updated and is relevant to any new functionalities. + - [ ] Commit & push + - local changes (e.g. from the change log updates): `git commit -am "..."` + - `git push` + +- [ ] Create a Pull Request + - [ ] Provide an appropriate title for your pull request. + - [ ] Choose a REVIEWER from the [Code Owners List](./CODEOWNERS.md). + - [ ] Use appropriate labels on your pull request, under Labels on the right hand side. + - [ ] If necessary link issues into your pull request by selecting an ID under Development on the right hand side. + - [ ] Check whether all tests have passed and that Sonarcloud has not flagged any issues. If there are any issues please make the changes on your branch. + +- [ ] Review and Approve a Pull Request + - [ ] The REVIEWER must review the code and ask questions/comment where appropriate. + - [ ] Before the REVIEWER can squash merge into Develop all comments/questions should be closed and all checks must be passed. + - [ ] The REVIEWER who approves the request must do a __SQUASH MERGE__ into Develop. + +### Merging Develop to Main + +- [ ] Create a Pull Request from Develop to Main + - [ ] Provide the the version number ie. vX.X.X as the title. + - [ ] Choose two REVIEWERs from the [Code Owners List](./CODEOWNERS.md), under Reviewers on the right hand side. + - [ ] Check whether all tests have passed and that Sonarcloud has not flagged any issues. If there are any issues you will not be able to merge. + +- [ ] Review and Approve a Pull Request + - [ ] The REVIEWERs must ensure that the only changes into Main are coming from Develop. + - [ ] The REVIEWER must review the code and ask questions/comment where appropriate. + - [ ] Before the REVIEWER can merge into Main all comments/questions should be closed and all checks must be passed. + - [ ] The REVIEWERs who approves the request must __MERGE__ into Main. + +### Creating a New Release from Main + +- [ ] Verify you have successfully merged into Main + +- [ ] Create a New Realease + - [ ] Click on the _Release_ tab on the right hand side of the repository. + - [ ] Click on the _Draft a new release_ button + - [ ] Provide an appropriate Tag Name, please use the version number ie. vX.X.X + - [ ] Select main as the Target + - [ ] Provide an appropriate Title, please use the version number ie. vX.X.X + + ![Release-Variable](./docs/images/release-images/Release%20Target%20Title.png) + + - [ ] Click the _Generate release notes_ button. + + ![Generate-Release-Notes](docs/images/release-images/Generate%20Release%20Notes.png) + + - [ ] Verify the Notes. + - [ ] Tick the _Set as the latest release_ option. + + ![Latest-Release](./docs/images/release-images/Set%20As%20Latest%20Release.png) + + - [ ] Create your release by clicking the _Publish release_ button. + + ![Publish-Release](./docs/images/release-images/Publish%20Release.png) + +- [ ] Verify the Release is Successful + - [ ] Check whether the relase is available on PyPi. + - [ ] If the release has failed troubleshoot the issue and re run the job. + - [ ] Update LF Energy on the release. + - [ ] Mention the release on suitable social media accounts. \ No newline at end of file diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 000000000..90c0e2cdf --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,21 @@ +# Getting Help + +To connect with the Real Time Data Ingestion Platform (RTDIP) project please create an issue under the general enquiry tag and one of the team will respond shortly. + +## How to Ask for Help + +If you have trouble installing, building, or using RTDIP, but there's not yet reason to suspect you've encountered a genuine bug, +start by posting a question. This is the place for question such as "How do I...". + +## How to report a bug or request an enhancement + +RTDIP manages bugs and enhancements using it's GitHub Projects board. The [issue template](.github/ISSUE_TEMPLATE) will guide you on making an effective report. + +## How to report a security vulnerability + +If you think you've found a potential vulnerability in https://github.com/rtdip/core/, please +email info@rtdip.io to responsibly disclose it. + +## Contributing a fix + +Please refer to [CONTRIBUTING.md](CONTRIBUTING.md) to make a project contribution. \ No newline at end of file diff --git a/docs/api/authentication.md b/docs/api/authentication.md new file mode 100644 index 000000000..79de6cf4f --- /dev/null +++ b/docs/api/authentication.md @@ -0,0 +1,20 @@ +# Authentication + +RTDIP REST APIs require Azure Active Directory Authentication and passing the token received as an `authorization` header in the form of a Bearer token. An example of the REST API header is `Authorization: Bearer <>` + +## End User Authentication + +If a developer or business user would like to leverage the RTDIP REST API suite, it is recommended that they use the Identity Packages provided by Azure to obtain a token. + +- [REST API](https://docs.microsoft.com/en-us/azure/azure-app-configuration/rest-api-authentication-azure-ad){target=_blank} +- [.NET](https://docs.microsoft.com/en-us/dotnet/api/overview/azure/identity-readme){target=_blank} +- [Java](https://docs.microsoft.com/en-us/java/api/overview/azure/identity-readme?view=azure-java-stable){target=_blank} +- [Python](https://docs.microsoft.com/en-us/python/api/overview/azure/identity-readme?view=azure-python){target=_blank} +- [Javascript](https://docs.microsoft.com/en-us/javascript/api/overview/azure/identity-readme?view=azure-node-latest){target=_blank} + +!!! note "Note" + Note that the above packages have the ability to obtain tokens for end users and service principals and support all available authentication options. + +Ensure to install the relevant package and obtain a token. + +See the [examples](./examples.md) section to see various authentication methods implemented. diff --git a/docs/api/deployment/azure.md b/docs/api/deployment/azure.md new file mode 100644 index 000000000..f1f126c68 --- /dev/null +++ b/docs/api/deployment/azure.md @@ -0,0 +1,35 @@ +# Deploy RTDIP APIs to Azure + +The RTDIP repository contains the code to deploy the RTDIP REST APIs to your own Azure Cloud environment. The APIs are built as part of the rtdip repository CI/CD pipelines and the image is deployed to Docker Hub repo `rtdip/api`. Below contains information on how to build and deploy the containers from source or to setup your function app to use the deployed container image provided by RTDIP. + +## Deploying the RTDIP APIs + +### Deployment from Build + +To deploy the RTDIP APIs directly from the repository, follow the steps below: + +1. Build the docker image using the following command: + ```bash + docker build --tag /rtdip-api:v0.1.0 -f src/api/Dockerfile . + ``` +1. Login to your container registry + ```bash + docker login + ``` +1. Push the docker image to your container registry + ```bash + docker push /rtdip-api:v0.1.0 + ``` +1. Configure your Function App to use the docker image + ```bash + az functionapp config container set --name --resource-group --docker-custom-image-name /rtdip-api:v0.1.0 + ``` + +### Deployment from Docker Hub + +To deploy the RTDIP APIs from Docker Hub, follow the steps below: + +1. Configure your Function App to use the docker image + ```bash + az functionapp config container set --name --resource-group --docker-custom-image-name rtdip/api:azure- + ``` diff --git a/docs/api/examples.md b/docs/api/examples.md new file mode 100644 index 000000000..64309e596 --- /dev/null +++ b/docs/api/examples.md @@ -0,0 +1,135 @@ +# Examples + +Below are examples of how to execute APIs using various authentication options and API methods. + +## End User Authentication + +### Python + +A python example of obtaining a token as a user can be found below using the `azure-identity` python package to authenticate with Azure AD. + +!!! note "POST Requests" + The POST request can be used to pass many tags to the API. This is the preferred method when passing large volumes of tags to the API. + +=== "GET Request" + ```python + from azure.identity import DefaultAzureCredential + import requests + + authentication = DefaultAzureCredential() + access_token = authentication.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + + params = { + "business_unit": "Business Unit", + "region": "Region", + "asset": "Asset Name", + "data_security_level": "Security Level", + "data_type": "float", + "tag_name": "TAG1", + "tag_name": "TAG2", + "start_date": "2022-01-01", + "end_date": "2022-01-01", + "include_bad_data": True + } + + url = "https://example.com/api/v1/events/raw" + + payload={} + headers = { + 'Authorization': 'Bearer {}'.format(access_token) + } + + response = requests.request("GET", url, headers=headers, params=params, data=payload) + + print(response.json()) + ``` + +=== "POST Request" + ```python + from azure.identity import DefaultAzureCredential + import requests + + authentication = DefaultAzureCredential() + access_token = authentication.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + + params = { + "business_unit": "Business Unit", + "region": "Region", + "asset": "Asset Name", + "data_security_level": "Security Level", + "data_type": "float", + "start_date": "2022-01-01T15:00:00", + "end_date": "2022-01-01T16:00:00", + "include_bad_data": True + } + + url = "https://example.com/api/v1/events/raw" + + payload={"tag_name": ["TAG1", "TAG2"]} + + headers = { + "Authorization": "Bearer {}".format(access_token), + } + + # Requests automatically sets the Content-Type to application/json when the request body is passed via the json parameter + response = requests.request("POST", url, headers=headers, params=params, json=payload) + + print(response.json()) + ``` + +## Service Principal Authentication + +### GET Request + +Authentication using Service Principals is similar to end user authentication. An example, using Python is provided below where the `azure-identity` package is not used, instead a direct REST API call is made to retrieve the token. + +=== "cURL" + ```curl + curl --location --request POST 'https://login.microsoftonline.com/{tenant id}/oauth2/v2.0/token' \ + --form 'grant_type="client_credentials"' \ + --form 'client_id="<>"' \ + --form 'client_secret="<>"' \ + --form 'scope="2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default"' + ``` + +=== "Python" + ```python + import requests + + url = "https://login.microsoftonline.com/{tenant id}/oauth2/v2.0/token" + + payload={'grant_type': 'client_credentials', + 'client_id': '<>', + 'client_secret': '<>', + 'scope': '2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default'} + files=[] + headers = {} + + response = requests.request("POST", url, headers=headers, data=payload, files=files) + + access_token = response.json()["access_token"]) + + params = { + "business_unit": "Business Unit", + "region": "Region", + "asset": "Asset Name", + "data_security_level": "Security Level", + "data_type": "float", + "tag_name": "TAG1", + "tag_name": "TAG2", + "start_date": "2022-01-01", + "end_date": "2022-01-01", + "include_bad_data": True + } + + url = "https://example.com/api/v1/events/raw" + + payload={} + headers = { + 'Authorization': 'Bearer {}'.format(access_token) + } + + response = requests.request("GET", url, headers=headers, params=params, data=payload) + + print(response.text) + ``` \ No newline at end of file diff --git a/docs/api/images/open-api.png b/docs/api/images/open-api.png new file mode 100644 index 000000000..6d9b4fdb2 Binary files /dev/null and b/docs/api/images/open-api.png differ diff --git a/docs/api/images/redoc-logo.png b/docs/api/images/redoc-logo.png new file mode 100644 index 000000000..63e5ba2fd Binary files /dev/null and b/docs/api/images/redoc-logo.png differ diff --git a/docs/api/images/rest-api-logo.png b/docs/api/images/rest-api-logo.png new file mode 100644 index 000000000..20045d855 Binary files /dev/null and b/docs/api/images/rest-api-logo.png differ diff --git a/docs/api/images/swagger.png b/docs/api/images/swagger.png new file mode 100644 index 000000000..c81811f8a Binary files /dev/null and b/docs/api/images/swagger.png differ diff --git a/docs/api/overview.md b/docs/api/overview.md new file mode 100644 index 000000000..df6163b65 --- /dev/null +++ b/docs/api/overview.md @@ -0,0 +1,5 @@ +
![rest](images/rest-api-logo.png)
+ +# RTDIP REST APIs + +RTDIP provides REST API endpoints for querying data in the platform. The APIs are a wrapper to the python [RTDIP SDK](../sdk/overview.md) and provide similar functionality for users and applications that are unable to leverage the python RTDIP SDK. It is recommended to read the [RTDIP SDK documentation](../sdk/overview.md) and in particular the [Functions](../sdk/code-reference/resample.md) section for more information about the options and logic behind each API. diff --git a/docs/api/rest_apis.md b/docs/api/rest_apis.md new file mode 100644 index 000000000..fbc498a1c --- /dev/null +++ b/docs/api/rest_apis.md @@ -0,0 +1,15 @@ +# RTDIP REST API Endpoints + +RTDIP REST API documentation is available in a number of formats, as described below. + +
![rest](images/open-api.png)
+ +RTDIP REST APIs are built to OpenAPI standard 3.0.2. You can obtain the OpenAPI JSON schema at the following endpoint of your deployed APIs `https://{domain name}/api/openapi.json` + +
![rest](images/swagger.png)
+ +It is recommended to review the **Swagger** documentation that can be found at the following endpoint of your deployed APIs `https://{domain name}/docs` for more information about the parameters and options for each API. It is also possible to try out each API from this link. + +
![rest](images/redoc-logo.png)
+ +Additionally, further information about each API can be found in Redoc format at the following endpoint of your deployed APIs `https://{domain name}/redoc` \ No newline at end of file diff --git a/docs/assets/extra.css b/docs/assets/extra.css new file mode 100644 index 000000000..8e16b6833 --- /dev/null +++ b/docs/assets/extra.css @@ -0,0 +1,45 @@ +/** + * Copyright 2022 RTDIP + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +:root { + --md-primary-fg-color: #4e08c7; + --md-primary-mg-color: #d445a3; + --md-accent-fg-color: #bb1fa4; + --md-primary-bg-color: white; + --md-primary-text-slate: white; + --md-primary-bg-slate: #2f303e; + } + +/* header font colour */ +.md-header { + color: white; +} + +.md-search__icon { + color: white !important; +} + +.md-tabs__item { + color: white !important; +} + +.md-footer-meta { + display: none; +} + +.md-nav__item .md-nav__link--active { + color:#d445a3; +} \ No newline at end of file diff --git a/docs/assets/favicon.png b/docs/assets/favicon.png new file mode 100644 index 000000000..c77d46338 Binary files /dev/null and b/docs/assets/favicon.png differ diff --git a/docs/assets/illustration.png b/docs/assets/illustration.png new file mode 100644 index 000000000..69f739c05 Binary files /dev/null and b/docs/assets/illustration.png differ diff --git a/docs/assets/logo_lfe.png b/docs/assets/logo_lfe.png new file mode 100644 index 000000000..15f387442 Binary files /dev/null and b/docs/assets/logo_lfe.png differ diff --git a/docs/blog/delta_and_rtdip.md b/docs/blog/delta_and_rtdip.md new file mode 100644 index 000000000..2c3269c5d --- /dev/null +++ b/docs/blog/delta_and_rtdip.md @@ -0,0 +1,106 @@ +# Delta Lakehouse and Real Time Data Ingestion Platform + +
![Delta Lakehouse](images/delta-lakehouse.svg)
+ +Real Time Data Ingestion Platform leverages Delta and the concept of a Lakehouse to ingest, store and manage it's data. There are many benefits to Delta for performing data engineering tasks on files stored in a data lake including ACID transactions, maintenance, SQL query capability and performance at scale. To find out more about Delta Lakehouse please see [here.](https://databricks.com/product/data-lakehouse) + +The Real Time Data Ingestion Platform team would like to share some lessons learnt from the implementation of Delta and the Lakehouse concept so that hopefully it helps others on their Delta Lakehouse journey. + +For reference, please consider the typical layout of timeseries data ingested by RTDIP: + +**Events** + +| Column Name | Desciption | +|-------------|------------| +| TagName | Typically represents a sensor name or a measurement | +| EventTime | A timestamp for a recorded value | +| Status | Status of the recording, normally indicating if the measurement value is good or bad | +| Value | The value of the measurement and can be of a number of types - float, double, string, integer | + +**Metadata** + +| Column Name | Desciption | +|-------------|------------| +| TagName | Typically represents a sensor name or a measurement | +| Description | A description for the sensor | +| UoM | UoM for the measurement | + +!!! note "Note" + Metadata can include a number of additional columns and depends on the system that provides the metadata. The above are the required columns for any sensor data ingested by RTDIP. + +## Design Considerations + +Delta, in its simplest definition, is a set of parquet files managed by an index file. This allows Spark to perform tasks like partition pruning and file pruning to find the exact parquet file to be used by any ACID transactions being performed on it. By reducing the number of files and the amount of data that Spark needs to read in a query means that it will perform much better. It is important to consider the following when designing a Delta Table to achieve performance benefits: + +- Columns that are likely to be used in most reads and writes +- Partition column(s) +- File Sizes + +### Partition Columns + +The biggest benefit achieved using Delta is to include a partition column in the design of a Delta Table. This is the fastest way for Spark to isolate the dataset it needs to work with in a query. The general rule of thumb is that each partition size should be roughly **1gb** in size, and ideally would be a column or columns that are used in every query to filter data for that table. + +This can be difficult to achieve. The most queried columns in RTDIP event data are TagName and EventTime, however, partitioning data by TagName creates far too many small parititons and a timestamp column like EventTime can not be used for partitioning for the same reason. The best outcome is typically to create an additional column that is an aggregation of the EventTime column, such as a Date, Month or Year Column, depending on the frequency of the data being ingested. + +!!! note "Note" + **Given the above, always query RTDIP delta events tables using EventDate in the filter to achieve the best results.** + +One of the best methods to analyse Spark query performance is to analyse the query plan of a query. It is essential that a Spark query plan leverages a partition column. This can be identified by reviewing the query plan in Spark. As per the below query plan, it can be seen that for this particular query only one partition was read by Spark. Make sure to try different queries to identify that the expected number of partitions are being used by Spark in every query. If it does not match your expected number of partitions, it is important to investigate why partition pruning is not being leveraged in your query. + +
![query plan](images/spark-query-plan.png)
+ +### ZORDER Columns + +Even though the rule is to achieve roughly **1gb** partitions for a Delta Table, Delta is likely to divide that partition into a number of files. The default target size is around 128gb per file. Due to this, it is possible to improve performance above and beyond partitioning by telling Spark which files within in a partition to read. This is where **ZORDER** becomes useful. + +Zordering organises the data within each file, and along with the Delta Index file, directs Spark to the exact files to use in its reads(and merge writes) on the table. It is important to find the right number of columns to ZORDER - the best outcome would be a combination of columns that does not cause the index to grow too large. For example, ZORDERING by TagName creates a small index, but ZORDERING by TagName and EventTime created a huge index as there are far more combinations to be indexed. + +The most obvious column to ZORDER on in RTDIP is the TagName as every query is likely to use this in its filter. Like parition pruning, it is possible to identify the impact of ZORDERING on your queries by reviewing the files read attribute in the query plan. As per the query plan below, you can see that two files were read within the one partition. + +
![query plan](images/spark-query-plan.png)
+ +### MERGE and File Sizes + +As stated above, the default target size for file sizes within a partition is 128gb. However, this is not always ideal and in certain scenarios, it is possible to improve performance of Spark jobs by reducing file sizes in cetain scenarios: +- MERGE commands +- Queries that target very small subsets of data within a file + +Due to the nature of Merges, its typically an action where small updates are being made to the dataset. Due to this, it is possible to get much better MERGE performance by setting the following attribute on the Delta Table `delta.tuneFileSizesForRewrites=true`. This targets smaller file sizes to reduce the amount of data in each read a MERGE operation performs on the data. RTDIP gained a significant performance improvement in reading and writing and was able to reduce the Spark cluster size by half by implementing this setting on its Delta Tables. + +However, even more performance gain was achieved when Databricks released [Low Shuffle Merge](https://databricks.com/blog/2021/09/08/announcing-public-preview-of-low-shuffle-merge.html) from DBR 9.0 onwards. This assists Spark to merge data into files without disrupting the ZORDER layout of Delta files, in turn assisting Merge commands to continue leveraging ZORDER performance benefits on an ongoing basis. RTDIP was able to improve MERGE performance by 5x with this change. To leverage Low Shuffle Merge, set the following Spark config in your notebook `spark.databricks.delta.merge.enableLowShuffle=true`. + +### Delta Table Additional Attributes + +It is recommendded to consider setting the following two attributes on all Delta Tables: + +- `delta.autoOptimize.autoCompact=true` +- `delta.autoOptimize.optimizeWrite=true` + +To understand more about optimization options you can set on Delta Tables, please refer to this [link.](https://docs.databricks.com/delta/optimizations/file-mgmt.html) + +## Maintenance Tasks + +One important step to be included with every Delta Table is maintenance. Most developers forget these very important maintenance tasks that need to run on a regular basis to maintain performance and cost on Delta Tables. + +As a standard, run a maintenance job every 24 hours to perform OPTIMIZE and VACUUM commands on Delta Tables. + +### OPTIMIZE + +OPTIMIZE is a Spark SQL command that can be run on any Delta Table and is the simplest way to optimize the file layout of a Delta Table. The biggest benefit of running OPTIMIZE however, is to organize Delta files using ZORDER. Due to how effecive ZORDER is on queries, its unlikely that OPTIMIZE would not be executed on a Delta Table regularly. + +It may be a question as to why one would run OPTIMIZE as well as set `delta.autoOptimize.autoCompact=true` on all its tables. Auto Compact does not ZORDER its data(at the time of writing this article), its task is simply to attempt to create larger files during writing and avoid the small file problem. Therefore, autoCompact does not provide ZORDER capability. Due to this, consider an OPTIMIZE strategy as follows: +- Auto Compact is used by default for any new files written to an RTDIP Delta Table between the execution of maintenance jobs. This ensures that any new data ingested by RTDIP is still being written in a suitable and performant manner. +- OPTIMIZE with ZORDER is run on a daily basis on any partitions that have changed(excluding the current day) to ensure ZORDER and updating of the Delta Index file is done. + +!!! note "Note" + RTDIP data is going to typically be ingesting using Spark Streaming - given the nature of a real time data ingestion platform, it makes sense that data ingestion is performed in real time. One complication this introduces is the impact of the OPTIMIZE command being executed at the same time as files being written to a partition. Due to this, execute OPTIMIZE on partitions where the EventDate is not equal to the current date, minimizing the possibility of an OPTIMIZE command and a file write command being executed on a partition at the same time. This logic reduces issues experienced by both the maintenance job and Spark streaming job. + +### VACUUM + +One of the most powerful features of Delta is time travel. This allows querying of data as at a certain point of time in the past, or a particular version of the Delta Table. Whilst incredibly useful, it does consume storage space and if these historical files are never removed, the size of Delta Tables can grow exponentially large and increase cost. + +To ensure only the required historical versions of a Delta Table are stored, its important to execute the VACUUM command every 24 hours. This deletes any files or versions that are outside the time travel retention period. + +## Conclusion + +Delta and the Lakehouse transformed the way RTDIP ingests its data and provides integration with other projects, applications and platforms. We hope the above assists others with their Delta development and we look forward to posting more content on RTDIP and its use of Spark in the future. \ No newline at end of file diff --git a/docs/blog/images/blog-icon.png b/docs/blog/images/blog-icon.png new file mode 100644 index 000000000..60eedd38b Binary files /dev/null and b/docs/blog/images/blog-icon.png differ diff --git a/docs/blog/images/delta-lakehouse.svg b/docs/blog/images/delta-lakehouse.svg new file mode 100644 index 000000000..f1eb8a338 --- /dev/null +++ b/docs/blog/images/delta-lakehouse.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/blog/images/spark-query-plan.png b/docs/blog/images/spark-query-plan.png new file mode 100644 index 000000000..c9b99c004 Binary files /dev/null and b/docs/blog/images/spark-query-plan.png differ diff --git a/docs/blog/overview.md b/docs/blog/overview.md new file mode 100644 index 000000000..87bf9fb36 --- /dev/null +++ b/docs/blog/overview.md @@ -0,0 +1,4 @@ +
![blog](images/blog-icon.png) + +## Blogs about the Real Time Data Ingestion Platform +
\ No newline at end of file diff --git a/docs/getting-started/about-rtdip.md b/docs/getting-started/about-rtdip.md new file mode 100644 index 000000000..159bc487b --- /dev/null +++ b/docs/getting-started/about-rtdip.md @@ -0,0 +1,11 @@ +# About RTDIP + +![timeseries](images/timeseries.gif){width=100%} + +## What is Real Time Data Ingestion Platform + +Organizations need data for day-to-day operations and to support advanced Data Science, Statistical and Machine Learning capabilities such as Optimization, Surveillance, Forecasting, and Predictive Analytics. **Real Time Data** forms a major part of the total data utilized in these activities. + +Real time data enables organizations to detect and respond to changes in their systems thus improving the efficiency of their operations. This data needs to be available in scalable and secure data platforms. + +**Real Time Data Ingestion Platform (RTDIP)** is the solution of choice leveraging **PaaS** (Platform as a Service) services along with some custom components to provide Data Ingestion, Data Transformation, and Data Sharing as a platform. RTDIP can interface with several data sources to ingest many different data types which include time series, alarms, video, photos and audio being provided from sources such as Historians, OPC Servers and Rest APIs, as well as data being sent from hardware such as IoT Sensors, Robots and Drones. diff --git a/docs/getting-started/images/timeseries.gif b/docs/getting-started/images/timeseries.gif new file mode 100644 index 000000000..d2b59e493 Binary files /dev/null and b/docs/getting-started/images/timeseries.gif differ diff --git a/docs/getting-started/images/timeseries2.gif b/docs/getting-started/images/timeseries2.gif new file mode 100644 index 000000000..da91f66c9 Binary files /dev/null and b/docs/getting-started/images/timeseries2.gif differ diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 000000000..858516c97 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,71 @@ +# Getting started with the RTDIP SDK + +This article provides a guide on how to install the RTDIP SDK. Get started by ensuring you have all the prerequisites before following the simple installation steps. + +* [Prerequisites](#prerequisites) + +* [Installing](#installing-the-rtdip-sdk) + +## Prerequisites +There are a few things to note before using the RTDIP SDK. The following prerequisites will need to be installed on your local machine. + +* Python version 3.8 >=, < 4.0 should be installed. Check which python version you have with the following command: + + python --version + + Find the latest python version [here](https://www.python.org/downloads/) and ensure your python path is set up correctly on your machine. + +* Ensure your pip python version matches the python version on your machine. Check which version of pip you have installed with the following command: + + pip --version + + There are two ways to ensure you have the correct versions installed. Either upgrade your Python and pip install or create an environment. + + **Option 1**: To upgrade your version of pip use the following command: + + python -m pip install --upgrade pip + + **OR** + + **Option 2**: To create an environment, you will need to create a **environment.yml** file with the following: + + name: rtdip-sdk + channels: + - conda-forge + - defaults + dependencies: + - python==3.8.8 + - pip==22.0.2 + - pip: + - rtdip-sdk + + Run the following command: + + conda env create -f environment.yml + + To update an environment previously created: + + conda env update -f environment.yml + +* To use pyodbc or turbodbc python libraries, ensure that the required ODBC driver is installed as per these [instructions](https://docs.microsoft.com/en-us/azure/databricks/integrations/bi/jdbc-odbc-bi#download-the-odbc-driver) + +* If you plan to use pyodbc, Microsoft Visual C++ 14.0 or greater is required. Get it from [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) + +* To use turbodbc python library, ensure that [Boost](https://turbodbc.readthedocs.io/en/latest/pages/getting_started.html) is installed correctly. + +## Installing the RTDIP SDK + +RTDIP SDK is a PyPi package that can be found [here](https://pypi.org/project/rtdip-sdk/). On this page you can find the **project description**, **release history**, **statistics**, **project links** and **maintainers**. + +1\. To install the latest released version of RTDIP SDK from PyPi use the following command: + + pip install rtdip-sdk + +If you have previously installed the RTDIP SDK and would like the latest version, see below. + + pip install rtdip-sdk --upgrade + +2\. Once the installation is complete you can learn how to use the SDK [here.](../sdk/rtdip-sdk-usage.md) + +!!! note "Note" + If you are having trouble installing the SDK, ensure you have installed all of the prerequisites. If problems persist please see [Troubleshooting](../sdk/troubleshooting.md) for more information. Please also reach out to the RTDIP team via Issues, we are always looking to improve the SDK and value your input.
diff --git a/docs/getting-started/why-rtdip.md b/docs/getting-started/why-rtdip.md new file mode 100644 index 000000000..922464f39 --- /dev/null +++ b/docs/getting-started/why-rtdip.md @@ -0,0 +1,24 @@ +# Why RTDIP? + +![timeseries](images/timeseries2.gif){:height=50% width=100%} + +## ​Why Real Time Data Ingestion Platform? + +**Real Time Data Ingestion Platform (RTDIP)** enables users to consume **Real Time Data** at scale, including historical and real time streaming data. **RTDIP** has proven to be capable of ingesting over 3 million sensors in a production environment across every geographical location in the world. + +The Real Time Data Ingestion Platform can be run in a customers own environment, allowing them to accelerate their cloud deployments while leveraging a proven design to scale their time series data needs. + +RTDIP also provides a number popular integration options, including: + +1. ODBC +1. JDBC +1. Rest API +1. Python SDK + +These options allow users to integrate with a wide variety of applications and tools, including: + +1. Data Visualization Tools such as ***Power BI, Seeq, Tableau, and Grafana*** +1. Data Science Tools such as ***Jupyter Notebooks, R Studio, and Python*** +1. Data Engineering Tools such as ***Apache Spark, Apache Kafka, and Apache Airflow*** + +RTDIP is architected to leverage Open Source technologies [Apache Spark](https://spark.apache.org/) and [Delta](https://delta.io/). This allows users to leverage the power of Open Source technologies to build their own custom applications and tools in whichever environment they prefer, whether that is in the cloud or on-premise on their own managed Spark Clusters. diff --git a/docs/images/background-curve-default.svg b/docs/images/background-curve-default.svg new file mode 100644 index 000000000..853517a73 --- /dev/null +++ b/docs/images/background-curve-default.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/images/background-curve-slate.svg b/docs/images/background-curve-slate.svg new file mode 100644 index 000000000..172eb178c --- /dev/null +++ b/docs/images/background-curve-slate.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/images/curve.png b/docs/images/curve.png new file mode 100644 index 000000000..75daea694 Binary files /dev/null and b/docs/images/curve.png differ diff --git a/docs/images/release-images/Generate Release Notes.png b/docs/images/release-images/Generate Release Notes.png new file mode 100644 index 000000000..a0fbe1ffc Binary files /dev/null and b/docs/images/release-images/Generate Release Notes.png differ diff --git a/docs/images/release-images/Publish Release.png b/docs/images/release-images/Publish Release.png new file mode 100644 index 000000000..9453eba7d Binary files /dev/null and b/docs/images/release-images/Publish Release.png differ diff --git a/docs/images/release-images/Release Target Title.png b/docs/images/release-images/Release Target Title.png new file mode 100644 index 000000000..af48b7a43 Binary files /dev/null and b/docs/images/release-images/Release Target Title.png differ diff --git a/docs/images/release-images/Set As Latest Release.png b/docs/images/release-images/Set As Latest Release.png new file mode 100644 index 000000000..8b8df803d Binary files /dev/null and b/docs/images/release-images/Set As Latest Release.png differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..7578f4464 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,99 @@ +--- +hide: + - navigation + - toc +--- + + + +
+
+
+

Welcome to the Real Time Data Ingestion Platform

+

Easy access to high volume, historical and real time process data for analytics applications, engineers, and data scientists wherever they are.

+
+
+
+ + Getting Started + +
+
+
+
+
+ +
+
+
diff --git a/docs/integration/images/bi-azure-signin.png b/docs/integration/images/bi-azure-signin.png new file mode 100644 index 000000000..6bd2e8ea0 Binary files /dev/null and b/docs/integration/images/bi-azure-signin.png differ diff --git a/docs/integration/images/bi-getdata-more.png b/docs/integration/images/bi-getdata-more.png new file mode 100644 index 000000000..03b01cfe8 Binary files /dev/null and b/docs/integration/images/bi-getdata-more.png differ diff --git a/docs/integration/images/bi-search-databricks.png b/docs/integration/images/bi-search-databricks.png new file mode 100644 index 000000000..1d9ea30b6 Binary files /dev/null and b/docs/integration/images/bi-search-databricks.png differ diff --git a/docs/integration/images/databricks_powerbi.png b/docs/integration/images/databricks_powerbi.png new file mode 100644 index 000000000..cd0cb3895 Binary files /dev/null and b/docs/integration/images/databricks_powerbi.png differ diff --git a/docs/integration/images/power-bi-desktop.png b/docs/integration/images/power-bi-desktop.png new file mode 100644 index 000000000..527b9cb37 Binary files /dev/null and b/docs/integration/images/power-bi-desktop.png differ diff --git a/docs/integration/images/power-bi-install.png b/docs/integration/images/power-bi-install.png new file mode 100644 index 000000000..03c07f878 Binary files /dev/null and b/docs/integration/images/power-bi-install.png differ diff --git a/docs/integration/power-bi.md b/docs/integration/power-bi.md new file mode 100644 index 000000000..dcee4514b --- /dev/null +++ b/docs/integration/power-bi.md @@ -0,0 +1,36 @@ +# Integration of Power BI with RTDIP + +## Integration with Power BI + +Microsoft Power BI is a business analytics service that provides interactive visualizations with self-service business intelligence capabilities +that enable end users to create reports and dashboards by themselves without having to depend on information technology staff or database administrators. + +
![Power BI Databricks](images/databricks_powerbi.png){width=100%}
+ +When you use Azure Databricks as a data source with Power BI, you can bring the advantages of Azure Databricks performance and technology beyond data scientists and data engineers to all business users. + +You can connect Power BI Desktop to your Azure Databricks clusters and Databricks SQL warehouses by using the built-in Azure Databricks connector. You can also publish Power BI reports to the Power BI service and enable users to access the underlying Azure Databricks data using single sign-on (SSO), passing along the same Azure Active Directory credentials they use to access the report. + +For more information on how to connect Power BI with databricks, see [here](https://docs.microsoft.com/en-us/azure/databricks/integrations/bi/power-bi). + +## Power BI Installation Instructions + +1. Install **Power BI** Desktop application from **Microsoft Store** using your **Microsoft Account** to sign in. +
![Power BI Desktop](images/power-bi-desktop.png)
+ +1. Open **Power BI** desktop. + +1. Click on **Home**, **Get data** and **More...** +![Power BI desktop](images/bi-getdata-more.png) + +1. Search for **Azure Databricks** and click **Connect**. +![Search Azure Databricks](images/bi-search-databricks.png) + +1. Fill in the details and click **OK**. + +1. Connect to the RTDIP data using your **Databricks SQL Warehouse** connection details including Hostname and HTTP Path. For **Data Connectivity mode**, select **DirectQuery**. + +1. Click **Azure Active Directory**, **Sign In** and select **Connect**. In **Power Query Editor**, there are different tables for different data types. +![Power BI Azure Databricks](images/bi-azure-signin.png) + +1. Once connected to the Databricks SQL Warehouse, navigate to the Business Unit in the navigator bar on the left and select the asset tables for the data you wish to use in your report. There is functionality to select multiple tables if required. Click **Load** to get the queried data. \ No newline at end of file diff --git a/docs/releases/images/cicd.png b/docs/releases/images/cicd.png new file mode 100644 index 000000000..c7f32de4f Binary files /dev/null and b/docs/releases/images/cicd.png differ diff --git a/docs/releases/releases.md b/docs/releases/releases.md new file mode 100644 index 000000000..610e5def3 --- /dev/null +++ b/docs/releases/releases.md @@ -0,0 +1,6 @@ +# Releases +RTDIP – Releases + +![cicd](images/cicd.png) + +Please visit the RTDIP Github Repository to view the latest [releases and changelogs.](https://github.com/rtdip/core/releases) diff --git a/docs/roadmap/images/roadmap-overview.jpeg b/docs/roadmap/images/roadmap-overview.jpeg new file mode 100644 index 000000000..acd6a10f5 Binary files /dev/null and b/docs/roadmap/images/roadmap-overview.jpeg differ diff --git a/docs/roadmap/roadmap-overview.md b/docs/roadmap/roadmap-overview.md new file mode 100644 index 000000000..59d43ecf4 --- /dev/null +++ b/docs/roadmap/roadmap-overview.md @@ -0,0 +1,7 @@ +# RTDIP Roadmap + +![roadmap overview](images/roadmap-overview.jpeg) + +This section provides periodical updates of what the RTDIP team will be working on over the short and long term. The team will provide updates at important periods of the year so that users of the platform can understand new capabilities and options coming to the platform. + +We welcome and encourage projects, developers, users and applications to contribute to our roadmaps. Please reach out to the RTDIP team if you have ideas or suggestions on what we could work on next. diff --git a/docs/roadmap/yearly-roadmaps/2022-development-roadmap.md b/docs/roadmap/yearly-roadmaps/2022-development-roadmap.md new file mode 100644 index 000000000..bc7340eff --- /dev/null +++ b/docs/roadmap/yearly-roadmaps/2022-development-roadmap.md @@ -0,0 +1,93 @@ +# RTDIP Development Roadmap in 2022 + +![roadmap](images/roadmap.png) + + +Defining a list of development items for RTDIP is always difficult because so much can change within Digital Technologies in 12 months. However, as we head towards the end of the year of 2021, we have outlined themes of what RTDIP will look at in the Development and Innovation space in 2022. + +We welcome and encourage projects, developers, users and applications to contribute to our roadmaps. Please reach out to the RTDIP Technical Steering Committee team if you have ideas or suggestions on what we could innovate on in this space. We will continue to evolve these development items all through 2022 so please come back to this article throughout the year to see any new items that may be brought into scope. + +!!! note "Note" + Development and Innovation items don't always make it to Production. + +## TL;DR + +A brief summary of development and innovation items planned for 2022. + +| Item | Description | Estimated Quarter for Delivery | +|------|-------------|--------------------------------| +| Power BI | Enable querying of RTDIP data via Power BI. While some work has started on this in 2021, this item explores rolling it out further and how users can combine RTDIP data with other data sources | Q1 2022 | +| Seeq Connector | Enable querying of RTDIP data via Seeq. Scope is limited to simply querying RTDIP data, we may look at what else is possible with the connector once the base capability has been achieved | Q1 2022 | +| Delta Live Tables | Leverage [Delta Live Tables](https://docs.microsoft.com/en-us/azure/databricks/data-engineering/delta-live-tables/) for ingestion of RTDIP data into Delta Format. Provides better processing, merging, data cleansing and monitoring capabilities to the RTDIP Delta Ingestion Pipelines | Q1-Q2 2022 | +| Multicloud | Build certain existing RTDIP Azure capabilities on AWS. Enables RTDIP in the clouds aligned with the business but also to ensure multicloud is cost effective and that products in the architecture work in Cloud Environments | Q1-Q3 2022 | +| SDK | An open source python SDK is developed to assist users with a simple python library for connecting, authenticating and querying RTDIP data | Q1-Q4 2022 | +| REST API | Wrap the python SDK in a REST API to allow non Python users to get similar functionality to the python SDK | Q1-Q4 2022 | +| Unity Catalog | Provides a multi-region Catalog of all data in RTDIP. Enables easier navigation and exploration of what datasets are available in RTDIP | Q3 2022 | +| Delta Sharing | Enables sharing of Delta data via a managed service that handles security, authentication and delivery of data. Particularly useful for sharing RTDIP data with third parties | Q4 2022 | + +## Power BI + +![power-bi](images/power-bi.png) + +Power BI is a popular tool amongst RTDIP End Users for querying and plotting RTDIP data. The use of Delta and Databricks SQL Warehouses in the RTDIP Platform brings native Power BI integration using connectors already available in Power BI versions after May 2021. + +The aim is to enable Power BI connectivity to RTDIP so that users can query their data by the end of Q1 2022. + +## Seeq + +![seeq](images/seeq.png) + +Similar to Power BI, Seeq is a popular tool amongst real time users to query and manipulate RTDIP data. Seeq and RTDIP are currently working on a connector that allows Seeq to query RTDIP data via the same Databricks SQL Warehouse that Power BI will use for querying data by the end of Q1 2022. + +## Delta Live Tables + +![delta-live-tables](images/delta-live-tables.png) + +For more information about the advantages of Delta Live Tables, please see this [link](https://docs.microsoft.com/en-us/azure/databricks/data-engineering/delta-live-tables/) and if you would like to see Bryce Bartmann, RTDIP team member, talking about Delta Live Tables at the Data & AI Summit 2021, please see the session [here](https://databricks.com/session_na21/make-reliable-etl-easy-on-delta-lake). + +RTDIP has been converting it's data to the open source format [Delta](https://delta.io/) using standard PySpark structured streaming jobs. Whilst this has been working well for converting RTDIP data to Delta, Delta Live Tables from Databricks provides similar capabilities as standard spark code, but with additional benefits: + +- Expectations: Allows developers to specify data cleansing rules on ingested data. This can assist to provide higher quality, more reliable data to users +- Data Flows: Visually describes the flow of the data through the data pipelines from source to target, including data schemas and record counts +- Maintenance: Delta Live Tables simplifies maintenance tasks required for Delta Tables by scheduling them automatically based on the deployment of the Delta Live Tables Job +- Monitoring: Delta Live Tables are easier to monitor as their graphical schematics help non-technical people to understand the status of the ingestion pipelines + +The RTDIP Team has actively worked with Databricks to build Delta Live Tables. Whilst the product is well understood, certain features like merging data needed to be made available before RTDIP could fully migrate existing spark jobs to Delta Live Tables. Databricks intend to provide the Merge function in late Q4 2021 which will then trigger this piece of work with a target of having a decision point to move to production in Q1 2022. + +## Multicloud + +![multicloud](images/multicloud.jpeg) + +As clouds mature, one of the most asked questions is how customers can leverage more than one cloud to provide a better and more economical solution to their customers. Even though this is a fairly new area to explore, there are a number of cloud agnostic technologies that are trying to help customers take advantage of and manage environments in more than one cloud. + +Multicloud design can be complex and requires significant analysis of existing technologies capabilities and how they translate into benefits for RTDIP customers. Databricks will be one good example of exploring their new multicloud environment management tool and how this could benefit businesses in the long run. We expect there to be more technologies that come out with multicloud capabilities throughout 2022 and we will continue to explore, test and understand how RTDIP can leverage these throughout 2022 and beyond. + +## SDK and REST API + +![sdk-vs-api](images/sdk-vs-api.png) + +A common theme we are seeing amongst applications and users of RTDIP data is a simple way to authenticate, query and manipulate RTDIP data. In an effort to also build a stronger developer community around RTDIP, we will be building a python SDK that python users can use in their code for performing common functions with RTDIP Data: + +- Authenticating with RTDIP +- Connecting to RTDIP data +- Querying RTDIP raw data +- Performing sampling on raw data +- Performing interpolation on sampled data + +We plan to deliver the first version of the python SDK early in 2022 and welcome all python developers to contribute to the repository. + +For non python users, we plan to wrap the SDK in a REST API. This facilitates a language agnostic way of benefitting from all the development of the python SDK. These REST APIs will be rolled out in line with functionality built with the python SDK. + +## Unity Catalog + +![unity-catalog](images/delta-unity-catalog.png) + +Cataloging data is a common activity when building data lakes that contain data from multiple sources and from multiple geographic regions. RTDIP will explore and deploy a catalog of all data sources currently being ingested into the platform. + +## Delta Sharing + +![delta-sharing](images/delta-sharing.png) + +One of the most common requests the RTDIP team receive is how to share RTDIP data with third parties. Delta Sharing is an open source capability that allows sharing of Delta data via a managed service that provides authentication, connection management and supply of Delta data to third parties. + +We aim to see what more we can do in this space to make sharing of data simpler from an architecture perspective while still meeting all the security requirements around sharing of data with third parties. diff --git a/docs/roadmap/yearly-roadmaps/images/delta-live-tables.png b/docs/roadmap/yearly-roadmaps/images/delta-live-tables.png new file mode 100644 index 000000000..3258ed665 Binary files /dev/null and b/docs/roadmap/yearly-roadmaps/images/delta-live-tables.png differ diff --git a/docs/roadmap/yearly-roadmaps/images/delta-sharing.png b/docs/roadmap/yearly-roadmaps/images/delta-sharing.png new file mode 100644 index 000000000..8eece62e9 Binary files /dev/null and b/docs/roadmap/yearly-roadmaps/images/delta-sharing.png differ diff --git a/docs/roadmap/yearly-roadmaps/images/delta-unity-catalog.png b/docs/roadmap/yearly-roadmaps/images/delta-unity-catalog.png new file mode 100644 index 000000000..84b47e558 Binary files /dev/null and b/docs/roadmap/yearly-roadmaps/images/delta-unity-catalog.png differ diff --git a/docs/roadmap/yearly-roadmaps/images/multicloud.jpeg b/docs/roadmap/yearly-roadmaps/images/multicloud.jpeg new file mode 100644 index 000000000..610ca2e14 Binary files /dev/null and b/docs/roadmap/yearly-roadmaps/images/multicloud.jpeg differ diff --git a/docs/roadmap/yearly-roadmaps/images/power-bi.png b/docs/roadmap/yearly-roadmaps/images/power-bi.png new file mode 100644 index 000000000..db1809242 Binary files /dev/null and b/docs/roadmap/yearly-roadmaps/images/power-bi.png differ diff --git a/docs/roadmap/yearly-roadmaps/images/roadmap.png b/docs/roadmap/yearly-roadmaps/images/roadmap.png new file mode 100644 index 000000000..4d0ddb068 Binary files /dev/null and b/docs/roadmap/yearly-roadmaps/images/roadmap.png differ diff --git a/docs/roadmap/yearly-roadmaps/images/sdk-vs-api.png b/docs/roadmap/yearly-roadmaps/images/sdk-vs-api.png new file mode 100644 index 000000000..d3a74a6c5 Binary files /dev/null and b/docs/roadmap/yearly-roadmaps/images/sdk-vs-api.png differ diff --git a/docs/roadmap/yearly-roadmaps/images/seeq.png b/docs/roadmap/yearly-roadmaps/images/seeq.png new file mode 100644 index 000000000..4ef2f6d49 Binary files /dev/null and b/docs/roadmap/yearly-roadmaps/images/seeq.png differ diff --git a/docs/sdk/code-reference/authenticate.md b/docs/sdk/code-reference/authenticate.md new file mode 100644 index 000000000..f9cb85710 --- /dev/null +++ b/docs/sdk/code-reference/authenticate.md @@ -0,0 +1,2 @@ +# Authentication +::: src.sdk.python.rtdip_sdk.authentication.authenticate diff --git a/docs/sdk/code-reference/db-sql-connector.md b/docs/sdk/code-reference/db-sql-connector.md new file mode 100644 index 000000000..1818f8807 --- /dev/null +++ b/docs/sdk/code-reference/db-sql-connector.md @@ -0,0 +1,2 @@ +# Databricks SQL Connector +::: src.sdk.python.rtdip_sdk.odbc.db_sql_connector diff --git a/docs/sdk/code-reference/interpolate.md b/docs/sdk/code-reference/interpolate.md new file mode 100644 index 000000000..80c2f85b0 --- /dev/null +++ b/docs/sdk/code-reference/interpolate.md @@ -0,0 +1,36 @@ +# Interpolate Function +::: src.sdk.python.rtdip_sdk.functions.interpolate + +## Example +```python +from rtdip_sdk.authentication.authenticate import DefaultAuth +from rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from rtdip_sdk.functions import interpolate + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +dict = { + "business_unit": "Business Unit", + "region": "Region", + "asset": "Asset Name", + "data_security_level": "Security Level", + "data_type": "float", #options:["float", "double", "integer", "string"] + "tag_names": ["tag_1", "tag_2"], #list of tags + "start_date": "01-01-2022", #start_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" + "end_date": "31-01-2022", #end_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" + "sample_rate": "1", #numeric input + "sample_unit": "hour", #options: ["second", "minute", "day", "hour"] + "agg_method": "first", #options: ["first", "last", "avg", "min", "max"] + "interpolation_method": "forward_fill", #options: ["forward_fill", "backward_fill"] + "include_bad_data": True, #options: [True, False] +} +x = interpolate.get(connection, dict) +print(x) +``` + +This example is using [```DefaultAuth()```](authenticate.md) and [```DatabricksSQLConnection()```](db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](authenticate.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](pyodbc-sql-connector.md) or [```TURBODBCSQLConnection()```](turbodbc-sql-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../sqlwarehouses/sql-warehouses.md).
diff --git a/docs/sdk/code-reference/metadata.md b/docs/sdk/code-reference/metadata.md new file mode 100644 index 000000000..97ec50b5e --- /dev/null +++ b/docs/sdk/code-reference/metadata.md @@ -0,0 +1,28 @@ +# Metadata Function +::: src.sdk.python.rtdip_sdk.functions.metadata + +## Example +```python +from rtdip_sdk.authentication.authenticate import DefaultAuth +from rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from rtdip_sdk.functions import metadata + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +dict = { + "business_unit": "Business Unit", + "region": "Region", + "asset": "Asset Name", + "data_security_level": "Security Level", + "tag_names": ["tag_1", "tag_2"], #list of tags +} +x = metadata.get(connection, dict) +print(x) +``` + +This example is using [```DefaultAuth()```](authenticate.md) and [```DatabricksSQLConnection()```](db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](authenticate.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](pyodbc-sql-connector.md) or [```TURBODBCSQLConnection()```](turbodbc-sql-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../sqlwarehouses/sql-warehouses.md).
\ No newline at end of file diff --git a/docs/sdk/code-reference/pyodbc-sql-connector.md b/docs/sdk/code-reference/pyodbc-sql-connector.md new file mode 100644 index 000000000..b69a43686 --- /dev/null +++ b/docs/sdk/code-reference/pyodbc-sql-connector.md @@ -0,0 +1,14 @@ +# PYODBC SQL Connector + +## PYODBC Driver Paths + +To use PYODBC SQL Connect you will require the driver path specified below per operating system. + +| Operating Systems | Driver Paths | +|--------|--------------| +| Windows | `C:\Program Files\Simba Spark ODBC Driver` | +| MacOS | `/Library/simba/spark/lib/libsparkodbc_sbu.dylib` | +| Linux 64-bit | `/opt/simba/spark/lib/64/libsparkodbc_sb64.so` | +| Linux 32-bit | `/opt/simba/spark/lib/32/libsparkodbc_sb32.so`| + +::: src.sdk.python.rtdip_sdk.odbc.pyodbc_sql_connector \ No newline at end of file diff --git a/docs/sdk/code-reference/raw.md b/docs/sdk/code-reference/raw.md new file mode 100644 index 000000000..d16ba6a93 --- /dev/null +++ b/docs/sdk/code-reference/raw.md @@ -0,0 +1,32 @@ +# Raw Function +::: src.sdk.python.rtdip_sdk.functions.raw + +## Example +```python +from rtdip_sdk.authentication.authenticate import DefaultAuth +from rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from rtdip_sdk.functions import raw + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +dict = { +"business_unit": "Business Unit", +"region": "Region", +"asset": "Asset Name", +"data_security_level": "Security Level", +"data_type": "float", #options:["float", "double", "integer", "string"] +"tag_names": ["tag_1", "tag_2"], #list of tags +"start_date": "01-01-2022", #start_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" +"end_date": "31-01-2022", #end_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" +"include_bad_data": True, #options: [True, False] +} +x = raw.get(connection, dict) +print(x) +``` + +This example is using [```DefaultAuth()```](authenticate.md) and [```DatabricksSQLConnection()```](db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](authenticate.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](pyodbc-sql-connector.md) or [```TURBODBCSQLConnection()```](turbodbc-sql-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../sqlwarehouses/sql-warehouses.md).
\ No newline at end of file diff --git a/docs/sdk/code-reference/resample.md b/docs/sdk/code-reference/resample.md new file mode 100644 index 000000000..fdf62c4d7 --- /dev/null +++ b/docs/sdk/code-reference/resample.md @@ -0,0 +1,35 @@ +# Resample Function +::: src.sdk.python.rtdip_sdk.functions.resample + +## Example +```python +from rtdip_sdk.authentication.authenticate import DefaultAuth +from rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from rtdip_sdk.functions import resample + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +dict = { + "business_unit": "Business Unit", + "region": "Region", + "asset": "Asset Name", + "data_security_level": "Security Level", + "data_type": "float", #options:["float", "double", "integer", "string"] + "tag_names": ["tag_1", "tag_2"], #list of tags + "start_date": "01-01-2022", #start_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" + "end_date": "31-01-2022", #end_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" + "sample_rate": "1", #numeric input + "sample_unit": "hour", #options: ["second", "minute", "day", "hour"] + "agg_method": "first", #options: ["first", "last", "avg", "min", "max"] + "include_bad_data": True, #options: [True, False] +} +x = resample.get(connection, dict) +print(x) +``` + +This example is using [```DefaultAuth()```](authenticate.md) and [```DatabricksSQLConnection()```](db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](authenticate.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](pyodbc-sql-connector.md) or [```TURBODBCSQLConnection()```](turbodbc-sql-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../sqlwarehouses/sql-warehouses.md).
\ No newline at end of file diff --git a/docs/sdk/code-reference/time-weighted-average.md b/docs/sdk/code-reference/time-weighted-average.md new file mode 100644 index 000000000..d3fdd719c --- /dev/null +++ b/docs/sdk/code-reference/time-weighted-average.md @@ -0,0 +1,36 @@ +# Time Weighted Average +::: src.sdk.python.rtdip_sdk.functions.time_weighted_average + +## Example + +```python +from rtdip_sdk.authentication.authenticate import DefaultAuth +from rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from rtdip_sdk.functions import time_weighted_average + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +dict = { +"business_unit": "Business Unit", +"region": "Region", +"asset": "Asset Name", +"data_security_level": "Security Level", +"data_type": "float", #options:["float", "double", "integer", "string"] +"tag_names": ["tag_1", "tag_2"], #list of tags +"start_date": "01-01-2022", #start_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" +"end_date": "31-01-2022", #end_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" +"window_size_mins": 15, #numeric input +"window_length": 20, #numeric input +"include_bad_data": True, #options: [True, False] +"step": True +} +x = time_weighted_average.get(connection, dict) +print(x) +``` + +This example is using [```DefaultAuth()```](authenticate.md) and [```DatabricksSQLConnection()```](db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](authenticate.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](pyodbc-sql-connector.md) or [```TURBODBCSQLConnection()```](turbodbc-sql-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../sqlwarehouses/sql-warehouses.md).
\ No newline at end of file diff --git a/docs/sdk/code-reference/turbodbc-sql-connector.md b/docs/sdk/code-reference/turbodbc-sql-connector.md new file mode 100644 index 000000000..4a39fa6f2 --- /dev/null +++ b/docs/sdk/code-reference/turbodbc-sql-connector.md @@ -0,0 +1,2 @@ +# TURBODBC SQL Connector +::: src.sdk.python.rtdip_sdk.odbc.turbodbc_sql_connector diff --git a/docs/sdk/images/sdk-logo.png b/docs/sdk/images/sdk-logo.png new file mode 100644 index 000000000..0c18b80c5 Binary files /dev/null and b/docs/sdk/images/sdk-logo.png differ diff --git a/docs/sdk/overview.md b/docs/sdk/overview.md new file mode 100644 index 000000000..a8cd7502c --- /dev/null +++ b/docs/sdk/overview.md @@ -0,0 +1,60 @@ +
![sdk](images/sdk-logo.png){width=50%}
+ +# What is RTDIP SDK? + +​​**Real Time Data Ingestion Platform (RTDIP) SDK** is a software development kit built to easily access some of RTDIP's transformation functions. + +The RTDIP SDK will give the end user the power to use some of the convenience methods for frequency conversions and resampling of Pi data all through a self-service platform. RTDIP is offering a flexible product with the ability to authenticate and connect to Databricks SQL Warehouses given the end users preferences. RTDIP have taken the initiative to cut out the middle man and instead wrap these commonly requested methods in a simple python module so that you can instead focus on the data. + +See [RTDIP SDK Usage](rtdip-sdk-usage.md) for more information on how to use the SDK. + + +## Authenticators + +The RTDIP SDK includes several authentication methods to cater to the preference of the user. See below: + +* [Default Authentication](code-reference/authenticate.md) - For authenticating users with Azure AD using the [azure-identity](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python) package. Note the order that Default Authentication uses to sign in a user and how it does it in this [documentation](https://azuresdkdocs.blob.core.windows.net/$web/python/azure-identity/1.10.0b1/index.html). From experience, the Visual Studio Code login is the easiest to setup, but the azure cli option is the most reliable option. This [page](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/identity/azure-identity/TROUBLESHOOTING.md#troubleshooting-default-azure-credential-authentication-issues) is useful for troubleshooting issues with this option to authenticate. + +!!! note "Visual Studio Code" + As per the guidance in the documentation - **To authenticate in Visual Studio Code, ensure version 0.9.11 or earlier of the Azure Account extension is installed. To track progress toward supporting newer extension versions, see [this GitHub issue](https://github.com/Azure/azure-sdk-for-net/issues/27263). Once installed, open the Command Palette and run the Azure: Sign In command**. + + +* [Certificate Authentication](code-reference/authenticate.md) - Service Principal authentication using a certificate + +* [Client Secret Authentication](code-reference/authenticate.md) - Service Principal authentication using a client id and client secret + +!!! note "Note" + RTDIP are continuously adding more to this list so check back regularly!
+ +## SQL Warehouses + +The RTDIP SDK uses Databricks SQL Warehouses to provide access to RTDIP data. See below for different endpoint options depending on the region required: + +* [SQL Warehouses](sqlwarehouses/sql-warehouses.md) + +## Connectors + +Once the user is authenticated, they can connect to Databricks SQL warehouses using one of the following methods: + +* [Databricks SQL Connector](code-reference/db-sql-connector.md) + +* [PYODBC SQL Connector](code-reference/pyodbc-sql-connector.md) + +* [TURBODBC SQL Connector](code-reference/turbodbc-sql-connector.md) + +## Functions + +Finally, once the user is authenticated and connected to Databricks SQL Warehouses they have access the following functions: + +* [Resample](code-reference/resample.md) + +* [Interpolate](code-reference/interpolate.md) + +* [Raw](code-reference/raw.md) + +* [Time Weighted Averages](code-reference/time-weighted-average.md) + +* [Metadata](code-reference/metadata.md) + +!!! note "Note" + RTDIP are continuously adding more to this list so check back regularly!
diff --git a/docs/sdk/rtdip-sdk-usage.md b/docs/sdk/rtdip-sdk-usage.md new file mode 100644 index 000000000..64c2762dd --- /dev/null +++ b/docs/sdk/rtdip-sdk-usage.md @@ -0,0 +1,186 @@ +# Getting started with using the RTDIP SDK + +This article provides a guide on how to use RTDIP SDK. Before you get started, ensure you have installed the [RTDIP Python Package](https://pypi.org/project/rtdip-sdk/) and check the [RTDIP Installation Page](../getting-started/installation.md) for all the required prerequisites. + + +## How to use the SDK + +The RTDIP SDK has many functionalities, such as allowing the user to authenticate, connect and/or use the most commonly requested methods. + +### Authenticate + +!!! note "Note" + If you are using the SDK on databricks please note that DefaultAuth will not work.
+ +1\. Import **rtdip-sdk** authentication methods with the following: + + from rtdip_sdk.authentication import authenticate as auth + +2\. Use any of the following authentication methods. Replace **tenant_id** , **client_id**, **certificate_path** or **client_secret** with your own details. + +#### Default Authentication + DefaultAzureCredential = auth.DefaultAuth().authenticate() + +#### Certificate Authentication + CertificateCredential = auth.CertificateAuth(tenant_id: str, client_id: str, certificate_path: str).authenticate() + +#### Client Secret Authentication + ClientSecretCredential = auth.ClientSecretAuth(tenant_id: str, client_id: str, client_secret: str).authenticate() + +3\. The methods above will return back a Client Object. The following example will show you how to retrieve the access_token from a credential object. The access token will be used in later steps to connect to RTDIP via the three options (Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect). + + access_token = DefaultAzureCredential.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + +### Sample Authentication Code: + +```python +from rtdip_sdk.authentication import authenticate as auth + +authentication = auth.DefaultAuth().authenticate() +access_token = authentication.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +``` + +For more information about each of the authentication methods, see [Code Reference](code-reference/authenticate.md) and navigate to the required section. + +!!! note "Note" + If you are experiencing any trouble authenticating please see [Troubleshooting - Authentication](troubleshooting.md)
+ +### Connect to RTDIP + +1\. The RTDIP SDK offers several ways to connect to a Databricks SQL Warehouse: + +- Databricks SQL Connect: The simplest method to connect to RTDIP and does not require any additional installation steps. +- PYODBC SQL Connect: A popular library that python developers use for ODBC connectivity but requires more setup steps. +- TURBODBC SQL Connect: The RTDIP development team have found this to be the most performant method of connecting to RTDIP leveraging the arrow implementation within Turbodbc to obtain data, but requires a number of addditional installation steps to get working on OSX, Linux and Windows. + + +2\. Replace **server_hostname**, **http_path** and **access_token** with your own details. + +#### Databricks SQL Connect + +```python +from rtdip_sdk.authentication import authenticate as auth +from rtdip_sdk.odbc import db_sql_connector as d + +server_hostname = "server_hostname" +http_path = "http_path" + +authentication = auth.DefaultAuth().authenticate() +access_token = authentication.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + +connection = d.DatabricksSQLConnection(server_hostname: str, http_path: str, access_token: str) +``` + +#### PYODBC SQL Connect + +* [ODBC](https://databricks.com/spark/odbc-drivers-download) or [JDBC](https://databricks.com/spark/jdbc-drivers-download) are required to leverage PYODBC. Follow these [instructions](https://docs.databricks.com/integrations/jdbc-odbc-bi.html) to install the drivers in your environment. + +* Microsoft Visual C++ 14.0 or greater is required. Get it from [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) + +* Driver paths can be found on [PYODBC Driver Paths](code-reference/pyodbc-sql-connector.md) + +```python +from rtdip_sdk.authentication import authenticate as auth +from rtdip_sdk.odbc import pyodbc_sql_connector as p + +server_hostname = "server_hostname" +http_path = "http_path" +driver_path = "/Library/simba/spark/lib/libsparkodbc_sbu.dylib" + +authentication = auth.DefaultAuth().authenticate() +access_token = authentication.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + +connection = p.PYODBCSQLConnection(driver_path: str, sever_hostname: str, http_path: str, access_token: str) +``` + +#### TURBODBC SQL Connect + +* [ODBC](https://databricks.com/spark/odbc-drivers-download) or [JDBC](https://databricks.com/spark/jdbc-drivers-download) are required to leverage TURBODBC. Follow these [instructions](https://docs.databricks.com/integrations/jdbc-odbc-bi.html) to install the drivers in your environment. +* [Boost](https://turbodbc.readthedocs.io/en/latest/pages/getting_started.html) needs to be installed locally to use the [TURBODBC SQL Connector](code-reference/turbodbc-sql-connector.md) (Optional) + +```python +from rtdip_sdk.authentication import authenticate as auth +from rtdip_sdk.odbc import turbodbc_sql_connector as t + +server_hostname = "server_hostname" +http_path = "http_path" + +authentication = auth.DefaultAuth().authenticate() +access_token = authentication.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + +connection = t.TURBODBCSQLConnection(server_hostname: str, http_path: str, access_token: str) +``` + +For more information about each of the connection methods, please see [Code Reference](code-reference/db-sql-connector.md) and navigate to the required section. + +### Functions + +Finally, after authenticating and connecting using one of the methods above, you have access to the commonly requested RTDIP functions such as **Resample**, **Interpolate**, **Raw**, **Time Weighted Averages** or **Metadata**. + +1\. To use any of the RTDIP functions, use the commands below. + +```python +from rtdip_sdk.functions import resample +from rtdip_sdk.functions import interpolate +from rtdip_sdk.functions import raw +from rtdip_sdk.functions import time_weighted_average +from rtdip_sdk.functions import metadata +``` + +2\. From functions you can use any of the following methods. + +#### Resample + resample.get(connection, parameters_dict) + +#### Interpolate + interpolate.get(connection, parameters_dict) + +#### Raw + raw.get(connection, parameters_dict) + +#### Time Weighted Average + time_weighted_average.get(connection, parameter_dict) + +#### Metadata + metadata.get(connection, parameter_dict) + +For more information about the function parameters see [Code Reference](code-reference/resample.md) and navigate through the required function. + +### Example + +This is a code example of the RTDIP SDK Interpolate function. You will need to replace the parameters with your own requirements and details. If you are unsure on the options please see [Code Reference - Interpolate](code-reference/interpolate.md) and navigate to the attributes section. + +```python +from rtdip_sdk.authentication import authenticate as auth +from rtdip_sdk.odbc import db_sql_connector as dbc +from rtdip_sdk.functions import interpolate + +authentication = auth.DefaultAuth().authenticate() +access_token = authentication.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = dbc.DatabricksSQLConnection("{server_hostname}", "{http_path}", access_token) + +dict = { + "business_unit": "{business_unit}", + "region": "{region}", + "asset": "{asset}", + "data_security_level": "{date_security_level}", + "data_type": "{data_type}", #options are float, integer, string and double (the majority of data is float) + "tag_names": ["{tag_name_1}, {tag_name_2}"], + "start_date": "2022-03-08", #start_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" + "end_date": "2022-03-10", #end_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" + "sample_rate": "1", #numeric input + "sample_unit": "hour", #options are second, minute, day, hour + "agg_method": "first", #options are first, last, avg, min, max + "interpolation_method": "forward_fill", #options are forward_fill or backward_fill + "include_bad_data": True #boolean options are True or False +} + +result = interpolate.get(connection, dict) +print(result) +``` +!!! note "Note" + If you are having problems please see [Troubleshooting](troubleshooting.md) for more information.
+ +### Conclusion + +Congratulations! You have now learnt how to use the RTDIP SDK. Please check back for regular updates and if you would like to contribute, you can open an issue on GitHub. See the [Contributing Guide](https://github.com/rtdip/core/blob/develop/CONTRIBUTING.md) for more help. \ No newline at end of file diff --git a/docs/sdk/sqlwarehouses/sql-warehouses.md b/docs/sdk/sqlwarehouses/sql-warehouses.md new file mode 100644 index 000000000..ad33d2378 --- /dev/null +++ b/docs/sdk/sqlwarehouses/sql-warehouses.md @@ -0,0 +1,9 @@ +# SQL Warehouses + +In order to connect to the data using the RTDIP SDK you will require Databricks SQL Warehouse information. Retrieve this information from your Databricks Workspace by following the steps below: + +1. Login to your Databricks Workspace +1. Switch to the SQL Option in the Workspace +1. Select the SQL Warehouse +1. Click on the Details tab +1. Copy the Host Name and HTTP Path details \ No newline at end of file diff --git a/docs/sdk/troubleshooting.md b/docs/sdk/troubleshooting.md new file mode 100644 index 000000000..1691b6c1c --- /dev/null +++ b/docs/sdk/troubleshooting.md @@ -0,0 +1,62 @@ +# Troubleshooting + +### Cannot install pyodbc + +Microsoft Visual C++ 14.0 or greater is required to install pyodbc. Get it with [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) + +### Cannot build wheels (Using legacy setup.py) + +To install rtdip-sdk using setup.py, you need to have **wheel** installed using the following command: + + pip install wheel + +### Authentication + +For Default Credential authentication, a number of troublshooting options are available [here](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/identity/azure-identity/TROUBLESHOOTING.md#troubleshooting-default-azure-credential-authentication-issues). + +For Visual Studio Code errors, the version of Azure Account extension is installed(0.9.11) - **To authenticate in Visual Studio Code, ensure version 0.9.11 or earlier of the Azure Account extension is installed. To track progress toward supporting newer extension versions, see [this GitHub issue](https://github.com/Azure/azure-sdk-for-net/issues/27263). Once installed, open the Command Palette and run the Azure: Sign In command** + +### Exception has occured: TypeError 'module' object is not callable + +Ensure you are importing and using the RTDIP SDK functions correctly. You will need to give the module a name and reference it when using the function. See below for a code example. + +```python +from rtdip_sdk.authentication import authenticate as auth +from rtdip_sdk.odbc import db_sql_connector as dbc +from rtdip_sdk.functions import interpolate + +authentication = auth.DefaultAuth().authenticate() +access_token = authentication.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = dbc.DatabricksSQLConnection("{server_hostname}", "{http_path}", access_token) + +dict = { + "business_unit": "{business_unit}", + "region": "{region}", + "asset": "{asset}", + "data_security_level": "{date_security_level}", + "data_type": "{data_type}", #options are float, integer, string and double (the majority of data is float) + "tag_names": ["{tag_name_1}, {tag_name_2}"], + "start_date": "2022-03-08", #start_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" + "end_date": "2022-03-10", #end_date can be a date in the format "YYYY-MM-DD" or a datetime in the format "YYYY-MM-DDTHH:MM:SS" + "sample_rate": "1", #numeric input + "sample_unit": "hour", #options are second, minute, day, hour + "agg_method": "first", #options are first, last, avg, min, max + "interpolation_method": "forward_fill", #options are forward_fill or backward_fill + "include_bad_data": True #boolean options are True or False +} + +result = interpolate.get(connection, dict) +print(result) +``` + +### Databricks ODBC/JDBC Driver issues + +#### General Troubleshooting + +Most issues related to the installation or performance of the ODBC/JDBC driver are documented [here](https://docs.microsoft.com/en-us/azure/databricks/kb/bi/jdbc-odbc-troubleshooting) + +#### ODBC with a proxy + +Follow this document to use the ODBC driver with a [proxy](https://docs.microsoft.com/en-us/azure/databricks/kb/bi/configure-simba-proxy-windows). + + diff --git a/environment.yml b/environment.yml new file mode 100644 index 000000000..23962a6c7 --- /dev/null +++ b/environment.yml @@ -0,0 +1,44 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: rtdip-sdk +channels: + - conda-forge + - defaults +dependencies: + - python==3.8.8 + - mkdocs-material==8.2.4 + - mkdocs-material-extensions==1.0.3 + - jinja2==3.0.3 + - jinjasql==0.1.8 + - pytest==7.2.0 + - pytest-mock==3.10.0 + - pytest-cov==4.0.0 + - pip==22.3.0 + - turbodbc==4.5.5 + - numpy>=1.22.0 + - pandas==1.4.2 + - oauthlib>=3.2.1 + - databricks-sql-connector==2.1.0 + - azure-identity==1.11.0 + - pyodbc==4.0.34 + - mkdocstrings==0.19.0 + - fastapi==0.85.1 + - pygments>=2.10,<2.12 + - pymdown-extensions<9.3 + - pip: + - azure-functions==1.12.0 + - strawberry-graphql[fastapi]==0.138.0 + - nest_asyncio==1.5.6 \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..5a8fd88ad --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,130 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Project information +site_name: Real Time Data Ingestion Platform +site_url: https://github.com/rtdip/core/ +site_author: Real Time Data Ingestion Platform +site_description: >- + Easy access to high volume, historical and real time process data + for analytics applications, engineers, and data scientists wherever they are. + +# Repository +repo_name: rtdip/core +repo_url: https://github.com/rtdip/core/ +edit_uri: "" + +docs_dir: docs + +# Custom Colour Pallete +extra_css: + - assets/extra.css + +# Configuration +theme: + name: material + # Default values, taken from mkdocs_theme.yml + language: en + features: + - content.code.annotate + - content.tabs.link + - content.tooltips + - navigation.expand + - navigation.indexes + - navigation.instant + - navigation.sections + - navigation.tabs + # - navigation.tabs.sticky + - navigation.top + # - navigation.tracking + - search.highlight + - search.share + - search.suggest + palette: + - scheme: default + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to light mode + font: + text: Roboto + code: Roboto Mono + favicon: assets/favicon.png + logo: assets/logo_lfe.png + +extra: + generator: false + +plugins: + - search + - autorefs + - mkdocstrings + +markdown_extensions: + - attr_list + - meta + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + +# Page tree +nav: + - Home: index.md + - Getting started: + - About RTDIP: getting-started/about-rtdip.md + - Why RTDIP: getting-started/why-rtdip.md + - Installation: + - RTDIP SDK: getting-started/installation.md + - Integration: + - Power BI: integration/power-bi.md + - SDK: + - Overview: sdk/overview.md + - Getting Started: sdk/rtdip-sdk-usage.md + - SQL Warehouses: sdk/sqlwarehouses/sql-warehouses.md + - Troubleshooting: sdk/troubleshooting.md + - Code Reference: + - Connectors: + - Databricks SQL Connector: sdk/code-reference/db-sql-connector.md + - PYODBC SQL Connector: sdk/code-reference/pyodbc-sql-connector.md + - TURBODBC SQL Connector: sdk/code-reference/turbodbc-sql-connector.md + - Authenticators: + - Authentication: sdk/code-reference/authenticate.md + - Functions: + - Resample: sdk/code-reference/resample.md + - Interpolate: sdk/code-reference/interpolate.md + - Raw: sdk/code-reference/raw.md + - Time Weighted Average: sdk/code-reference/time-weighted-average.md + - Metadata: sdk/code-reference/metadata.md + - API: + - Overview: api/overview.md + - Authentication: api/authentication.md + - Examples: api/examples.md + - REST API Documentation: api/rest_apis.md + - Deployment: + - Azure: api/deployment/azure.md + - Roadmaps: + - Overview: roadmap/roadmap-overview.md + - 2022 Development Roadmap: roadmap/yearly-roadmaps/2022-development-roadmap.md + - Releases: + - Overview: releases/releases.md + - Blog: + - Overview: blog/overview.md + - Posts: + - Delta Lakehouse and RTDIP: blog/delta_and_rtdip.md diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..2f4ff540c --- /dev/null +++ b/setup.py @@ -0,0 +1,54 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A setuptools based setup module. +See: +https://packaging.python.org/guides/distributing-packages-using-setuptools/ +https://github.com/pypa/sampleproject +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages, sic +from setuptools.extern import packaging +import pathlib +import os + +here = pathlib.Path(__file__).parent.resolve() + +long_description = (here / "PYPI-README.md").read_text() +packaging.version.Version = packaging.version.LegacyVersion + +setup( + name='rtdip-sdk', + long_description=long_description, + long_description_content_type='text/markdown', + url="https://github.com/rtdip/core", + classifiers=[ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + ], + project_urls={ + "Issue Tracker": "https://github.com/rtdip/core/issues", + "Source": "https://github.com/rtdip/core/", + "Documentation": "https://www.rtdip.io/" + }, + version=sic(os.environ["RTDIP_SDK_NEXT_VER"]), + package_dir={'': 'src/sdk/python'}, + packages=find_packages(where='src/sdk/python'), + python_requires='>=3.8, <4', + install_requires=['databricks-sql-connector','azure-identity','azure-storage-file-datalake','pyodbc','pandas', 'jinja2==3.0.3', 'jinjasql==0.1.8'], + setup_requires=['pytest-runner','setuptools_scm'], + tests_require=['pytest'], + test_suite='tests', +) \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/api/.dockerignore b/src/api/.dockerignore new file mode 100644 index 000000000..1927772bc --- /dev/null +++ b/src/api/.dockerignore @@ -0,0 +1 @@ +local.settings.json \ No newline at end of file diff --git a/src/api/.funcignore b/src/api/.funcignore new file mode 100644 index 000000000..4609ac717 --- /dev/null +++ b/src/api/.funcignore @@ -0,0 +1,3 @@ + + +.venv \ No newline at end of file diff --git a/src/api/.gitignore b/src/api/.gitignore new file mode 100644 index 000000000..f15ac3fc6 --- /dev/null +++ b/src/api/.gitignore @@ -0,0 +1,48 @@ +bin +obj +csx +.vs +edge +Publish + +*.user +*.suo +*.cscfg +*.Cache +project.lock.json + +/packages +/TestResults + +/tools/NuGet.exe +/App_Data +/secrets +/data +.secrets +appsettings.json +local.settings.json + +node_modules +dist + +# Local python packages +.python_packages/ + +# Python Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Azurite artifacts +__blobstorage__ +__queuestorage__ +__azurite_db*__.json \ No newline at end of file diff --git a/src/api/Dockerfile b/src/api/Dockerfile new file mode 100644 index 000000000..2f635d647 --- /dev/null +++ b/src/api/Dockerfile @@ -0,0 +1,32 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# To enable ssh & remote debugging on app service change the base image to the one below +# FROM mcr.microsoft.com/azure-functions/python:3.0-python3.8-appservice +FROM mcr.microsoft.com/azure-functions/python:3.0-python3.8 + +ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ + AzureFunctionsJobHost__Logging__Console__IsEnabled=true + +RUN rm -rf /var/lib/apt/lists/partial \ + && apt-get clean \ + && apt-get update -o Acquire::CompressionTypes::Order::=gz \ + && apt-get install -y gcc g++ odbcinst1debian2 libodbc1 odbcinst unixodbc-dev libsasl2-dev libsasl2-modules-gssapi-mit \ + && apt-get install -y ca-certificates curl + +COPY src/api/requirements.txt / +RUN pip install -r /requirements.txt + +COPY src/api/ /home/site/wwwroot +COPY src /home/site/wwwroot/src \ No newline at end of file diff --git a/src/api/FastAPIApp/__init__.py b/src/api/FastAPIApp/__init__.py new file mode 100644 index 000000000..981be7634 --- /dev/null +++ b/src/api/FastAPIApp/__init__.py @@ -0,0 +1,108 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fastapi import APIRouter, FastAPI +from fastapi.responses import RedirectResponse +from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html, get_swagger_ui_oauth2_redirect_html +from fastapi.security import OAuth2AuthorizationCodeBearer +from fastapi.middleware.gzip import GZipMiddleware +import os + +api_v1_router = APIRouter(prefix='/api/v1') +TITLE = "Real Time Data Ingestion Platform" + +tags_metadata = [ + { + "name": "Events", + "description": "Retrieval of timeseries data with options to resample and interpolate the result.", + }, + { + "name": "Metadata", + "description": "Contextual metadata about timeseries events", + } +] + +oauth2_scheme = OAuth2AuthorizationCodeBearer( + authorizationUrl = "https://login.microsoftonline.com/xxxxx/oauth2/v2.0/authorize", + tokenUrl= "https://login.microsoftonline.com/xxxxx/oauth2/v2.0/token", + refreshUrl="https://login.microsoftonline.com/xxxxx/oauth2/v2.0/refresh", +) + +description = """ +APIs to interact with Real Time Data Ingestion Platform. + +## Authentication + +The below APIs use OAuth 2.0 authentication and Bearer tokens for authentication. For more information, please refer to our [documentation](https://www.rtdip.io/api/authentication/) for further information and examples on how to authenticate with the APIs. + +### Azure Active Directory OAuth 2.0 + +Important Azure AD Values for Authentication are below. + +| Parameter | Value | +|-----------|-------| +| Token Url | https://login.microsoftonline.com/xxxxx/oauth2/v2.0/token | +| Scope | xxxxx | + +## Documentation + +Please refer to the following links for further information about these APIs and RTDIP in general: + +[ReDoc](/redoc) + +[Real Time Data Ingestion Platform](https://www.rtdip.io/) +""" + +app=FastAPI( + title=TITLE, + description=description, + version="1.0.0", + openapi_tags=tags_metadata, + openapi_url="/api/openapi.json", + docs_url=None, + redoc_url=None, +) + +app.add_middleware(GZipMiddleware, minimum_size=1000) + +@app.get("/", include_in_schema=False) +async def home(): + return RedirectResponse(url='/docs') + +@app.get("/docs", include_in_schema=False) +async def swagger_ui_html(): + client_id = os.getenv("MICROSOFT_PROVIDER_AUTHENTICATION_ID") + return get_swagger_ui_html( + openapi_url="api/openapi.json", + title=TITLE + " - Swagger", + swagger_favicon_url="xxxxx", + init_oauth={ + "usePkceWithAuthorizationCodeGrant": True, + "clientId": client_id, + "scopes": "xxxxx" + }, + oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url + ) + +@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False) +async def swagger_ui_redirect(): + return get_swagger_ui_oauth2_redirect_html() + +@app.get("/redoc", include_in_schema=False) +async def redoc_ui_html(): + return get_redoc_html( + openapi_url="api/openapi.json", + title=TITLE + " - ReDoc", + redoc_favicon_url="xxxxx" + ) \ No newline at end of file diff --git a/src/api/README.md b/src/api/README.md new file mode 100644 index 000000000..a762d81f7 --- /dev/null +++ b/src/api/README.md @@ -0,0 +1,65 @@ +# Getting Started with Azure Functions for RTDIP REST APIs + +## Specific Dev Information for RTDIP REST APIs + +[Fast API](https://fastapi.tiangolo.com/) is the Web Framework used for developing the APIs. It is recommended to read the documentation to understand the functionality it provides to quickly create REST APIs. Additionally, Azure Functions support Fast API with [documentation and examples](https://docs.microsoft.com/en-us/samples/azure-samples/fastapi-on-azure-functions/azure-functions-python-create-fastapi-app/) for your reference. + +Due to the package installation requirements for RTDIP SDK, it is required to deploy the Azure Functions using a docker container. It is fairly simple to test your development in docker, by executing the following steps + +### Docker development + +Ensure that you are in the root folder of the repository when you run the below commands. This ensures the correct folders can be copied into the container from the repo. + +```bash +docker build --tag rtdip-api:v0.1.0 -f src/api/Dockerfile . +docker run -p 8080:80 -it rtdip-api:v0.1.0 +``` + +REST APIs are then available at `http://localhost:8080/api/v1/{route}` + +### Debugging + +It is also possible to debug using the standard debugger in VS Code. The `.vscode` folder contains the relevant settings to automatically start debugging the APIs. **NOTE** that the endpoints for debugger sessions are `http://localhost:7071/api/v1/{route}` + +Ensure that you setup the **local.settings.json** file with the relevant parameters to execute the code on your machine. The below Databricks SQL settings are available in Databricks workspaces. + +|Encvironment Variable| Value | +|---------|-------| +|DATABRICKS_SQL_SERVER_HOSTNAME|adb-xxxxx.x.azuredatabricks.net| +|DATABRICKS_SQL_HTTP_PATH|/sql/1.0/endpoints/xxx| + +### Swagger and Redoc + +Fast API provides endpoints for Swagger and Redoc pages. Ensure that you review these pages after any updates to confirm they are working as expected. + +## General Dev Information for developing APIs using Azure Functions + +Below is general information regarding Azure Functions which is the framework for creating REST APIs for RTDIP. This enables serverless capabilities that allows for easy scaling the APIs according to demand. + +### Project Structure +The main project folder () can contain the following files: + +* **local.settings.json** - Used to store app settings and connection strings when running locally. This file doesn't get published to Azure. To learn more, see [local.settings.file](https://aka.ms/azure-functions/python/local-settings). +* **requirements.txt** - Contains the list of Python packages the system installs when publishing to Azure. +* **host.json** - Contains global configuration options that affect all functions in a function app. This file does get published to Azure. Not all options are supported when running locally. To learn more, see [host.json](https://aka.ms/azure-functions/python/host.json). +* **.vscode/** - (Optional) Contains store VSCode configuration. To learn more, see [VSCode setting](https://aka.ms/azure-functions/python/vscode-getting-started). +* **.venv/** - (Optional) Contains a Python virtual environment used by local development. +* **Dockerfile** - (Optional) Used when publishing your project in a [custom container](https://aka.ms/azure-functions/python/custom-container). +* **tests/** - (Optional) Contains the test cases of your function app. For more information, see [Unit Testing](https://aka.ms/azure-functions/python/unit-testing). +* **.funcignore** - (Optional) Declares files that shouldn't get published to Azure. Usually, this file contains .vscode/ to ignore your editor setting, .venv/ to ignore local Python virtual environment, tests/ to ignore test cases, and local.settings.json to prevent local app settings being published. + +Each function has its own code file and binding configuration file ([**function.json**](https://aka.ms/azure-functions/python/function.json)). + +### Developing your first Python function using VS Code + +If you have not already, please checkout our [quickstart](https://aka.ms/azure-functions/python/quickstart) to get you started with Azure Functions developments in Python. + +### Publishing your function app to Azure + +For more information on deployment options for Azure Functions, please visit this [guide](https://docs.microsoft.com/en-us/azure/azure-functions/create-first-function-vs-code-python#publish-the-project-to-azure). + +### Next Steps + +* To learn more about developing Azure Functions, please visit [Azure Functions Developer Guide](https://aka.ms/azure-functions/python/developer-guide). + +* To learn more specific guidance on developing Azure Functions with Python, please visit [Azure Functions Developer Python Guide](https://aka.ms/azure-functions/python/python-developer-guide). \ No newline at end of file diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/api/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/api/assets/__init__.py b/src/api/assets/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/api/assets/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/api/assets/favicon.png b/src/api/assets/favicon.png new file mode 100644 index 000000000..172c4dc91 Binary files /dev/null and b/src/api/assets/favicon.png differ diff --git a/src/api/auth/__init__.py b/src/api/auth/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/api/auth/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/api/auth/azuread.py b/src/api/auth/azuread.py new file mode 100644 index 000000000..04026a3ed --- /dev/null +++ b/src/api/auth/azuread.py @@ -0,0 +1,31 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fastapi.security import OAuth2AuthorizationCodeBearer +from src.sdk.python.rtdip_sdk.authentication.authenticate import DefaultAuth + +oauth2_scheme = OAuth2AuthorizationCodeBearer( + authorizationUrl = "https://login.microsoftonline.com/xxxxx/oauth2/v2.0/authorize", + tokenUrl= "https://login.microsoftonline.com/xxxxx/oauth2/v2.0/token", + refreshUrl="https://login.microsoftonline.com/xxxxx/oauth2/v2.0/refresh", +) + +def get_azure_ad_token(authorization = None): + if authorization == None or authorization == "No Token": + access_token = DefaultAuth().authenticate() + token = access_token.get_token("xxxxx").token + else: + token = authorization.replace("Bearer ", "") + + return token \ No newline at end of file diff --git a/src/api/host.json b/src/api/host.json new file mode 100644 index 000000000..79edb2c32 --- /dev/null +++ b/src/api/host.json @@ -0,0 +1,20 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensions": { + "http": { + "routePrefix": "" + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[2.*, 3.0.0)" + } +} \ No newline at end of file diff --git a/src/api/requirements.txt b/src/api/requirements.txt new file mode 100644 index 000000000..a5eda0a65 --- /dev/null +++ b/src/api/requirements.txt @@ -0,0 +1,12 @@ +# Do not include azure-functions-worker as it may conflict with the Azure Functions platform +azure-functions +fastapi +nest_asyncio +strawberry-graphql[fastapi] +databricks-sql-connector +azure-identity==1.11.0 +oauthlib>=3.2.1 +pandas==1.4.2 +numpy>=1.22.0 +jinja2==3.0.3 +jinjasql==0.1.8 \ No newline at end of file diff --git a/src/api/v1/__init__.py b/src/api/v1/__init__.py new file mode 100644 index 000000000..961828886 --- /dev/null +++ b/src/api/v1/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import azure.functions as func +from fastapi import Depends + +from src.api.FastAPIApp import app, api_v1_router +import src.api.v1.metadata +import src.api.v1.raw +import src.api.v1.resample +import src.api.v1.interpolate +import src.api.v1.graphql +from src.api.v1.graphql import graphql_router +from src.api.auth.azuread import oauth2_scheme + +app.include_router(api_v1_router) +app.include_router(graphql_router, prefix="/graphql", include_in_schema=False, dependencies=[Depends(oauth2_scheme)]) + +def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: + return func.AsgiMiddleware(app).handle(req, context) \ No newline at end of file diff --git a/src/api/v1/common.py b/src/api/v1/common.py new file mode 100644 index 000000000..df5857890 --- /dev/null +++ b/src/api/v1/common.py @@ -0,0 +1,50 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from src.api.auth import azuread + +def common_api_setup_tasks(base_query_parameters, metadata_query_parameters = None, raw_query_parameters = None, tag_query_parameters = None, resample_query_parameters = None, interpolate_query_parameters = None): + token = azuread.get_azure_ad_token(base_query_parameters.authorization) + + connection = DatabricksSQLConnection(os.environ.get("DATABRICKS_SQL_SERVER_HOSTNAME"), os.environ.get("DATABRICKS_SQL_HTTP_PATH"), token) + + parameters = base_query_parameters.__dict__ + + if metadata_query_parameters != None: + parameters = dict(parameters, **metadata_query_parameters.__dict__) + if "tag_name" in parameters: + if parameters["tag_name"] == None: + parameters["tag_names"] = [] + parameters.pop("tag_name") + else: + parameters["tag_names"] = parameters.pop("tag_name") + + if raw_query_parameters != None: + parameters = dict(parameters, **raw_query_parameters.__dict__) + parameters["start_date"] = str(raw_query_parameters.start_date) + parameters["end_date"] = str(raw_query_parameters.end_date) + + if tag_query_parameters != None: + parameters = dict(parameters, **tag_query_parameters.__dict__) + parameters["tag_names"] = parameters.pop("tag_name") + + if resample_query_parameters != None: + parameters = dict(parameters, **resample_query_parameters.__dict__) + + if interpolate_query_parameters != None: + parameters = dict(parameters, **interpolate_query_parameters.__dict__) + + return connection, parameters \ No newline at end of file diff --git a/src/api/v1/function.json b/src/api/v1/function.json new file mode 100644 index 000000000..2e27e7669 --- /dev/null +++ b/src/api/v1/function.json @@ -0,0 +1,21 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "Anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ], + "route": "{*route}" + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/src/api/v1/graphql.py b/src/api/v1/graphql.py new file mode 100644 index 000000000..ca0b32ca4 --- /dev/null +++ b/src/api/v1/graphql.py @@ -0,0 +1,73 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import strawberry +import os +from strawberry.fastapi import GraphQLRouter +from src.api.v1.models import RawResponseQL +import logging +from typing import Any +import json +from fastapi import Query, HTTPException, Header, Depends +from typing import List +from datetime import date +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from src.sdk.python.rtdip_sdk.functions import raw +from src.api.v1.models import RawResponse +from src.api.auth import azuread +import nest_asyncio +nest_asyncio.apply() + +@strawberry.type +class Query: + @strawberry.field + async def raw_get( + business_unit: str = Query(..., description="Business Unit Name"), + region: str = Query(..., description="Region"), + asset: str = Query(..., description="Asset"), + data_security_level: str = Query(..., description="Data Security Level"), + data_type: str = Query(..., description="Data Type", examples={"float": {"value": "float"}, "integer": {"value": "integer"}, "string": {"value": "string"}}), + tag_name: List[str] = Query(..., description="Tag Name"), + include_bad_data: bool = Query(True, description="Include or remove Bad data points"), + start_date: date = Query(..., description="Start Date", example="2022-01-01"), + end_date: date = Query(..., description="End Date", example="2022-01-02"), + authorization: str = Header(None, include_in_schema=False), + # authorization: str = Depends(oauth2_scheme) + ) -> RawResponseQL: + try: + token = azuread.get_azure_ad_token(authorization) + + connection = DatabricksSQLConnection(os.environ.get("DATABRICKS_SQL_SERVER_HOSTNAME"), os.environ.get("DATABRICKS_SQL_HTTP_PATH"), token) + + parameters = { + "business_unit": business_unit, + "region": region, + "asset": asset, + "data_security_level": data_security_level, + "data_type": data_type, + "tag_names": tag_name, + "include_bad_data": include_bad_data, + "start_date": str(start_date), + "end_date": str(end_date), + } + data = raw.get(connection, parameters) + response = data.to_json(orient="table", index=False) + return RawResponse(**json.loads(response)) + except Exception as e: + logging.error(str(e)) + return HTTPException(status_code=500, detail=str(e)) + +schema = strawberry.Schema(Query) + +graphql_router = GraphQLRouter(schema) \ No newline at end of file diff --git a/src/api/v1/interpolate.py b/src/api/v1/interpolate.py new file mode 100644 index 000000000..7f8e1b7ee --- /dev/null +++ b/src/api/v1/interpolate.py @@ -0,0 +1,85 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from src.api.FastAPIApp import api_v1_router +from fastapi import HTTPException, Depends, Body +import nest_asyncio +import json +from src.sdk.python.rtdip_sdk.functions import interpolate +from src.api.v1.models import BaseQueryParams, ResampleInterpolateResponse, HTTPError, RawQueryParams, TagsQueryParams, TagsBodyParams, ResampleQueryParams, InterpolateQueryParams +import src.api.v1.common + +nest_asyncio.apply() + +def interpolate_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters, resample_parameters, interpolate_parameters): + try: + (connection, parameters) = src.api.v1.common.common_api_setup_tasks( + base_query_parameters, + raw_query_parameters=raw_query_parameters, + resample_query_parameters=resample_parameters, + tag_query_parameters=tag_query_parameters, + interpolate_query_parameters=interpolate_parameters + ) + + data = interpolate.get(connection, parameters) + response = data.to_json(orient="table", index=False) + return ResampleInterpolateResponse(**json.loads(response)) + except Exception as e: + logging.error(str(e)) + raise HTTPException(status_code=400, detail=str(e)) + +get_description = """ +## Interpolate + +Interpolation of raw timeseries data. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/interpolate/) for further information. +""" + +@api_v1_router.get( + path="/events/interpolate", + name="Interpolate GET", + description=get_description, + tags=["Events"], + responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}} +) +async def interpolate_get( + base_query_parameters: BaseQueryParams = Depends(), + raw_query_parameters: RawQueryParams = Depends(), + tag_query_parameters: TagsQueryParams = Depends(), + resample_parameters: ResampleQueryParams = Depends(), + interpolate_parameters: InterpolateQueryParams = Depends() + ): + return interpolate_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters, resample_parameters, interpolate_parameters) + +post_description = """ +## Interpolate + +Interpolation of raw timeseries data via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/interpolate/) for further information. +""" + +@api_v1_router.post( + path="/events/interpolate", + name="Interpolate POST", + description=get_description, + tags=["Events"], + responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}} +) +async def interpolate_post( + base_query_parameters: BaseQueryParams = Depends(), + raw_query_parameters: RawQueryParams = Depends(), + tag_query_parameters: TagsBodyParams = Body(default=...), + resample_parameters: ResampleQueryParams = Depends(), + interpolate_parameters: InterpolateQueryParams = Depends() + ): + return interpolate_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters, resample_parameters, interpolate_parameters) \ No newline at end of file diff --git a/src/api/v1/metadata.py b/src/api/v1/metadata.py new file mode 100644 index 000000000..55bee46be --- /dev/null +++ b/src/api/v1/metadata.py @@ -0,0 +1,71 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import json +from fastapi import Query, HTTPException, Depends, Body +import nest_asyncio +from src.sdk.python.rtdip_sdk.functions import metadata +from src.api.v1.models import BaseQueryParams, MetadataQueryParams, TagsBodyParams, MetadataResponse, HTTPError +from src.api.auth.azuread import oauth2_scheme +from src.api.FastAPIApp import api_v1_router +from src.api.v1 import common + +nest_asyncio.apply() + +def metadata_retrieval_get(query_parameters, metadata_query_parameters): + try: + (connection, parameters) = common.common_api_setup_tasks(query_parameters, metadata_query_parameters=metadata_query_parameters) + + data = metadata.get(connection, parameters) + response = data.to_json(orient="table", index=False) + return MetadataResponse(**json.loads(response)) + except Exception as e: + logging.error(str(e)) + raise HTTPException(status_code=400, detail=str(e)) + +get_description = """ +## Metadata + +Retrieval of metadata, including UoM, Description and any other possible fields, if available. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/metadata/) for further information. +""" + +@api_v1_router.get( + path="/metadata", + name="Metadata GET", + description=get_description, + tags=["Metadata"], + dependencies=[Depends(oauth2_scheme)], + responses={200: {"model": MetadataResponse}, 400: {"model": HTTPError}} +) +async def metadata_get(query_parameters: BaseQueryParams = Depends(), metadata_query_parameters: MetadataQueryParams = Depends()): + return metadata_retrieval_get(query_parameters, metadata_query_parameters) + +post_description = """ +## Metadata + +Retrieval of metadata, including UoM, Description and any other possible fields, if available via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/metadata/) for further information. +""" + +@api_v1_router.post( + path="/metadata", + name="Metadata POST", + description=post_description, + tags=["Metadata"], + dependencies=[Depends(oauth2_scheme)], + responses={200: {"model": MetadataResponse}, 400: {"model": HTTPError}} +) +async def metadata_post(query_parameters: BaseQueryParams = Depends(), metadata_query_parameters: TagsBodyParams = Body(default=...)): + return metadata_retrieval_get(query_parameters, metadata_query_parameters) \ No newline at end of file diff --git a/src/api/v1/models.py b/src/api/v1/models.py new file mode 100644 index 000000000..647c94e26 --- /dev/null +++ b/src/api/v1/models.py @@ -0,0 +1,152 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from tracemalloc import start +from pydantic import BaseModel, Field, Extra +import strawberry +from typing import List, Union, Dict, Any +from fastapi import Query, Header, Depends +from datetime import date +from src.api.auth.azuread import oauth2_scheme + +class Fields(BaseModel): + name: str + type: str + +@strawberry.experimental.pydantic.type(model=Fields, all_fields=True) +class FieldsQL: + pass + +class FieldSchema(BaseModel): + fields: List[Fields] + pandas_version: str + +@strawberry.type +class FieldSchemaQL: + fields: List[FieldsQL] + pandas_version: str + +class MetadataRow(BaseModel): + TagName: str + UoM: str + Description: str + class Config: + extra = Extra.allow + +class RawRow(BaseModel): + EventTime: datetime + TagName: str + Status: str + Value: Union[float, int, str, None] + +@strawberry.type +class RawRowQL: + EventTime: datetime + TagName: str + Status: str + Value: float + +class MetadataResponse(BaseModel): + field_schema: FieldSchema = Field(None, alias='schema') + data: List[MetadataRow] + +class RawResponse(BaseModel): + field_schema: FieldSchema = Field(None, alias='schema') + data: List[RawRow] + +@strawberry.type +class RawResponseQL: + schema: FieldSchemaQL + data: List[RawRowQL] + +class ResampleInterpolateRow(BaseModel): + EventTime: datetime + TagName: str + Value: Union[float, int, str, None] + +class ResampleInterpolateResponse(BaseModel): + field_schema: FieldSchema = Field(None, alias='schema') + data: List[ResampleInterpolateRow] + +class HTTPError(BaseModel): + detail: str + + class Config: + schema_extra = { + "example": {"detail": "HTTPException raised."}, + } + +class BaseQueryParams: + def __init__( + self, + business_unit: str = Query(..., description="Business Unit Name"), + region: str = Query(..., description="Region"), + asset: str = Query(..., description="Asset"), + data_security_level: str = Query(..., description="Data Security Level"), + authorization: str = Depends(oauth2_scheme) + ): + self.business_unit = business_unit + self.region = region + self.asset = asset + self.data_security_level = data_security_level + self.authorization = authorization + +class MetadataQueryParams: + def __init__( + self, + tag_name: List[str] = Query(None, description="Tag Name"), + ): + self.tag_name = tag_name + +class RawQueryParams: + def __init__( + self, + data_type: str = Query(..., description="Data Type"), + include_bad_data: bool = Query(..., description="Include or remove Bad data points"), + start_date: Union[date, datetime] = Query(..., description="Start Date in format YYYY-MM-DD or YYYY-MM-DDTHH:mm:ss", examples={"2022-01-01": {"value": "2022-01-01"}, "2022-01-01T15:00:00": {"value": "2022-01-01T15:00:00"}}), + end_date: Union[date, datetime] = Query(..., description="End Date in format YYYY-MM-DD or YYYY-MM-DDTHH:mm:ss", examples={"2022-01-02": {"value": "2022-01-02"}, "2022-01-01T16:00:00": {"value": "2022-01-01T16:00:00"}}), + ): + self.data_type = data_type + self.include_bad_data = include_bad_data + self.start_date = start_date + self.end_date = end_date + +class TagsQueryParams: + def __init__( + self, + tag_name: List[str] = Query(..., description="Tag Name"), + ): + self.tag_name = tag_name + +class TagsBodyParams(BaseModel): + tag_name: List[str] + +class ResampleQueryParams: + def __init__( + self, + sample_rate: str = Query(..., description="Sample Rate", example="5"), + sample_unit: str = Query(..., description="Sample Unit", examples={"second": {"value": "second"}, "minute": {"value": "minute"}, "hour": {"value": "hour"}, "day": {"value": "day"}}), + agg_method: str = Query(..., escription="Aggregation Method", examples={"first": {"value": "first"}, "last": {"value": "last"}, "avg": {"value": "avg"}, "min": {"value": "min"}, "max": {"value": "max"}}), + ): + self.sample_rate = sample_rate + self.sample_unit = sample_unit + self.agg_method = agg_method + +class InterpolateQueryParams: + def __init__( + self, + interpolation_method: str = Query(..., description="Interpolation Method", examples={"forward_fill": {"value": "forward_fill"}, "backward_fill": {"value": "backward_fill"}}), + ): + self.interpolation_method = interpolation_method \ No newline at end of file diff --git a/src/api/v1/raw.py b/src/api/v1/raw.py new file mode 100644 index 000000000..21377c142 --- /dev/null +++ b/src/api/v1/raw.py @@ -0,0 +1,81 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import json +from fastapi import Query, HTTPException, Depends, Body +import nest_asyncio +from src.sdk.python.rtdip_sdk.functions import raw +from src.api.v1.models import BaseQueryParams, RawResponse, RawQueryParams, TagsQueryParams, TagsBodyParams,HTTPError +from src.api.auth.azuread import oauth2_scheme +from src.api.FastAPIApp import api_v1_router +import src.api.v1.common + +nest_asyncio.apply() + +def raw_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters): + try: + (connection, parameters) = src.api.v1.common.common_api_setup_tasks(base_query_parameters, raw_query_parameters=raw_query_parameters, tag_query_parameters=tag_query_parameters) + + data = raw.get(connection, parameters) + response = data.to_json(orient="table", index=False) + return RawResponse(**json.loads(response)) + except Exception as e: + logging.error(str(e)) + raise HTTPException(status_code=400, detail=str(e)) + +get_description = """ +## Raw + +Retrieval of raw timeseries data. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/raw/) for further information. +""" + +@api_v1_router.get( + path="/events/raw", + name="Raw GET", + description=get_description, + tags=["Events"], + dependencies=[Depends(oauth2_scheme)], + responses={200: {"model": RawResponse}, 400: {"model": HTTPError}} +) +async def raw_get( + base_query_parameters: BaseQueryParams = Depends(), + raw_query_parameters: RawQueryParams = Depends(), + tag_query_parameters: TagsQueryParams = Depends(), + ): + return raw_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters) + + +post_description = """ +## Raw + +Retrieval of raw timeseries data via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/raw/) for further information. +""" + +@api_v1_router.post( + path="/events/raw", + name="Raw POST", + description=post_description, + tags=["Events"], + dependencies=[Depends(oauth2_scheme)], + responses={200: {"model": RawResponse}, 400: {"model": HTTPError}} +) +async def raw_post( + base_query_parameters: BaseQueryParams = Depends(), + raw_query_parameters: RawQueryParams = Depends(), + tag_query_parameters: TagsBodyParams = Body(default=...), + ): + return raw_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters) + \ No newline at end of file diff --git a/src/api/v1/resample.py b/src/api/v1/resample.py new file mode 100644 index 000000000..2e9d9b725 --- /dev/null +++ b/src/api/v1/resample.py @@ -0,0 +1,79 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +from requests import request +from src.api.FastAPIApp import api_v1_router +from fastapi import HTTPException, Depends, Body +import nest_asyncio +import json +from src.sdk.python.rtdip_sdk.functions import resample +from src.api.v1.models import BaseQueryParams, ResampleInterpolateResponse, HTTPError, RawQueryParams, TagsQueryParams, TagsBodyParams,ResampleQueryParams +import src.api.v1.common + +nest_asyncio.apply() + +def resample_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters, resample_parameters): + try: + (connection, parameters) = src.api.v1.common.common_api_setup_tasks(base_query_parameters, raw_query_parameters=raw_query_parameters, tag_query_parameters=tag_query_parameters,resample_query_parameters=resample_parameters) + + data = resample.get(connection, parameters) + response = data.to_json(orient="table", index=False) + return ResampleInterpolateResponse(**json.loads(response)) + except Exception as e: + logging.error(str(e)) + raise HTTPException(status_code=400, detail=str(e)) + +get_description = """ +## Resample + +Resampling of raw timeseries data. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/resample/) for further information. +""" + +@api_v1_router.get( + path="/events/resample", + name="Resample GET", + description=get_description, + tags=["Events"], + responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}} +) +async def resample_get( + base_query_parameters: BaseQueryParams = Depends(), + raw_query_parameters: RawQueryParams = Depends(), + tag_query_parameters: TagsQueryParams = Depends(), + resample_parameters: ResampleQueryParams = Depends(), + ): + return resample_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters, resample_parameters) + +post_description = """ +## Resample + +Resampling of raw timeseries data via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/resample/) for further information. +""" + +@api_v1_router.post( + path="/events/resample", + name="Resample POST", + description=post_description, + tags=["Events"], + responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}} +) +async def resample_post( + base_query_parameters: BaseQueryParams = Depends(), + raw_query_parameters: RawQueryParams = Depends(), + tag_query_parameters: TagsBodyParams = Body(default=...), + resample_parameters: ResampleQueryParams = Depends(), + ): + return resample_events_get(base_query_parameters, raw_query_parameters, tag_query_parameters, resample_parameters) \ No newline at end of file diff --git a/src/apps/docs/staticwebapp.config.json b/src/apps/docs/staticwebapp.config.json new file mode 100644 index 000000000..2565635eb --- /dev/null +++ b/src/apps/docs/staticwebapp.config.json @@ -0,0 +1,43 @@ +{ + "navigationFallback": { + "rewrite": "index.html", + "exclude": ["/images/*.{png,jpg,gif}", "/css/*"] + }, + "routes": [ + { + "route": "/.auth/*", + "allowedRoles": ["anonymous"] + }, + { + "route": "/*", + "allowedRoles": ["authenticated"] + } + ], + "responseOverrides": { + "401": { + "redirect": "/.auth/login/aad", + "statusCode": 302 + }, + "404": { + "rewrite": "/404.html" + } + }, + "globalHeaders": { + "content-security-policy": "default-src https: 'unsafe-eval' 'unsafe-inline'; object-src 'none'" + }, + "mimeTypes": { + ".json": "text/json", + ".svg": "image/svg+xml" + }, + "auth": { + "identityProviders": { + "azureActiveDirectory": { + "registration": { + "openIdIssuer": "https://login.microsoftonline.com/xxxxx/v2.0", + "clientIdSettingName": "AAD_CLIENT_ID", + "clientSecretSettingName": "AAD_CLIENT_SECRET" + } + } + } + } +} \ No newline at end of file diff --git a/src/sdk/__init__.py b/src/sdk/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/sdk/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/sdk/python/__init__.py b/src/sdk/python/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/sdk/python/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/sdk/python/rtdip_sdk/__init__.py b/src/sdk/python/rtdip_sdk/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/sdk/python/rtdip_sdk/authentication/__init__.py b/src/sdk/python/rtdip_sdk/authentication/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/authentication/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/sdk/python/rtdip_sdk/authentication/authenticate.py b/src/sdk/python/rtdip_sdk/authentication/authenticate.py new file mode 100644 index 000000000..7e5cce03a --- /dev/null +++ b/src/sdk/python/rtdip_sdk/authentication/authenticate.py @@ -0,0 +1,145 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from azure.identity import ClientSecretCredential, CertificateCredential, DefaultAzureCredential +import logging + +class ClientSecretAuth: + """ + Enables authentication to Azure Active Directory using a client secret that was generated for an App Registration. + + Args: + tenant_id: The Azure Active Directory tenant (directory) Id of the service principal. + client_id: The client (application) ID of the service principal + client_secret: A client secret that was generated for the App Registration used to authenticate the client. + """ + def __init__(self, tenant_id: str, client_id: str, client_secret: str) -> None: + self.tenant_id=tenant_id + self.client_id=client_id + self.client_secret=client_secret + + def authenticate(self) -> ClientSecretCredential: + """ + Authenticates as a service principal using a client secret. + + Returns: + ClientSecretCredential: Authenticates as a service principal using a client secret. + """ + try: + access_token = ClientSecretCredential(self.tenant_id, self.client_id, self.client_secret) + return access_token + except Exception as e: + logging.exception('error returning client secret credential') + raise e + +class CertificateAuth: + """ + Enables authentication to Azure Active Directory using a certificate that was generated for an App Registration. + + The certificate must have an RSA private key, because this credential signs assertions using RS256 + + Args: + tenant_id: The Azure Active Directory tenant (directory) Id of the service principal. + client_id: The client (application) ID of the service principal + certificate_path: Optional path to a certificate file in PEM or PKCS12 format, including the private key. If not provided, certificate_data is required. + """ + def __init__(self, tenant_id: str, client_id: str, certificate_path:str=None) -> None: + self.tenant_id=tenant_id + self.client_id=client_id + self.certificate_path=certificate_path + + def authenticate(self) -> CertificateCredential: + """ + Authenticates as a service principal using a certificate. + + Returns: + CertificateCredential: Authenticates as a service principal using a certificate. + """ + try: + access_token = CertificateCredential(self.tenant_id, self.client_id, self.certificate_path) + return access_token + except Exception as e: + logging.exception('error returning certificate credential') + raise e + +class DefaultAuth: + """ + A default credential capable of handling most Azure SDK authentication scenarios. + + The identity it uses depends on the environment. When an access token is needed, it requests one using these identities in turn, stopping when one provides a token: + + 1) A service principal configured by environment variables. + + 2) An Azure managed identity. + + 3) On Windows only: a user who has signed in with a Microsoft application, such as Visual Studio. If multiple identities are in the cache, then the value of the environment variable AZURE_USERNAME is used to select which identity to use. + + 4) The user currently signed in to Visual Studio Code. + + 5) The identity currently logged in to the Azure CLI. + + 6) The identity currently logged in to Azure PowerShell. + + Args: + exclude_cli_credential (Optional): Whether to exclude the Azure CLI from the credential. Defaults to False. + exclude_environment_credential (Optional): Whether to exclude a service principal configured by environment variables from the credential. Defaults to True. + exclude_managed_identity_credential (Optional): Whether to exclude managed identity from the credential. Defaults to True + exclude_powershell_credential (Optional): Whether to exclude Azure PowerShell. Defaults to False. + exclude_visual_studio_code_credential (Optional): Whether to exclude stored credential from VS Code. Defaults to False + exclude_shared_token_cache_credential (Optional): Whether to exclude the shared token cache. Defaults to False. + exclude_interactive_browser_credential (Optional): Whether to exclude interactive browser authentication (see InteractiveBrowserCredential). Defaults to False + logging_enable (Optional): Turn on or off logging. Defaults to False. + """ + def __init__(self, exclude_cli_credential=False, exclude_environment_credential=True, exclude_managed_identity_credential=True, exclude_powershell_credential=False, exclude_visual_studio_code_credential=False, exclude_shared_token_cache_credential=False, exclude_interactive_browser_credential=False, logging_enable=False) -> None: + self.exclude_cli_credential=exclude_cli_credential + self.exclude_environment_credential=exclude_environment_credential + self.exclude_managed_identity_credential=exclude_managed_identity_credential + self.exclude_powershell_credential=exclude_powershell_credential + self.exclude_visual_studio_code_credential=exclude_visual_studio_code_credential + self.exclude_shared_token_cache_credential=exclude_shared_token_cache_credential + self.exclude_interactive_browser_credential=exclude_interactive_browser_credential + self.logging_enable=logging_enable + + def authenticate(self) -> DefaultAzureCredential: + """ + A default credential capable of handling most Azure SDK authentication scenarios. + + Returns: + DefaultAzureCredential: A default credential capable of handling most Azure SDK authentication scenarios. + """ + try: + access_token = DefaultAzureCredential( + exclude_cli_credential=self.exclude_cli_credential, + exclude_environment_credential=self.exclude_environment_credential, + exclude_managed_identity_credential=self.exclude_managed_identity_credential, + exclude_powershell_credential=self.exclude_powershell_credential, + exclude_visual_studio_code_credential=self.exclude_visual_studio_code_credential, + exclude_shared_token_cache_credential=self.exclude_shared_token_cache_credential, + exclude_interactive_browser_credential=self.exclude_interactive_browser_credential, + logging_enable=self.logging_enable) + return access_token + except Exception as e: + logging.exception('error returning default azure credential') + raise e + +class Authenticator: + """ + The class used to authenticate to different systems. + + Args: + auth_class: Authentication class containing the users credentials + """ + + def __init__(self, auth_class) -> None: + self.auth_class = auth_class \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/functions/__init__.py b/src/sdk/python/rtdip_sdk/functions/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/functions/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/sdk/python/rtdip_sdk/functions/_query_builder.py b/src/sdk/python/rtdip_sdk/functions/_query_builder.py new file mode 100644 index 000000000..69c7dccc8 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/functions/_query_builder.py @@ -0,0 +1,190 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from jinjasql import JinjaSql +from six import string_types +from copy import deepcopy + +def _fix_dates(parameters_dict): + if len(parameters_dict['start_date']) == 10: + parameters_dict['start_date'] = parameters_dict['start_date']+'T00:00:00' + + if len(parameters_dict['end_date']) == 10: + parameters_dict['end_date'] = parameters_dict['end_date']+'T23:59:59' + + return parameters_dict + +def _quote_sql_string(value): + ''' + If `value` is a string type, escapes single quotes in the string + and returns the string enclosed in single quotes. + ''' + if isinstance(value, string_types): + new_value = str(value) + new_value = new_value.replace("'", "''") + return "'{}'".format(new_value) + return value + + +def _get_sql_from_template(query: str, bind_params: dict) -> str: + ''' + Given a query and binding parameters produced by JinjaSql's prepare_query(), + produce and return a complete SQL query string. + ''' + if not bind_params: + return query + params = deepcopy(bind_params) + for key, val in params.items(): + params[key] = _quote_sql_string(val) + return query % params + + +def _raw_query(parameters_dict: dict) -> str: + + raw_query = ( + "SELECT EventTime, TagName, Status, Value FROM " + "{{ business_unit | sqlsafe }}.sensors.{{ asset | sqlsafe }}_{{ data_security_level | sqlsafe }}_events_{{ data_type | sqlsafe }} " + "WHERE EventDate BETWEEN to_date({{ start_date }}) AND to_date({{ end_date }}) AND EventTime BETWEEN to_timestamp({{ start_date }}) AND to_timestamp({{ end_date }}) AND TagName in {{ tag_names | inclause }} " + "{% if include_bad_data is defined and include_bad_data == false %}" + "AND Status = 'Good'" + "{% endif %}" + ) + + raw_parameters = { + "business_unit": parameters_dict['business_unit'].lower(), + "region": parameters_dict['region'].lower(), + "asset": parameters_dict['asset'].lower(), + "data_security_level": parameters_dict['data_security_level'].lower(), + "data_type": parameters_dict['data_type'].lower(), + "start_date": parameters_dict['start_date'], + "end_date": parameters_dict['end_date'], + "tag_names": list(dict.fromkeys(parameters_dict['tag_names'])), + "include_bad_data": parameters_dict['include_bad_data'] + } + + sql_template = JinjaSql(param_style='pyformat') + query, bind_params = sql_template.prepare_query(raw_query, raw_parameters) + sql_query = _get_sql_from_template(query, bind_params) + return sql_query + +def _sample_query(parameters_dict: dict) -> tuple: + + sample_query = ( + "SELECT DISTINCT TagName, w.start AS EventTime, {{ agg_method | sqlsafe }}(Value) OVER " + "(PARTITION BY TagName, w.start ORDER BY EventTime ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS Value FROM (" + "SELECT EventTime, WINDOW(EventTime, {{ sample_rate + ' ' + sample_unit }}) w, TagName, Status, Value FROM " + "{{ business_unit | sqlsafe }}.sensors.{{ asset | sqlsafe }}_{{ data_security_level | sqlsafe }}_events_{{ data_type | sqlsafe }} " + "WHERE EventDate BETWEEN to_date({{ start_date }}) AND to_date({{ end_date }}) AND EventTime BETWEEN to_timestamp({{ start_date }}) AND to_timestamp({{ end_date }}) AND TagName in {{ tag_names | inclause }} " + "{% if include_bad_data is defined and include_bad_data == false %}" + "AND Status = 'Good'" + "{% endif %}" + ")" + ) + + sample_parameters = { + "business_unit": parameters_dict['business_unit'].lower(), + "region": parameters_dict['region'].lower(), + "asset": parameters_dict['asset'].lower(), + "data_security_level": parameters_dict['data_security_level'].lower(), + "data_type": parameters_dict['data_type'].lower(), + "start_date": parameters_dict['start_date'], + "end_date": parameters_dict['end_date'], + "tag_names": list(dict.fromkeys(parameters_dict['tag_names'])), + "include_bad_data": parameters_dict['include_bad_data'], + "sample_rate": parameters_dict['sample_rate'], + "sample_unit": parameters_dict['sample_unit'], + "agg_method": parameters_dict['agg_method'] + } + + sql_template = JinjaSql(param_style='pyformat') + query, bind_params = sql_template.prepare_query(sample_query, sample_parameters) + sql_query = _get_sql_from_template(query, bind_params) + return sql_query, sample_query, sample_parameters + +def _interpolation_query(parameters_dict: dict, sample_query: str, sample_parameters: dict, tag_name_string: str) -> str: + + if parameters_dict["interpolation_method"] == "forward_fill": + interpolation_method = 'last_value/UNBOUNDED PRECEDING/CURRENT ROW' + + if parameters_dict["interpolation_method"] == "backward_fill": + interpolation_method = 'first_value/CURRENT ROW/UNBOUNDED FOLLOWING' + + interpolation_options = interpolation_method.split('/') + + interpolate_query = ( + "SELECT a.EventTime, a.TagName, {{ interpolation_options_0 | sqlsafe }}(b.Value, true) OVER (PARTITION BY a.TagName ORDER BY a.EventTime ROWS BETWEEN {{ interpolation_options_1 | sqlsafe }} AND {{ interpolation_options_2 | sqlsafe }}) AS Value FROM " + "(SELECT explode(sequence(to_timestamp({{ start_date }}), to_timestamp({{ end_date }}), INTERVAL {{ sample_rate + ' ' + sample_unit }})) AS EventTime, explode(array({{ tag_name_string }})) AS TagName) a " + f"LEFT OUTER JOIN ({sample_query}) b " + "ON a.EventTime = b.EventTime " + "AND a.TagName = b.TagName" + ) + + interpolate_parameters = sample_parameters.copy() + interpolate_parameters["interpolation_options_0"] = interpolation_options[0] + interpolate_parameters["interpolation_options_1"] = interpolation_options[1] + interpolate_parameters["interpolation_options_2"] = interpolation_options[2] + interpolate_parameters["tag_name_string"] = tag_name_string + + sql_template = JinjaSql(param_style='pyformat') + query, bind_params = sql_template.prepare_query(interpolate_query, interpolate_parameters) + sql_query = _get_sql_from_template(query, bind_params) + return sql_query + +def _metadata_query(parameters_dict: dict) -> str: + + metadata_query = ( + "SELECT * FROM " + "{{ business_unit | sqlsafe }}.sensors.{{ asset | sqlsafe }}_{{ data_security_level | sqlsafe }}_metadata " + "{% if tag_names is defined and tag_names|length > 0 %}" + + "WHERE TagName in {{ tag_names | inclause }}" + "{% endif %}" + ) + + metadata_parameters = { + "business_unit": parameters_dict['business_unit'].lower(), + "region": parameters_dict['region'].lower(), + "asset": parameters_dict['asset'].lower(), + "data_security_level": parameters_dict['data_security_level'].lower(), + "tag_names": list(dict.fromkeys(parameters_dict['tag_names'])) + } + + sql_template = JinjaSql(param_style='pyformat') + query, bind_params = sql_template.prepare_query(metadata_query, metadata_parameters) + sql_query = _get_sql_from_template(query, bind_params) + return sql_query + +def _query_builder(parameters_dict: dict, metadata=False) -> str: + if "tag_names" not in parameters_dict: + parameters_dict["tag_names"] = [] + tagnames_deduplicated = list(dict.fromkeys(parameters_dict['tag_names'])) #remove potential duplicates in tags + parameters_dict["tag_names"] = tagnames_deduplicated.copy() + tag_name_string = ', '.join('"{0}"'.format(tagname) for tagname in tagnames_deduplicated) + + if metadata: + return _metadata_query(parameters_dict) + + parameters_dict = _fix_dates(parameters_dict) + + if "agg_method" not in parameters_dict: + return _raw_query(parameters_dict) + + if "sample_rate" in parameters_dict: + sample_prepared_query, sample_query, sample_parameters = _sample_query(parameters_dict) + + if "interpolation_method" not in parameters_dict: + return sample_prepared_query + + if "interpolation_method" in parameters_dict: + return _interpolation_query(parameters_dict, sample_query, sample_parameters, tag_name_string) \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/functions/interpolate.py b/src/sdk/python/rtdip_sdk/functions/interpolate.py new file mode 100644 index 000000000..fbea50e88 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/functions/interpolate.py @@ -0,0 +1,69 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +sys.path.insert(0, 'src/sdk/python') +import pandas as pd +from rtdip_sdk.functions._query_builder import _query_builder + +def get(connection: object, parameters_dict: dict) -> pd.DataFrame: + ''' + An RTDIP interpolation function that is intertwined with the RTDIP Resampling function. + + The Interpolation function will forward fill or backward fill the resampled data depending users specified interpolation method. + + This function requires the user to input a dictionary of parameters. (See Attributes table below.) + + Args: + connection: Connection chosen by the user (Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect) + parameters_dict: A dictionary of parameters (see Attributes table below) + + Attributes: + buisness_unit (str): Business unit of the data + region (str): Region + asset (str): Asset + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) + tag_names (list): List of tagname or tagnames ["tag_1", "tag_2"] + start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS) + end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS) + sample_rate (int): The resampling rate (numeric input) + sample_unit (str): The resampling unit (second, minute, day, hour) + agg_method (str): Aggregation Method (first, last, avg, min, max) + interpolation_method (str): Optional. Interpolation method (forward_fill, backward_fill) + include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False + + Returns: + DataFrame: A resampled and interpolated dataframe. + ''' + if isinstance(parameters_dict["tag_names"], list) is False: + raise ValueError("tag_names must be a list") + + try: + query = _query_builder(parameters_dict) + + try: + cursor = connection.cursor() + cursor.execute(query) + df = cursor.fetch_all() + cursor.close() + return df + except Exception as e: + logging.exception('error returning dataframe') + raise e + + except Exception as e: + logging.exception('error with interpolate function') + raise e \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/functions/metadata.py b/src/sdk/python/rtdip_sdk/functions/metadata.py new file mode 100644 index 000000000..1b6a67ff8 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/functions/metadata.py @@ -0,0 +1,60 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +sys.path.insert(0, 'src/sdk/python') +import pandas as pd +from rtdip_sdk.functions._query_builder import _query_builder + +def get(connection: object, parameters_dict: dict) -> pd.DataFrame: + ''' + A function to return back the metadata by querying databricks SQL Warehouse using a connection specified by the user. + + The available connectors by RTDIP are Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect. + + The available authentcation methods are Certificate Authentication, Client Secret Authentication or Default Authentication. See documentation. + + This function requires the user to input a dictionary of parameters. (See Attributes table below) + + Args: + connection: Connection chosen by the user (Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect) + parameters_dict: A dictionary of parameters (see Attributes table below) + + Attributes: + buisness_unit (str): Business unit + region (str): Region + asset (str): Asset + data_security_level (str): Level of data security + tag_names (list): (Optional) Either pass a list of tagname/tagnames ["tag_1", "tag_2"] or leave the list blank [] or leave the parameter out completely + + Returns: + DataFrame: A dataframe of metadata. + ''' + try: + query = _query_builder(parameters_dict, metadata=True) + + try: + cursor = connection.cursor() + cursor.execute(query) + df = cursor.fetch_all() + cursor.close() + return df + except Exception as e: + logging.exception('error returning dataframe') + raise e + + except Exception as e: + logging.exception('error returning metadata function') + raise e \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/functions/raw.py b/src/sdk/python/rtdip_sdk/functions/raw.py new file mode 100644 index 000000000..1a5f4bf69 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/functions/raw.py @@ -0,0 +1,67 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import pytz +sys.path.insert(0, 'src/sdk/python') +import pandas as pd +from rtdip_sdk.functions._query_builder import _query_builder + +def get(connection: object, parameters_dict: dict) -> pd.DataFrame: + ''' + A function to return back raw data by querying databricks SQL Warehouse using a connection specified by the user. + + The available connectors by RTDIP are Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect. + + The available authentcation methods are Certificate Authentication, Client Secret Authentication or Default Authentication. See documentation. + + This function requires the user to input a dictionary of parameters. (See Attributes table below) + + Args: + connection: Connection chosen by the user (Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect) + parameters_dict: A dictionary of parameters (see Attributes table below) + + Attributes: + buisness_unit (str): Business unit + region (str): Region + asset (str): Asset + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) + tag_names (list): List of tagname or tagnames ["tag_1", "tag_2"] + start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS) + end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS) + include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False + + Returns: + DataFrame: A dataframe of raw timeseries data. + ''' + try: + query = _query_builder(parameters_dict) + + try: + cursor = connection.cursor() + cursor.execute(query) + df = cursor.fetch_all() + if df['EventTime'][0].tzinfo is None: + df['EventTime'] = df['EventTime'].apply(lambda x: x.replace(tzinfo=pytz.timezone("Etc/UTC"))) + cursor.close() + return df + except Exception as e: + logging.exception('error returning dataframe') + raise e + + except Exception as e: + logging.exception('error with raw function') + raise e \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/functions/resample.py b/src/sdk/python/rtdip_sdk/functions/resample.py new file mode 100644 index 000000000..cafbf02c2 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/functions/resample.py @@ -0,0 +1,70 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +sys.path.insert(0, 'src/sdk/python') +import pandas as pd +from rtdip_sdk.functions._query_builder import _query_builder + +def get(connection: object, parameters_dict: dict) -> pd.DataFrame: + ''' + An RTDIP Resampling function in spark to resample data by querying databricks SQL warehouses using a connection and authentication method specified by the user. This spark resample function will return a resampled dataframe. + + The available connectors by RTDIP are Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect. + + The available authentcation methods are Certificate Authentication, Client Secret Authentication or Default Authentication. See documentation. + + This function requires the user to input a dictionary of parameters. (See Attributes table below) + + Args: + connection: Connection chosen by the user (Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect) + parameters_dict: A dictionary of parameters (see Attributes table below) + + Attributes: + buisness_unit (str): Business unit of the data + region (str): Region + asset (str): Asset + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) + tag_names (list): List of tagname or tagnames ["tag_1", "tag_2"] + start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS) + end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS) + sample_rate (int): The resampling rate (numeric input) + sample_unit (str): The resampling unit (second, minute, day, hour) + agg_method (str): Aggregation Method (first, last, avg, min, max) + include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False + + Returns: + DataFrame: A resampled dataframe. + ''' + if isinstance(parameters_dict["tag_names"], list) is False: + raise ValueError("tag_names must be a list") + + try: + query = _query_builder(parameters_dict) + + try: + cursor = connection.cursor() + cursor.execute(query) + df = cursor.fetch_all() + cursor.close() + return df + except Exception as e: + logging.exception('error returning dataframe') + raise e + + except Exception as e: + logging.exception('error with resampling function') + raise e \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/functions/time_weighted_average.py b/src/sdk/python/rtdip_sdk/functions/time_weighted_average.py new file mode 100644 index 000000000..55b971ff8 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/functions/time_weighted_average.py @@ -0,0 +1,129 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +sys.path.insert(0, 'src/sdk/python') +import pandas as pd +from rtdip_sdk.functions.raw import get as raw_get +from rtdip_sdk.functions.metadata import get as metadata_get +from datetime import datetime, timedelta +import pytz +import numpy as np + +def get(connection: object, parameters_dict: dict) -> pd.DataFrame: + ''' + A function that recieves a dataframe of raw tag data and performs a timeweighted average, returning the results. + + This function requires the input of a pandas dataframe acquired via the rtdip.functions.raw() method and the user to input a dictionary of parameters. (See Attributes table below) + + Pi data points will either have step enabled (True) or step disabled (False). You can specify whether you want step to be fetched by "Pi" or you can set the step parameter to True/False in the dictionary below. + + Args: + connection: Connection chosen by the user (Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect) + parameter_dict (dict): A dictionary of parameters (see Attributes table below) + + Attributes: + buisness_unit (str): Business unit + region (str): Region + asset (str): Asset + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) + tag_names (list): List of tagname or tagnames + start_date (str): Start date (Either a utc date in the format YYYY-MM-DD or a utc datetime in the format YYYY-MM-DDTHH:MM:SS) + end_date (str): End date (Either a utc date in the format YYYY-MM-DD or a utc datetime in the format YYYY-MM-DDTHH:MM:SS) + window_size_mins (int): Window size in minutes + window_length (int): (Optional) add longer window time for the start or end of specified date to cater for edge cases + include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False + step (str/bool): data points with step "enabled" or "disabled". The options for step are "Pi" (string), True or False (bool) + + Returns: + DataFrame: A dataframe containing the time weighted averages. + ''' + try: + datetime_format = "%Y-%m-%dT%H:%M:%S" + utc="Etc/UTC" + + if len(parameters_dict["start_date"]) == 10: + original_start_date = datetime.strptime(parameters_dict["start_date"] + "T00:00:00", datetime_format) + parameters_dict["start_date"] = parameters_dict["start_date"] + "T00:00:00" + else: + original_start_date = datetime.strptime(parameters_dict["start_date"], datetime_format) + + if len(parameters_dict["end_date"]) == 10: + original_end_date = datetime.strptime(parameters_dict["end_date"] + "T23:59:59", datetime_format) + parameters_dict["end_date"] = parameters_dict["end_date"] + "T23:59:59" + else: + original_end_date = datetime.strptime(parameters_dict["end_date"], datetime_format) + + if "window_length" in parameters_dict: + parameters_dict["start_date"] = (datetime.strptime(parameters_dict["start_date"], datetime_format) - timedelta(minutes = parameters_dict["window_length"])).strftime(datetime_format) + parameters_dict["end_date"] = (datetime.strptime(parameters_dict["end_date"], datetime_format) + timedelta(minutes = parameters_dict["window_length"])).strftime(datetime_format) + else: + parameters_dict["start_date"] = (datetime.strptime(parameters_dict["start_date"], datetime_format) - timedelta(minutes = parameters_dict["window_size_mins"])).strftime(datetime_format) + parameters_dict["end_date"] = (datetime.strptime(parameters_dict["end_date"], datetime_format) + timedelta(minutes = parameters_dict["window_size_mins"])).strftime(datetime_format) + + pandas_df = raw_get(connection, parameters_dict) + + pandas_df["EventDate"] = pd.to_datetime(pandas_df["EventTime"]).dt.date + + boundaries_df = pd.DataFrame(columns=["EventTime", "TagName", "Value", "EventDate"]) + for tag in parameters_dict["tag_names"]: + boundaries_df = boundaries_df.append({"EventTime": pd.to_datetime(parameters_dict["start_date"]).replace(tzinfo=pytz.timezone(utc)), "TagName": tag, "Value": 0, "EventDate": datetime.strptime(parameters_dict["start_date"], datetime_format).date()}, ignore_index=True) + boundaries_df = boundaries_df.append({"EventTime": pd.to_datetime(parameters_dict["end_date"]).replace(tzinfo=pytz.timezone(utc)), "TagName": tag, "Value": 0, "EventDate": datetime.strptime(parameters_dict["end_date"], datetime_format).date()}, ignore_index=True) + boundaries_df.set_index(pd.DatetimeIndex(boundaries_df["EventTime"]), inplace=True) + boundaries_df.drop(columns="EventTime", inplace=True) + boundaries_df = boundaries_df.groupby(["TagName"]).resample("{}T".format(str(parameters_dict["window_size_mins"]))).mean().drop(columns="Value") + + #preprocess - add boundaries and time interpolate missing boundary values + preprocess_df = pandas_df.copy() + preprocess_df["EventTime"] = preprocess_df["EventTime"].round("S") + preprocess_df.set_index(["EventTime", "TagName", "EventDate"], inplace=True) + preprocess_df = preprocess_df.join(boundaries_df, how="outer", rsuffix="right") + if parameters_dict["step"] == "pi" or parameters_dict["step"] == "Pi": + metadata_df = metadata_get(connection, parameters_dict) + metadata_df.set_index("TagName", inplace=True) + metadata_df = metadata_df.loc[:, "Step"] + preprocess_df = preprocess_df.merge(metadata_df, left_index=True, right_index=True) + elif parameters_dict["step"] == True: + preprocess_df["Step"] = True + elif parameters_dict["step"] == False: + preprocess_df["Step"] = False + + def process_time_weighted_averages_step(pandas_df): + if pandas_df["Step"].any() == False: + pandas_df = pandas_df.reset_index(level=["TagName", "EventDate"]).sort_index().interpolate(method='time') + shift_raw_df = pandas_df.copy() + shift_raw_df["CalcValue"] = (shift_raw_df.index.to_series().diff().dt.seconds/86400) * shift_raw_df.Value.rolling(2).sum() + time_weighted_averages = shift_raw_df.resample("{}T".format(str(parameters_dict["window_size_mins"])), closed="right", label="right").CalcValue.sum() * 0.5 / parameters_dict["window_size_mins"] * 24 * 60 + return time_weighted_averages + else: + pandas_df = pandas_df.reset_index(level=["TagName", "EventDate"]).sort_index().interpolate(method='pad', limit_direction='forward') + shift_raw_df = pandas_df.copy() + shift_raw_df["CalcValue"] = (shift_raw_df.index.to_series().diff().dt.seconds/86400) * shift_raw_df.Value.shift(1) + time_weighted_averages = shift_raw_df.resample("{}T".format(str(parameters_dict["window_size_mins"])), closed="right", label="right").CalcValue.sum() / parameters_dict["window_size_mins"] * 24 * 60 + return time_weighted_averages + + #calculate time weighted averages + time_weighted_averages = preprocess_df.groupby(["TagName"]).apply(process_time_weighted_averages_step).reset_index() + time_weighted_averages = time_weighted_averages.melt(id_vars="TagName", var_name="EventTime", value_name="Value").set_index("EventTime").sort_values(by=["TagName", "EventTime"]) + time_weighted_averages_datetime = time_weighted_averages.index.to_pydatetime() + weighted_averages_timezones = np.array([z.replace(tzinfo=pytz.timezone(utc)) for z in time_weighted_averages_datetime]) + time_weighted_averages = time_weighted_averages[(original_start_date.replace(tzinfo=pytz.timezone(utc)) < weighted_averages_timezones) & (weighted_averages_timezones <= original_end_date.replace(tzinfo=pytz.timezone(utc)) + timedelta(seconds = 1))] + + return time_weighted_averages + + except Exception as e: + logging.exception('error with time weighted average function') + raise e \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/odbc/__init__.py b/src/sdk/python/rtdip_sdk/odbc/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/odbc/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/sdk/python/rtdip_sdk/odbc/connection_interface.py b/src/sdk/python/rtdip_sdk/odbc/connection_interface.py new file mode 100644 index 000000000..d7947884a --- /dev/null +++ b/src/sdk/python/rtdip_sdk/odbc/connection_interface.py @@ -0,0 +1,32 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, 'src/sdk/python') +from abc import ABCMeta, abstractmethod +from rtdip_sdk.odbc.cursor_interface import CursorInterface + +class ConnectionInterface(metaclass=ABCMeta): + + @classmethod + def __subclasshook__(cls, subclass): + return NotImplemented + + @abstractmethod + def close(self) -> None: + pass + + @abstractmethod + def cursor(self) -> CursorInterface: + pass \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/odbc/cursor_interface.py b/src/sdk/python/rtdip_sdk/odbc/cursor_interface.py new file mode 100644 index 000000000..d554dea1a --- /dev/null +++ b/src/sdk/python/rtdip_sdk/odbc/cursor_interface.py @@ -0,0 +1,32 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABCMeta, abstractmethod + +class CursorInterface(metaclass=ABCMeta): + @classmethod + def __subclasshook__(cls, subclass): + return NotImplemented + + @abstractmethod + def execute(self, query: str) -> None: + pass + + @abstractmethod + def fetch_all(self) -> list: + pass + + @abstractmethod + def close(self) -> None: + pass \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/odbc/db_sql_connector.py b/src/sdk/python/rtdip_sdk/odbc/db_sql_connector.py new file mode 100644 index 000000000..90d199887 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/odbc/db_sql_connector.py @@ -0,0 +1,110 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, 'src/sdk/python') +from databricks import sql +import pandas as pd +from rtdip_sdk.odbc.connection_interface import ConnectionInterface +from rtdip_sdk.odbc.cursor_interface import CursorInterface +import logging + +class DatabricksSQLConnection(ConnectionInterface): + """ + The Databricks SQL Connector for Python is a Python library that allows you to use Python code to run SQL commands on Databricks clusters and Databricks SQL warehouses. + + The connection class represents a connection to a database and uses the Databricks SQL Connector API's for Python to intereact with cluster/jobs. + To find details for SQL warehouses server_hostname and http_path location to the SQL Warehouse tab in the documentation. + + Args: + server_hostname: Server hostname for the cluster or SQL Warehouse + http_path: Http path for the cluster or SQL Warehouse + access_token: Azure AD token + """ + def __init__(self, server_hostname: str, http_path: str, access_token: str) -> None: + #call auth method + self.connection = sql.connect( + server_hostname=server_hostname, + http_path=http_path, + access_token=access_token) + + def close(self) -> None: + """Closes connection to database.""" + try: + self.connection.close() + except Exception as e: + logging.exception('error while closing connection') + raise e + + def cursor(self) -> object: + """ + Intiates the cursor and returns it. + + Returns: + DatabricksSQLCursor: Object to represent a databricks workspace with methods to interact with clusters/jobs. + """ + try: + return DatabricksSQLCursor(self.connection.cursor()) + except Exception as e: + logging.exception('error with cursor object') + raise e + + +class DatabricksSQLCursor(CursorInterface): + """ + Object to represent a databricks workspace with methods to interact with clusters/jobs. + + Args: + cursor: controls execution of commands on cluster or SQL Warehouse + """ + def __init__(self, cursor: object) -> None: + self.cursor = cursor + + def execute(self, query: str) -> None: + """ + Prepares and runs a database query. + + Args: + query: sql query to execute on the cluster or SQL Warehouse + """ + try: + self.cursor.execute(query) + except Exception as e: + logging.exception('error while executing the query') + raise e + + def fetch_all(self) -> list: + """ + Gets all rows of a query. + + Returns: + list: list of results + """ + try: + result = self.cursor.fetchall() + cols = [column[0] for column in self.cursor.description] + df = pd.DataFrame(result) + df.columns = cols + return df + except Exception as e: + logging.exception('error while fetching the rows of a query') + raise e + + def close(self) -> None: + """Closes the cursor.""" + try: + self.cursor.close() + except Exception as e: + logging.exception('error while closing the cursor') + raise e \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/odbc/pyodbc_sql_connector.py b/src/sdk/python/rtdip_sdk/odbc/pyodbc_sql_connector.py new file mode 100644 index 000000000..e8b599729 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/odbc/pyodbc_sql_connector.py @@ -0,0 +1,128 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, 'src/sdk/python') +import pyodbc +import pandas as pd +from rtdip_sdk.odbc.connection_interface import ConnectionInterface +from rtdip_sdk.odbc.cursor_interface import CursorInterface +import logging + +class PYODBCSQLConnection(ConnectionInterface): + """ + PYODBC is an open source python module which allows access to ODBC databases. + This allows the user to connect through ODBC to data in azure databricks clusters or sql warehouses. + + Uses the databricks API's (2.0) to connect to the sql server. + + Args: + driver_path: Driver installed to work with PYODBC + server_hostname: Server hostname for the cluster or SQL Warehouse + http_path: Http path for the cluster or SQL Warehouse + access_token: Azure AD Token + + Note 1: + More fields can be configured here in the connection ie PORT, Schema, etc. + + Note 2: + When using Unix, Linux or Mac OS brew installation of PYODBC is required for connection. + """ + def __init__(self, driver_path: str, server_hostname: str, http_path: str, access_token: str) -> None: + + self.connection = pyodbc.connect('Driver=' + driver_path +';' + + 'HOST=' + server_hostname + ';' + + 'PORT=443;' + + 'Schema=default;' + + 'SparkServerType=3;' + + 'AuthMech=11;' + + 'UID=token;' + + #'PWD=' + access_token+ ";" + + 'Auth_AccessToken='+ access_token +';' + 'ThriftTransport=2;' + + 'SSL=1;' + + 'HTTPPath=' + http_path, + autocommit=True) + + def close(self) -> None: + """Closes connection to database.""" + try: + self.connection.close() + except Exception as e: + logging.exception('error while closing the connection') + raise e + + def cursor(self) -> object: + """ + Intiates the cursor and returns it. + + Returns: + PYODBCSQLCursor: Object to represent a databricks workspace with methods to interact with clusters/jobs. + """ + try: + return PYODBCSQLCursor(self.connection.cursor()) + except Exception as e: + logging.exception('error with cursor object') + raise e + + +class PYODBCSQLCursor(CursorInterface): + """ + Object to represent a databricks workspace with methods to interact with clusters/jobs. + + Args: + cursor: controls execution of commands on cluster or SQL Warehouse + """ + def __init__(self, cursor: object) -> None: + self.cursor = cursor + + def execute(self, query: str) -> None: + """ + Prepares and runs a database query. + + Args: + query: sql query to execute on the cluster or SQL Warehouse + """ + try: + self.cursor.execute(query) + + except Exception as e: + logging.exception('error while executing the query') + raise e + + def fetch_all(self) -> list: + """ + Gets all rows of a query. + + Returns: + list: list of results + """ + try: + result = self.cursor.fetchall() + cols = [column[0] for column in self.cursor.description] + result = [list(x) for x in result] + df = pd.DataFrame(result) + df.columns = cols + return df + except Exception as e: + logging.exception('error while fetching rows from the query') + raise e + + def close(self) -> None: + """Closes the cursor.""" + try: + self.cursor.close() + except Exception as e: + logging.exception('error while closing the cursor') + raise e \ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/odbc/turbodbc_sql_connector.py b/src/sdk/python/rtdip_sdk/odbc/turbodbc_sql_connector.py new file mode 100644 index 000000000..f310d7d03 --- /dev/null +++ b/src/sdk/python/rtdip_sdk/odbc/turbodbc_sql_connector.py @@ -0,0 +1,130 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, 'src/sdk/python') +from turbodbc import connect, make_options, Megabytes +import pandas as pd +from rtdip_sdk.odbc.connection_interface import ConnectionInterface +from rtdip_sdk.odbc.cursor_interface import CursorInterface +import logging + +class TURBODBCSQLConnection(ConnectionInterface): + """ + Turbodbc is a python module used to access relational databases through an ODBC interface. It will allow a user to connect to databricks clusters or sql warehouses. + + Turbodbc offers built-in NumPy support allowing it to be much faster for processing compared to other connectors. + To find details for SQL warehouses server_hostname and http_path location to the SQL Warehouse tab in the documentation. + + Args: + server_hostname: hostname for the cluster or SQL Warehouse + http_path: Http path for the cluster or SQL Warehouse + access_token: Azure AD Token + + Note: + More fields such as driver can be configured upon extension. + """ + def __init__(self, server_hostname: str, http_path: str, access_token: str) -> None: + options = make_options( + autocommit=True, + read_buffer_size=Megabytes(100), + use_async_io=True) + self.connection = connect(Driver="Simba Spark ODBC Driver", + Server=server_hostname, + HOST=server_hostname, + PORT=443, + SparkServerType=3, + Schema="default", + ThriftTransport=2, + SSL=1, + AuthMech=11, + Auth_AccessToken=access_token, + Auth_Flow=0, + HTTPPath=http_path, + UseNativeQuery=1, + FastSQLPrepare=1, + ApplyFastSQLPrepareToAllQueries=1, + DisableLimitZero=1, + EnableAsyncExec=1, + turbodbc_options=options) + + def close(self) -> None: + """Closes connection to database.""" + try: + self.connection.close() + except Exception as e: + logging.exception('error while closing the connection') + raise e + + def cursor(self) -> object: + """ + Intiates the cursor and returns it. + + Returns: + TURBODBCSQLCursor: Object to represent a databricks workspace with methods to interact with clusters/jobs. + """ + try: + return TURBODBCSQLCursor(self.connection.cursor()) + except Exception as e: + logging.exception('error with cursor object') + raise e + + +class TURBODBCSQLCursor(CursorInterface): + """ + Object to represent a databricks workspace with methods to interact with clusters/jobs. + + Args: + cursor: controls execution of commands on cluster or SQL Warehouse + """ + def __init__(self, cursor: object) -> None: + self.cursor = cursor + + def execute(self, query: str) -> None: + """ + Prepares and runs a database query. + + Args: + query: sql query to execute on the cluster or SQL Warehouse + """ + try: + self.cursor.execute(query) + except Exception as e: + logging.exception('error while executing the query') + raise e + + def fetch_all(self) -> list: + """ + Gets all rows of a query. + + Returns: + list: list of results + """ + try: + result = self.cursor.fetchall() + cols = [column[0] for column in self.cursor.description] + df = pd.DataFrame(result) + df.columns = cols + return df + except Exception as e: + logging.exception('error while fetching the rows from the query') + raise e + + def close(self) -> None: + """Closes the cursor.""" + try: + self.cursor.close() + except Exception as e: + logging.exception('error while closing the cursor') + raise e \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/api/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/api/auth/__init__.py b/tests/api/auth/__init__.py new file mode 100644 index 000000000..e2c93f658 --- /dev/null +++ b/tests/api/auth/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from pytest_mock import MockerFixture \ No newline at end of file diff --git a/tests/api/auth/test_auth_azuread.py b/tests/api/auth/test_auth_azuread.py new file mode 100644 index 000000000..fa0afe747 --- /dev/null +++ b/tests/api/auth/test_auth_azuread.py @@ -0,0 +1,34 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_mock import MockerFixture +from src.api.auth import azuread +from azure.identity import DefaultAzureCredential +from azure.core.credentials import AccessToken + +def test_auth_azuread_no_auth_header(mocker: MockerFixture): + mocker.patch("src.sdk.python.rtdip_sdk.authentication.authenticate.DefaultAuth.authenticate", return_value=DefaultAzureCredential) + mocker.patch("azure.identity.DefaultAzureCredential.get_token", return_value = AccessToken) + mocker.patch("azure.core.credentials.AccessToken.token", return_value = "token") + + token = azuread.get_azure_ad_token(None) + assert token.return_value == "token" + +def test_auth_azuread_bearer_token(mocker: MockerFixture): + token = azuread.get_azure_ad_token("Bearer token") + assert token == "token" + +def test_auth_azuread_bearer_token_no_prefix(mocker: MockerFixture): + token = azuread.get_azure_ad_token("token") + assert token == "token" \ No newline at end of file diff --git a/tests/api/v1/__init__.py b/tests/api/v1/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/api/v1/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/api/v1/api_test_objects.py b/tests/api/v1/api_test_objects.py new file mode 100644 index 000000000..4eb4ff923 --- /dev/null +++ b/tests/api/v1/api_test_objects.py @@ -0,0 +1,89 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_mock import MockerFixture +from tests.sdk.python.rtdip_sdk.odbc.test_db_sql_connector import MockedDBConnection +from tests.sdk.python.rtdip_sdk.functions.test_raw import DATABRICKS_SQL_CONNECT + +BASE_MOCKED_PARAMETER_DICT = { + "business_unit": "mocked-buiness-unit", + "region": "mocked-region", + "asset": "mocked-asset", + "data_security_level": "mocked-data-security-level", + } + +METADATA_MOCKED_PARAMETER_DICT = BASE_MOCKED_PARAMETER_DICT.copy() +METADATA_MOCKED_PARAMETER_DICT["tag_name"] = "MOCKED-TAGNAME1" + +METADATA_MOCKED_PARAMETER_ERROR_DICT = METADATA_MOCKED_PARAMETER_DICT.copy() +METADATA_MOCKED_PARAMETER_ERROR_DICT.pop("business_unit") + +METADATA_POST_MOCKED_PARAMETER_DICT = METADATA_MOCKED_PARAMETER_DICT.copy() +METADATA_POST_MOCKED_PARAMETER_DICT.pop("tag_name") + +METADATA_POST_BODY_MOCKED_PARAMETER_DICT = {} +METADATA_POST_BODY_MOCKED_PARAMETER_DICT["tag_name"] = ["MOCKED-TAGNAME1", "MOCKED-TAGNAME2"] + +RAW_MOCKED_PARAMETER_DICT = BASE_MOCKED_PARAMETER_DICT.copy() +RAW_MOCKED_PARAMETER_DICT["data_type"] = "mocked-data-type" +RAW_MOCKED_PARAMETER_DICT["tag_name"] = "MOCKED-TAGNAME1" +RAW_MOCKED_PARAMETER_DICT["include_bad_data"] = True +RAW_MOCKED_PARAMETER_DICT["start_date"] = "2011-01-01" +RAW_MOCKED_PARAMETER_DICT["end_date"] = "2011-01-02" + +RAW_POST_MOCKED_PARAMETER_DICT = RAW_MOCKED_PARAMETER_DICT.copy() +RAW_POST_MOCKED_PARAMETER_DICT.pop("tag_name") + +RAW_POST_BODY_MOCKED_PARAMETER_DICT = {} +RAW_POST_BODY_MOCKED_PARAMETER_DICT["tag_name"] = ["MOCKED-TAGNAME1", "MOCKED-TAGNAME2"] + +RAW_MOCKED_PARAMETER_ERROR_DICT = RAW_MOCKED_PARAMETER_DICT.copy() +RAW_MOCKED_PARAMETER_ERROR_DICT.pop("start_date") + +RESAMPLE_MOCKED_PARAMETER_DICT = RAW_MOCKED_PARAMETER_DICT.copy() +RESAMPLE_MOCKED_PARAMETER_ERROR_DICT = RAW_MOCKED_PARAMETER_ERROR_DICT.copy() + +RESAMPLE_MOCKED_PARAMETER_DICT["sample_rate"] = 1 +RESAMPLE_MOCKED_PARAMETER_DICT["sample_unit"] = "minute" +RESAMPLE_MOCKED_PARAMETER_DICT["agg_method"] = "avg" +RESAMPLE_MOCKED_PARAMETER_ERROR_DICT["sample_rate"] = 1 +RESAMPLE_MOCKED_PARAMETER_ERROR_DICT["sample_unit"] = "minute" +RESAMPLE_MOCKED_PARAMETER_ERROR_DICT["agg_method"] = "avg" + +RESAMPLE_POST_MOCKED_PARAMETER_DICT = RESAMPLE_MOCKED_PARAMETER_DICT.copy() +RESAMPLE_POST_MOCKED_PARAMETER_DICT.pop("tag_name") + +RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT = {} +RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT["tag_name"] = ["MOCKED-TAGNAME1", "MOCKED-TAGNAME2"] + +INTERPOLATE_MOCKED_PARAMETER_DICT = RESAMPLE_MOCKED_PARAMETER_DICT.copy() +INTERPOLATE_MOCKED_PARAMETER_ERROR_DICT = RESAMPLE_MOCKED_PARAMETER_ERROR_DICT.copy() + +INTERPOLATE_MOCKED_PARAMETER_DICT["interpolation_method"] = "forward_fill" +INTERPOLATE_MOCKED_PARAMETER_ERROR_DICT["interpolation_method"] = "forward_fill" + +INTERPOLATE_POST_MOCKED_PARAMETER_DICT = INTERPOLATE_MOCKED_PARAMETER_DICT.copy() +INTERPOLATE_POST_MOCKED_PARAMETER_DICT.pop("tag_name") + +INTERPOLATE_POST_BODY_MOCKED_PARAMETER_DICT = {} +INTERPOLATE_POST_BODY_MOCKED_PARAMETER_DICT["tag_name"] = ["MOCKED-TAGNAME1", "MOCKED-TAGNAME2"] + +TEST_HEADERS = {"Authorization": "Bearer Test Token"} +TEST_HEADERS_POST = {"Authorization": "Bearer Test Token", "Content-Type": "application/json"} + +def mocker_setup(mocker: MockerFixture, patch_method, test_data, side_effect=None): + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection(), side_effect=side_effect) + mocker.patch(patch_method, return_value = test_data) + mocker.patch("src.api.auth.azuread.get_azure_ad_token", return_value = "token") + return mocker \ No newline at end of file diff --git a/tests/api/v1/test_api_interpolate.py b/tests/api/v1/test_api_interpolate.py new file mode 100644 index 000000000..171a14b39 --- /dev/null +++ b/tests/api/v1/test_api_interpolate.py @@ -0,0 +1,99 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_mock import MockerFixture +import pandas as pd +from datetime import datetime +from tests.sdk.python.rtdip_sdk.odbc.test_db_sql_connector import MockedDBConnection +from tests.sdk.python.rtdip_sdk.functions.test_raw import DATABRICKS_SQL_CONNECT +from tests.api.v1.api_test_objects import INTERPOLATE_MOCKED_PARAMETER_DICT, INTERPOLATE_MOCKED_PARAMETER_ERROR_DICT, INTERPOLATE_POST_MOCKED_PARAMETER_DICT, INTERPOLATE_POST_BODY_MOCKED_PARAMETER_DICT, mocker_setup, TEST_HEADERS +from fastapi.testclient import TestClient +from src.api.v1 import app + +MOCK_METHOD = "src.sdk.python.rtdip_sdk.functions.interpolate.get" +MOCK_API_NAME = "/api/v1/events/interpolate" + +def test_api_interpolate_get_success(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=INTERPOLATE_MOCKED_PARAMETER_DICT) + actual = response.text + expected = test_data.to_json(orient="table", index=False).replace("Z", "000+00:00") + + assert response.status_code == 200 + assert actual == expected + +def test_api_interpolate_get_validation_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=INTERPOLATE_MOCKED_PARAMETER_ERROR_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["query","start_date"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_interpolate_get_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data, Exception("Error Connecting to Database")) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=INTERPOLATE_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' + +def test_api_interpolate_post_success(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=INTERPOLATE_POST_MOCKED_PARAMETER_DICT, json=INTERPOLATE_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + expected = test_data.to_json(orient="table", index=False).replace("Z", "000+00:00") + + assert response.status_code == 200 + assert actual == expected + +def test_api_interpolate_post_validation_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=INTERPOLATE_MOCKED_PARAMETER_ERROR_DICT, json=INTERPOLATE_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["query","start_date"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_interpolate_post_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data, Exception("Error Connecting to Database")) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=INTERPOLATE_MOCKED_PARAMETER_DICT, json=INTERPOLATE_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' \ No newline at end of file diff --git a/tests/api/v1/test_api_metadata.py b/tests/api/v1/test_api_metadata.py new file mode 100644 index 000000000..a0aab804f --- /dev/null +++ b/tests/api/v1/test_api_metadata.py @@ -0,0 +1,118 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_mock import MockerFixture +import pandas as pd +from tests.api.v1.api_test_objects import METADATA_MOCKED_PARAMETER_DICT, METADATA_MOCKED_PARAMETER_ERROR_DICT, METADATA_POST_MOCKED_PARAMETER_DICT, METADATA_POST_BODY_MOCKED_PARAMETER_DICT, mocker_setup, TEST_HEADERS +from fastapi.testclient import TestClient +from src.api.v1 import app + +MOCK_METHOD = "src.sdk.python.rtdip_sdk.functions.metadata.get" +MOCK_API_NAME = "/api/v1/metadata" +TEST_DATA = pd.DataFrame({"TagName": ["TestTag"], "UoM": ["UoM1"], "Description": ["Test Description"]}) + +def test_api_metadata_get_tags_provided_success(mocker: MockerFixture): + client = TestClient(app) + + mocker = mocker_setup(mocker, MOCK_METHOD, TEST_DATA) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=METADATA_MOCKED_PARAMETER_DICT) + actual = response.text + expected = TEST_DATA.to_json(orient="table", index=False) + + assert response.status_code == 200 + assert actual == expected + +def test_api_metadata_get_no_tags_provided_success(mocker: MockerFixture): + client = TestClient(app) + + mocker = mocker_setup(mocker, MOCK_METHOD, TEST_DATA) + + METADATA_MOCKED_PARAMETER_NO_TAG_DICT = METADATA_MOCKED_PARAMETER_DICT.copy() + METADATA_MOCKED_PARAMETER_NO_TAG_DICT.pop("tag_name") + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=METADATA_MOCKED_PARAMETER_NO_TAG_DICT) + actual = response.text + expected = TEST_DATA.to_json(orient="table", index=False) + + assert response.status_code == 200 + assert actual == expected + +def test_api_metadata_get_validation_error(mocker: MockerFixture): + client = TestClient(app) + + mocker = mocker_setup(mocker, MOCK_METHOD, TEST_DATA) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=METADATA_MOCKED_PARAMETER_ERROR_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["query","business_unit"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_raw_get_error(mocker: MockerFixture): + client = TestClient(app) + + mocker = mocker_setup(mocker, MOCK_METHOD, TEST_DATA, Exception("Error Connecting to Database")) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=METADATA_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' + +def test_api_metadata_post_tags_provided_success(mocker: MockerFixture): + client = TestClient(app) + + mocker = mocker_setup(mocker, MOCK_METHOD, TEST_DATA) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=METADATA_POST_MOCKED_PARAMETER_DICT, json=METADATA_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + expected = TEST_DATA.to_json(orient="table", index=False) + + assert response.status_code == 200 + assert actual == expected + +def test_api_metadata_post_no_tags_provided_error(mocker: MockerFixture): + client = TestClient(app) + + mocker = mocker_setup(mocker, MOCK_METHOD, TEST_DATA) + + METADATA_MOCKED_PARAMETER_NO_TAG_DICT = METADATA_MOCKED_PARAMETER_DICT.copy() + METADATA_MOCKED_PARAMETER_NO_TAG_DICT.pop("tag_name") + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=METADATA_MOCKED_PARAMETER_NO_TAG_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["body"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_metadata_post_validation_error(mocker: MockerFixture): + client = TestClient(app) + + mocker = mocker_setup(mocker, MOCK_METHOD, TEST_DATA) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=METADATA_MOCKED_PARAMETER_ERROR_DICT, json=METADATA_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["query","business_unit"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_raw_post_error(mocker: MockerFixture): + client = TestClient(app) + + mocker = mocker_setup(mocker, MOCK_METHOD, TEST_DATA, Exception("Error Connecting to Database")) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=METADATA_MOCKED_PARAMETER_DICT, json=METADATA_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' \ No newline at end of file diff --git a/tests/api/v1/test_api_raw.py b/tests/api/v1/test_api_raw.py new file mode 100644 index 000000000..859e30c70 --- /dev/null +++ b/tests/api/v1/test_api_raw.py @@ -0,0 +1,98 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_mock import MockerFixture +import pandas as pd +from datetime import datetime +from tests.api.v1.api_test_objects import RAW_MOCKED_PARAMETER_DICT, RAW_MOCKED_PARAMETER_ERROR_DICT, RAW_POST_MOCKED_PARAMETER_DICT, RAW_POST_BODY_MOCKED_PARAMETER_DICT, mocker_setup, TEST_HEADERS +from fastapi.testclient import TestClient +from src.api.v1 import app +import json + +MOCK_METHOD = "src.sdk.python.rtdip_sdk.functions.raw.get" +MOCK_API_NAME = "/api/v1/events/raw" + +def test_api_raw_get_success(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Status": ["Good"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=RAW_MOCKED_PARAMETER_DICT) + actual = response.text + expected = test_data.to_json(orient="table", index=False).replace("Z", "000+00:00") + + assert response.status_code == 200 + assert actual == expected + +def test_api_raw_get_validation_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Status": ["Good"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=RAW_MOCKED_PARAMETER_ERROR_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["query","start_date"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_raw_get_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Status": ["Good"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data, Exception("Error Connecting to Database")) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=RAW_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' + +def test_api_raw_post_success(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Status": ["Good"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=RAW_POST_MOCKED_PARAMETER_DICT, json=RAW_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + expected = test_data.to_json(orient="table", index=False).replace("Z", "000+00:00") + + assert response.status_code == 200 + assert actual == expected + +def test_api_raw_post_validation_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Status": ["Good"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=RAW_MOCKED_PARAMETER_ERROR_DICT, json=RAW_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["query","start_date"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_raw_post_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Status": ["Good"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data, Exception("Error Connecting to Database")) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=RAW_MOCKED_PARAMETER_DICT, json=RAW_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' \ No newline at end of file diff --git a/tests/api/v1/test_api_resample.py b/tests/api/v1/test_api_resample.py new file mode 100644 index 000000000..b9fedee5b --- /dev/null +++ b/tests/api/v1/test_api_resample.py @@ -0,0 +1,99 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_mock import MockerFixture +import pandas as pd +from datetime import datetime +from tests.sdk.python.rtdip_sdk.odbc.test_db_sql_connector import MockedDBConnection +from tests.sdk.python.rtdip_sdk.functions.test_raw import DATABRICKS_SQL_CONNECT +from tests.api.v1.api_test_objects import RESAMPLE_MOCKED_PARAMETER_DICT, RESAMPLE_MOCKED_PARAMETER_ERROR_DICT, RESAMPLE_POST_MOCKED_PARAMETER_DICT, RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT, mocker_setup, TEST_HEADERS +from fastapi.testclient import TestClient +from src.api.v1 import app + +MOCK_METHOD = "src.sdk.python.rtdip_sdk.functions.resample.get" +MOCK_API_NAME = "/api/v1/events/resample" + +def test_api_resample_get_success(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=RESAMPLE_MOCKED_PARAMETER_DICT) + actual = response.text + expected = test_data.to_json(orient="table", index=False).replace("Z", "000+00:00") + + assert response.status_code == 200 + assert actual == expected + +def test_api_resample_get_validation_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=RESAMPLE_MOCKED_PARAMETER_ERROR_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["query","start_date"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_resample_get_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data, Exception("Error Connecting to Database")) + + response = client.get(MOCK_API_NAME, headers=TEST_HEADERS, params=RESAMPLE_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' + +def test_api_resample_post_success(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=RESAMPLE_POST_MOCKED_PARAMETER_DICT, json=RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + expected = test_data.to_json(orient="table", index=False).replace("Z", "000+00:00") + + assert response.status_code == 200 + assert actual == expected + +def test_api_resample_post_validation_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=RESAMPLE_MOCKED_PARAMETER_ERROR_DICT, json=RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 422 + assert actual == '{"detail":[{"loc":["query","start_date"],"msg":"field required","type":"value_error.missing"}]}' + +def test_api_resample_post_error(mocker: MockerFixture): + client = TestClient(app) + + test_data = pd.DataFrame({"EventTime": [datetime.utcnow()], "TagName": ["TestTag"], "Value": [1.01]}) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data, Exception("Error Connecting to Database")) + + response = client.post(MOCK_API_NAME, headers=TEST_HEADERS, params=RESAMPLE_MOCKED_PARAMETER_DICT, json=RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' \ No newline at end of file diff --git a/tests/api/v1/test_api_utilities.py b/tests/api/v1/test_api_utilities.py new file mode 100644 index 000000000..2b3cc308b --- /dev/null +++ b/tests/api/v1/test_api_utilities.py @@ -0,0 +1,39 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_mock import MockerFixture +from fastapi.testclient import TestClient +from src.api.v1 import app + +def test_api_home(mocker: MockerFixture): + client = TestClient(app) + + response = client.get("/") + + assert response.status_code == 200 + +def test_api_docs(mocker: MockerFixture): + client = TestClient(app) + + response = client.get("/docs") + + assert response.status_code == 200 + +def test_api_redoc(mocker: MockerFixture): + client = TestClient(app) + + response = client.get("/redoc") + + assert response.status_code == 200 + \ No newline at end of file diff --git a/tests/apps/__init__.py b/tests/apps/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/apps/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/apps/docs/__init__.py b/tests/apps/docs/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/apps/docs/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/apps/docs/test_config.py b/tests/apps/docs/test_config.py new file mode 100644 index 000000000..581d55f6e --- /dev/null +++ b/tests/apps/docs/test_config.py @@ -0,0 +1,21 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +def test_json_config(): + with open('src/apps/docs/staticwebapp.config.json') as json_file: + data = json.load(json_file) + assert "auth" in data + assert "routes" in data diff --git a/tests/sdk/__init__.py b/tests/sdk/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/sdk/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/sdk/python/__init__.py b/tests/sdk/python/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/sdk/python/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/sdk/python/rtdip_sdk/__init__.py b/tests/sdk/python/rtdip_sdk/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/sdk/python/rtdip_sdk/authentication/__init__.py b/tests/sdk/python/rtdip_sdk/authentication/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/authentication/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/sdk/python/rtdip_sdk/authentication/test_authenticate.py b/tests/sdk/python/rtdip_sdk/authentication/test_authenticate.py new file mode 100644 index 000000000..b97cb9ace --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/authentication/test_authenticate.py @@ -0,0 +1,69 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_mock import MockerFixture +from src.sdk.python.rtdip_sdk.authentication.authenticate import CertificateAuth, ClientSecretAuth, Authenticator, DefaultAuth +import pytest + +TENANT_ID = "tenantid123" +CLIENT_ID = "clientid123" +CLIENT_SECRET = "clientsecret123" +CERTIFICATE_PATH = "/test/test-certificate.pem" +STORAGE_ACCOUNT= "teststorageaccount" +FILE_SYSTEM = "test" + +class MockedAuthClass: + def authenticate(self) -> object: + return object + + mocked_client_secret_auth = ClientSecretAuth(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + result = mocked_client_secret_auth.authenticate() + + assert isinstance(result, object) + assert result._tenant_id == "tenantid123" + +def test_certificate_auth(mocker: MockerFixture): + mocker.patch("src.sdk.python.rtdip_sdk.authentication.authenticate.CertificateCredential") + + mocked_certificate_auth = CertificateAuth(TENANT_ID, CLIENT_ID, CERTIFICATE_PATH) + result = mocked_certificate_auth.authenticate() + + assert isinstance(result, object) + +def test_default_auth(): + mocked_default_auth = DefaultAuth() + result = mocked_default_auth.authenticate() + + assert isinstance(result, object) + +def test_client_secret_auth_fails(mocker: MockerFixture): + mocker.patch("src.sdk.python.rtdip_sdk.authentication.authenticate.ClientSecretCredential", side_effect = Exception) + mocked_client_secret_auth = ClientSecretAuth(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + + with pytest.raises(Exception): + assert mocked_client_secret_auth.authenticate() + +def test_certificate_auth_fails(mocker: MockerFixture): + mocker.patch("src.sdk.python.rtdip_sdk.authentication.authenticate.CertificateCredential", side_effect = Exception) + mocked_certificate_auth = CertificateAuth(TENANT_ID, CLIENT_ID, CERTIFICATE_PATH) + + with pytest.raises(Exception): + assert mocked_certificate_auth.authenticate() + +def test_default_auth_fails(mocker: MockerFixture): + mocker.patch("src.sdk.python.rtdip_sdk.authentication.authenticate.DefaultAzureCredential", side_effect = Exception) + mocked_default_auth = DefaultAuth() + + with pytest.raises(Exception): + assert mocked_default_auth.authenticate() \ No newline at end of file diff --git a/tests/sdk/python/rtdip_sdk/functions/__init__.py b/tests/sdk/python/rtdip_sdk/functions/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/functions/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/sdk/python/rtdip_sdk/functions/test_interpolate.py b/tests/sdk/python/rtdip_sdk/functions/test_interpolate.py new file mode 100644 index 000000000..b19163fa8 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/functions/test_interpolate.py @@ -0,0 +1,73 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, '.') +import pandas as pd +import pytest +from pytest_mock import MockerFixture +from tests.sdk.python.rtdip_sdk.odbc.test_db_sql_connector import MockedDBConnection, MockedCursor +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from src.sdk.python.rtdip_sdk.functions.interpolate import get as interpolate_get + +SERVER_HOSTNAME = "mock.cloud.databricks.com" +HTTP_PATH = "sql/mock/mock-test" +ACCESS_TOKEN = "mock_databricks_token" +DATABRICKS_SQL_CONNECT = 'databricks.sql.connect' +DATABRICKS_SQL_CONNECT_CURSOR = 'databricks.sql.connect.cursor' +MOCKED_QUERY='SELECT a.EventTime, a.TagName, last_value(b.Value, true) OVER (PARTITION BY a.TagName ORDER BY a.EventTime ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS Value FROM (SELECT explode(sequence(to_timestamp(\'2011-01-01T00:00:00\'), to_timestamp(\'2011-01-02T23:59:59\'), INTERVAL \'1 hour\')) AS EventTime, explode(array(\'"MOCKED-TAGNAME"\')) AS TagName) a LEFT OUTER JOIN (SELECT DISTINCT TagName, w.start AS EventTime, avg(Value) OVER (PARTITION BY TagName, w.start ORDER BY EventTime ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS Value FROM (SELECT EventTime, WINDOW(EventTime, \'1 hour\') w, TagName, Status, Value FROM mocked-buiness-unit.sensors.mocked-asset_mocked-data-security-level_events_mocked-data-type WHERE EventDate BETWEEN to_date(\'2011-01-01T00:00:00\') AND to_date(\'2011-01-02T23:59:59\') AND EventTime BETWEEN to_timestamp(\'2011-01-01T00:00:00\') AND to_timestamp(\'2011-01-02T23:59:59\') AND TagName in (\'MOCKED-TAGNAME\') AND Status = \'Good\')) b ON a.EventTime = b.EventTime AND a.TagName = b.TagName' +MOCKED_PARAMETER_DICT = { + "business_unit": "mocked-buiness-unit", + "region": "mocked-region", + "asset": "mocked-asset", + "data_security_level": "mocked-data-security-level", + "data_type": "mocked-data-type", + "tag_names": ["MOCKED-TAGNAME"], + "start_date": "2011-01-01", + "end_date": "2011-01-02", + "sample_rate": "1", + "sample_unit": "hour", + "agg_method": "avg", + "interpolation_method": "forward_fill", + "include_bad_data": False + } + +def test_interpolate(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data={'a': [1], 'b': [2], 'c': [3], 'd': [4]})) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + actual = interpolate_get(mocked_connection, MOCKED_PARAMETER_DICT) + + mocked_cursor.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_QUERY) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + +def test_interpolate_fails(mocker: MockerFixture): + mocker.spy(MockedDBConnection, "cursor") + mocker.spy(MockedCursor, "execute") + mocker.patch.object(MockedCursor, "fetchall", side_effect=Exception) + mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + interpolate_get(mocked_connection, MOCKED_PARAMETER_DICT) \ No newline at end of file diff --git a/tests/sdk/python/rtdip_sdk/functions/test_metadata.py b/tests/sdk/python/rtdip_sdk/functions/test_metadata.py new file mode 100644 index 000000000..d084db902 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/functions/test_metadata.py @@ -0,0 +1,91 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, '.') +import pandas as pd +import pytest +from pytest_mock import MockerFixture +from tests.sdk.python.rtdip_sdk.odbc.test_db_sql_connector import MockedDBConnection, MockedCursor +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from src.sdk.python.rtdip_sdk.functions.metadata import get as metadata_raw + +SERVER_HOSTNAME = "mock.cloud.databricks.com" +HTTP_PATH = "sql/mock/mock-test" +ACCESS_TOKEN = "mock_databricks_token" +DATABRICKS_SQL_CONNECT = 'databricks.sql.connect' +DATABRICKS_SQL_CONNECT_CURSOR = 'databricks.sql.connect.cursor' +INTERPOLATION_METHOD = "test/test/test" +MOCKED_QUERY="SELECT * FROM mocked-buiness-unit.sensors.mocked-asset_mocked-data-security-level_metadata WHERE TagName in ('MOCKED-TAGNAME')" +MOCKED_PARAMETER_DICT = { + "business_unit": "mocked-buiness-unit", + "region": "mocked-region", + "asset": "mocked-asset", + "data_security_level": "mocked-data-security-level", + "tag_names": ["MOCKED-TAGNAME"], + } + +MOCKED_NO_TAG_QUERY='SELECT * FROM mocked-buiness-unit.sensors.mocked-asset_mocked-data-security-level_metadata ' +MOCKED_PARAMETER_NO_TAGS_DICT = { + "business_unit": "mocked-buiness-unit", + "region": "mocked-region", + "asset": "mocked-asset", + "data_security_level": "mocked-data-security-level", + } + +def test_metadata(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data={'a': [1], 'b': [2], 'c': [3], 'd': [4]})) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + actual = metadata_raw(mocked_connection, MOCKED_PARAMETER_DICT) + + mocked_cursor.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_QUERY) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + +def test_no_tag_metadata(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data={'a': [1], 'b': [2], 'c': [3], 'd': [4]})) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + actual = metadata_raw(mocked_connection, MOCKED_PARAMETER_NO_TAGS_DICT) + + mocked_cursor.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_NO_TAG_QUERY) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + +def test_metadata_fails(mocker: MockerFixture): + mocker.spy(MockedDBConnection, "cursor") + mocker.spy(MockedCursor, "execute") + mocker.patch.object(MockedCursor, "fetchall", side_effect=Exception) + mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + metadata_raw(mocked_connection, MOCKED_PARAMETER_DICT) diff --git a/tests/sdk/python/rtdip_sdk/functions/test_raw.py b/tests/sdk/python/rtdip_sdk/functions/test_raw.py new file mode 100644 index 000000000..00424777e --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/functions/test_raw.py @@ -0,0 +1,70 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, '.') +import pandas as pd +import pytest +from pytest_mock import MockerFixture +from tests.sdk.python.rtdip_sdk.odbc.test_db_sql_connector import MockedDBConnection, MockedCursor +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from src.sdk.python.rtdip_sdk.functions.raw import get as raw_get + +SERVER_HOSTNAME = "mock.cloud.databricks.com" +HTTP_PATH = "sql/mock/mock-test" +ACCESS_TOKEN = "mock_databricks_token" +DATABRICKS_SQL_CONNECT = 'databricks.sql.connect' +DATABRICKS_SQL_CONNECT_CURSOR = 'databricks.sql.connect.cursor' +INTERPOLATION_METHOD = "test/test/test" +MOCKED_QUERY="SELECT EventTime, TagName, Status, Value FROM mocked-buiness-unit.sensors.mocked-asset_mocked-data-security-level_events_mocked-data-type WHERE EventDate BETWEEN to_date(\'2011-01-01T00:00:00\') AND to_date(\'2011-01-02T23:59:59\') AND EventTime BETWEEN to_timestamp(\'2011-01-01T00:00:00\') AND to_timestamp(\'2011-01-02T23:59:59\') AND TagName in ('MOCKED-TAGNAME') " +MOCKED_PARAMETER_DICT = { + "business_unit": "mocked-buiness-unit", + "region": "mocked-region", + "asset": "mocked-asset", + "data_security_level": "mocked-data-security-level", + "data_type": "mocked-data-type", + "tag_names": ["MOCKED-TAGNAME"], + "start_date": "2011-01-01", + "end_date": "2011-01-02", + "include_bad_data": True + } + +def test_raw(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data={'EventTime': [pd.to_datetime("2022-01-01 00:10:00+00:00")], 'TagName': ["MOCKED-TAGNAME"], 'Status': ["Good"], 'Value':[177.09220]})) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + actual = raw_get(mocked_connection, MOCKED_PARAMETER_DICT) + + mocked_cursor.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_QUERY) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + +def test_raw_fails(mocker: MockerFixture): + mocker.spy(MockedDBConnection, "cursor") + mocker.spy(MockedCursor, "execute") + mocker.patch.object(MockedCursor, "fetchall", side_effect=Exception) + mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + raw_get(mocked_connection, MOCKED_PARAMETER_DICT) diff --git a/tests/sdk/python/rtdip_sdk/functions/test_resample.py b/tests/sdk/python/rtdip_sdk/functions/test_resample.py new file mode 100644 index 000000000..f4a5a38e5 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/functions/test_resample.py @@ -0,0 +1,73 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, '.') +import pandas as pd +import pytest +from pytest_mock import MockerFixture +from tests.sdk.python.rtdip_sdk.odbc.test_db_sql_connector import MockedDBConnection, MockedCursor +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from src.sdk.python.rtdip_sdk.functions.resample import get as resample_get + +SERVER_HOSTNAME = "mock.cloud.databricks.com" +HTTP_PATH = "sql/mock/mock-test" +ACCESS_TOKEN = "mock_databricks_token" +DATABRICKS_SQL_CONNECT = 'databricks.sql.connect' +DATABRICKS_SQL_CONNECT_CURSOR = 'databricks.sql.connect.cursor' +INTERPOLATION_METHOD = "test/test/test" +MOCKED_QUERY="SELECT DISTINCT TagName, w.start AS EventTime, avg(Value) OVER (PARTITION BY TagName, w.start ORDER BY EventTime ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS Value FROM (SELECT EventTime, WINDOW(EventTime, \'1 hour\') w, TagName, Status, Value FROM mocked-buiness-unit.sensors.mocked-asset_mocked-data-security-level_events_mocked-data-type WHERE EventDate BETWEEN to_date(\'2011-01-01T00:00:00\') AND to_date(\'2011-01-02T23:59:59\') AND EventTime BETWEEN to_timestamp(\'2011-01-01T00:00:00\') AND to_timestamp(\'2011-01-02T23:59:59\') AND TagName in ('MOCKED-TAGNAME') AND Status = \'Good\')" +MOCKED_PARAMETER_DICT = { + "business_unit": "mocked-buiness-unit", + "region": "mocked-region", + "asset": "mocked-asset", + "data_security_level": "mocked-data-security-level", + "data_type": "mocked-data-type", + "tag_names": ["MOCKED-TAGNAME"], + "start_date": "2011-01-01", + "end_date": "2011-01-02", + "sample_rate": "1", + "sample_unit": "hour", + "agg_method": "avg", + "include_bad_data": False + } + +def test_resample(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data={'a': [1], 'b': [2], 'c': [3], 'd': [4]})) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + actual = resample_get(mocked_connection, MOCKED_PARAMETER_DICT) + + mocked_cursor.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_QUERY) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + +def test_resample_fails(mocker: MockerFixture): + mocker.spy(MockedDBConnection, "cursor") + mocker.spy(MockedCursor, "execute") + mocker.patch.object(MockedCursor, "fetchall", side_effect=Exception) + mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + resample_get(mocked_connection, MOCKED_PARAMETER_DICT) diff --git a/tests/sdk/python/rtdip_sdk/functions/test_time_weighted_average.py b/tests/sdk/python/rtdip_sdk/functions/test_time_weighted_average.py new file mode 100644 index 000000000..5628eb0d6 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/functions/test_time_weighted_average.py @@ -0,0 +1,85 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +sys.path.insert(0, '.') +from src.sdk.python.rtdip_sdk.functions.time_weighted_average import get as time_weighted_get +import pandas as pd +import pytest +import pytz +from pytest_mock import MockerFixture +from tests.sdk.python.rtdip_sdk.odbc.test_db_sql_connector import MockedDBConnection, MockedCursor +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection + +SERVER_HOSTNAME = "mock.cloud.databricks.com" +HTTP_PATH = "sql/mock/mock-test" +ACCESS_TOKEN = "mock_databricks_token" +DATABRICKS_SQL_CONNECT = 'databricks.sql.connect' +MOCKED_PARAMETER_DICT = { + "business_unit": "mocked-buiness-unit", + "region": "mocked-region", + "asset": "mocked-asset", + "data_security_level": "mocked-data-security-level", + "data_type": "mocked-data-type", + "tag_names": ["MOCKED-TAGNAME"], + "start_date": "2022-01-01", + "end_date": "2022-01-02", + "window_size_mins": 10, + "include_bad_data": False, + "step": True + } + +df = {"EventTime": [pd.to_datetime("2022-01-01 00:10:00+00:00").replace(tzinfo=pytz.timezone("Etc/UTC")), pd.to_datetime("2022-01-01 14:10:00+00:00").replace(tzinfo=pytz.timezone("Etc/UTC"))], "TagName": ["MOCKED-TAGNAME", "MOCKED-TAGNAME"], "Status": ["Good", "Good"], "Value":[177.09220, 160.01111]} + +def test_time_weighted_average_step_true(mocker: MockerFixture): + mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data=df)) + + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + actual = time_weighted_get(mocked_connection, MOCKED_PARAMETER_DICT) + + assert isinstance(actual, pd.DataFrame) + +def test_time_weighted_average_step_false(mocker: MockerFixture): + mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data=df)) + + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + MOCKED_PARAMETER_DICT["step"]=False + actual = time_weighted_get(mocked_connection, MOCKED_PARAMETER_DICT) + + assert isinstance(actual, pd.DataFrame) + +def test_time_weighted_average_with_window_length(mocker: MockerFixture): + mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data=df)) + + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + MOCKED_PARAMETER_DICT["window_length"]=10 + actual = time_weighted_get(mocked_connection, MOCKED_PARAMETER_DICT) + + assert isinstance(actual, pd.DataFrame) + +def test_time_weighted_average_fails(mocker: MockerFixture): + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + mocker.patch('src.sdk.python.rtdip_sdk.functions.time_weighted_average', return_value = Exception) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + time_weighted_get(mocked_connection, MOCKED_PARAMETER_DICT) \ No newline at end of file diff --git a/tests/sdk/python/rtdip_sdk/odbc/__init__.py b/tests/sdk/python/rtdip_sdk/odbc/__init__.py new file mode 100644 index 000000000..64e8b9c18 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/odbc/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/sdk/python/rtdip_sdk/odbc/mock/turbodbc b/tests/sdk/python/rtdip_sdk/odbc/mock/turbodbc new file mode 100644 index 000000000..e69de29bb diff --git a/tests/sdk/python/rtdip_sdk/odbc/test_connection_interface.py b/tests/sdk/python/rtdip_sdk/odbc/test_connection_interface.py new file mode 100644 index 000000000..fdb64fa79 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/odbc/test_connection_interface.py @@ -0,0 +1,54 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from src.sdk.python.rtdip_sdk.odbc.connection_interface import ConnectionInterface + +class TestConnection(ConnectionInterface): + def __init__(self): + pass #passing because method does not need to be not implemented + def close(self): + raise NotImplementedError + def cursor(self): + raise NotImplementedError + +def test_close_method_missing(): + class TestConnectionClose(ConnectionInterface): + def __init__(self): + pass #passing because method does not need to be not implemented + def cursor(self): + raise NotImplementedError + + with pytest.raises(TypeError): + TestConnectionClose() + +def test_cursor_method_missing(): + class TestConnectionCursor(ConnectionInterface): + def __init__(self): + pass #passing because method does not need to be not implemented + def close(self): + raise NotImplementedError + + with pytest.raises(TypeError): + TestConnectionCursor() + +def test_close_method(): + test_connection = TestConnection() + with pytest.raises(NotImplementedError): + test_connection.close() + +def test_cursor_method(): + test_connection = TestConnection() + with pytest.raises(NotImplementedError): + test_connection.cursor() \ No newline at end of file diff --git a/tests/sdk/python/rtdip_sdk/odbc/test_cursor_interface.py b/tests/sdk/python/rtdip_sdk/odbc/test_cursor_interface.py new file mode 100644 index 000000000..32d3535d3 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/odbc/test_cursor_interface.py @@ -0,0 +1,76 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from src.sdk.python.rtdip_sdk.odbc.cursor_interface import CursorInterface + +class TestCursor(CursorInterface): + def execute(self, query: str): + raise NotImplementedError + def fetch_all(self): + raise NotImplementedError + def close(self): + raise NotImplementedError + +def test_execute_method_missing(): + class TestCursorExecute(CursorInterface): + def __init__(self): + pass #passing because method does not need to be not implemented + def fetch_all(self): + raise NotImplementedError + def close(self): + raise NotImplementedError + + with pytest.raises(TypeError): + TestCursorExecute() + +def test_fetch_all_method_missing(): + class TestCursorFetchAll(CursorInterface): + def __init__(self): + pass #passing because method does not need to be not implemented + def execute(self, query: str): + raise NotImplementedError + def close(self): + raise NotImplementedError + + with pytest.raises(TypeError): + TestCursorFetchAll() + +def test_close_method_missing(): + class TestCursorFetchAll(CursorInterface): + def __init__(self): + pass #passing because method does not need to be not implemented + def execute(self, query: str): + raise NotImplementedError + def fetch_all(self): + raise NotImplementedError + + with pytest.raises(TypeError): + TestCursorFetchAll() + +def test_execute_method(): + test_cursor = TestCursor() + query = "test" + with pytest.raises(NotImplementedError): + test_cursor.execute(query) + +def test_fetch_all_method(): + test_cursor = TestCursor() + with pytest.raises(NotImplementedError): + test_cursor.fetch_all() + +def test_close_method(): + test_cursor = TestCursor() + with pytest.raises(NotImplementedError): + test_cursor.close() \ No newline at end of file diff --git a/tests/sdk/python/rtdip_sdk/odbc/test_db_sql_connector.py b/tests/sdk/python/rtdip_sdk/odbc/test_db_sql_connector.py new file mode 100644 index 000000000..bdd209a2d --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/odbc/test_db_sql_connector.py @@ -0,0 +1,126 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLConnection +from src.sdk.python.rtdip_sdk.odbc.db_sql_connector import DatabricksSQLCursor +import pandas as pd +from pytest_mock import MockerFixture +import pytest + +SERVER_HOSTNAME = "mock.cloud.databricks.com" +HTTP_PATH = "sql/mock/mock-test" +ACCESS_TOKEN = "mock_databricks_token" +DATABRICKS_SQL_CONNECT = 'databricks.sql.connect' +DATABRICKS_SQL_CONNECT_CURSOR = 'databricks.sql.connect.cursor' + +class MockedDBConnection: + def close(self) -> None: + return None + def cursor(self) -> object: + return MockedCursor() + +class MockedCursor: + def __init__(self): + self.description = [("EventTime",), ("TagName",), ("Status",), ("Value",)] + def execute(self, query) -> None: + return None + def fetchall(self) -> list: + return list + def close(self) -> None: + return None + +def test_connection_close(mocker: MockerFixture): + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + mocked_close = mocker.spy(MockedDBConnection, "close") + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + mocked_connection.close() + + mocked_close.assert_called_once() + +def test_connection_cursor(mocker: MockerFixture): + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + result = mocked_connection.cursor() + + assert isinstance(result, object) + mocked_cursor.assert_called_once() + +def test_cursor_execute(mocker: MockerFixture): + mocked_execute = mocker.spy(MockedCursor, "execute") + + mocked_cursor = DatabricksSQLCursor(MockedCursor()) + mocked_cursor.execute("test") + + mocked_execute.assert_called_with(mocker.ANY, query="test") + +def test_cursor_fetch_all(mocker: MockerFixture): + mocker.patch.object(MockedCursor, "fetchall", return_value = pd.DataFrame(data={'EventTime': [pd.to_datetime("2022-01-01 00:10:00+00:00")], 'TagName': ["MOCKED-TAGNAME"], 'Status': ["Good"], 'Value':[177.09220]})) + + mocked_cursor = DatabricksSQLCursor(MockedCursor()) + result = mocked_cursor.fetch_all() + + assert isinstance(result, pd.DataFrame) + +def test_cursor_close(mocker: MockerFixture): + mocked_close = mocker.spy(MockedCursor, "close") + + mocked_cursor = DatabricksSQLCursor(MockedCursor()) + mocked_cursor.close() + + mocked_close.assert_called_once() + +def test_connection_close_fails(mocker: MockerFixture): + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + mocker.patch.object(MockedDBConnection, "close", side_effect = Exception) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + mocked_connection.close() + +def test_connection_cursor_fails(mocker: MockerFixture): + mocker.patch(DATABRICKS_SQL_CONNECT, return_value = MockedDBConnection()) + mocker.patch.object(MockedDBConnection, "cursor", side_effect = Exception) + + mocked_connection = DatabricksSQLConnection(SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + assert mocked_connection.cursor() + +def test_cursor_execute_fails(mocker: MockerFixture): + mocker.patch.object(MockedCursor, "execute", side_effect = Exception) + + mocked_cursor = DatabricksSQLCursor(MockedCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.execute("test") + +def test_cursor_fetch_all_fails(mocker: MockerFixture): + mocker.patch.object(MockedCursor, "fetchall", side_effect = Exception) + + mocked_cursor = DatabricksSQLCursor(MockedCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.fetch_all() + +def test_cursor_close_fails(mocker: MockerFixture): + mocker.patch.object(MockedCursor, "close", side_effect = Exception) + + mocked_cursor = DatabricksSQLCursor(MockedCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.close() diff --git a/tests/sdk/python/rtdip_sdk/odbc/test_pyodbc_sql_connector.py b/tests/sdk/python/rtdip_sdk/odbc/test_pyodbc_sql_connector.py new file mode 100644 index 000000000..fe3752b00 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/odbc/test_pyodbc_sql_connector.py @@ -0,0 +1,137 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from src.sdk.python.rtdip_sdk.odbc.pyodbc_sql_connector import PYODBCSQLConnection +from src.sdk.python.rtdip_sdk.odbc.pyodbc_sql_connector import PYODBCSQLCursor +import pandas as pd +from pytest_mock import MockerFixture +import pytest + +DRIVER_PATH = "sql/driver/mock-path" +SERVER_HOSTNAME = "mock.cloud.databricks.com" +ACCESS_TOKEN = "mock_databricks_token" +HTTP_PATH = "sql/mock/mock-test" + +PYODBC_CONNECT = 'pyodbc.connect' +PYODBC_CONNECT_CURSOR = 'pyodbc.connect.cursor' +CURSOR_DESCIPTION = 'cursor.description' + + +class PMockedDBConnection: + def close(self) -> None: + return None + + def cursor(self) -> object: + return PMockedCursor() + + +class PMockedCursor: + def execute(self, query) -> None: + return None + + def fetchall(self) -> pd.DataFrame: + return pd.DataFrame + + def close(self) -> None: + return None + +def test_connection_close(mocker: MockerFixture): + mocker.patch(PYODBC_CONNECT, return_value=PMockedDBConnection()) + mocked_close = mocker.spy(PMockedDBConnection, "close") + + pmocked_connection = PYODBCSQLConnection(DRIVER_PATH, SERVER_HOSTNAME, ACCESS_TOKEN, HTTP_PATH) + pmocked_connection.close() + + mocked_close.assert_called_once() + +def test_connection_cursor(mocker: MockerFixture): + mocker.patch(PYODBC_CONNECT, return_value=PMockedDBConnection()) + pmocked_cursor = mocker.spy(PMockedDBConnection, "cursor") + + pmocked_connection = PYODBCSQLConnection(DRIVER_PATH, SERVER_HOSTNAME, ACCESS_TOKEN, HTTP_PATH) + result = pmocked_connection.cursor() + + assert isinstance(result, object) + pmocked_cursor.assert_called_once() + +def test_cursor_execute(mocker: MockerFixture): + mocked_execute = mocker.spy(PMockedCursor, "execute") + + pmocked_cursor = PYODBCSQLCursor(PMockedCursor()) + pmocked_cursor.execute("test") + + mocked_execute.assert_called_with(mocker.ANY, query="test") + +def test_cursor_fetch_all(mocker: MockerFixture): + mocker.patch.object(PMockedCursor, "fetchall", return_value = [('1', '2', '3', '4')]) + + foo = PMockedCursor() + foo.description = (('column_name_1', 0, 1, 2), ('column_name_2', 2, 3, 4), ('column_name_3', 4, 5, 6), ('column_name_4', 6, 7, 8 )) + + pmocked_cursor = PYODBCSQLCursor(foo) + result = pmocked_cursor.fetch_all() + + assert isinstance(result, pd.DataFrame) + assert len(result.columns) == 4 + +def test_cursor_close(mocker: MockerFixture): + mocked_close = mocker.spy(PMockedCursor, "close") + + pmocked_cursor = PYODBCSQLCursor(PMockedCursor()) + pmocked_cursor.close() + + mocked_close.assert_called_once() + +def test_connection_close_fails(mocker: MockerFixture): + mocker.patch(PYODBC_CONNECT, return_value = PMockedDBConnection) + mocker.patch.object(PMockedDBConnection, "close", side_effect = Exception) + + mocked_connection = PYODBCSQLConnection(DRIVER_PATH, SERVER_HOSTNAME, ACCESS_TOKEN, HTTP_PATH) + + with pytest.raises(Exception): + mocked_connection.close() + +def test_connection_cursor_fails(mocker: MockerFixture): + mocker.patch(PYODBC_CONNECT, return_value = PMockedDBConnection()) + mocker.patch.object(PMockedDBConnection, "cursor", side_effect = Exception) + + mocked_connection = PYODBCSQLConnection(DRIVER_PATH, SERVER_HOSTNAME, ACCESS_TOKEN, HTTP_PATH) + + with pytest.raises(Exception): + mocked_connection.cursor() + +def test_cursor_execute_fails(mocker: MockerFixture): + mocker.patch.object(PMockedCursor, "execute", side_effect = Exception) + + mocked_cursor = PYODBCSQLCursor(PMockedCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.execute("test") + +def test_cursor_fetch_all_fails(mocker: MockerFixture): + mocker.patch.object(PMockedCursor, "fetchall", side_effect=Exception) + + mocked_cursor = PYODBCSQLCursor(PMockedCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.fetch_all() + + +def test_cursor_close_fails(mocker: MockerFixture): + mocker.patch.object(PMockedCursor, "close", side_effect=Exception) + + mocked_cursor = PYODBCSQLCursor(PMockedCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.close() diff --git a/tests/sdk/python/rtdip_sdk/odbc/test_turbodbc_sql_connector.py b/tests/sdk/python/rtdip_sdk/odbc/test_turbodbc_sql_connector.py new file mode 100644 index 000000000..1f7f5c713 --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/odbc/test_turbodbc_sql_connector.py @@ -0,0 +1,131 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from src.sdk.python.rtdip_sdk.odbc.turbodbc_sql_connector import TURBODBCSQLConnection, TURBODBCSQLCursor +from pytest_mock import MockerFixture +import pytest +import pandas as pd + +HOST_NAME= "myHostName" +HTTP_PATH = "myServerAddress" +ACCESS_TOKEN = "myToken" +TURBODBC_CONNECT = 'src.sdk.python.rtdip_sdk.odbc.turbodbc_sql_connector.connect' +TURBODBC_CONNECT_CURSOR = 'src.sdk.python.rtdip_sdk.odbc.turbodbc_sql_connector.connect.cursor' + +class MockedTURBODBCConnection: + def close(self) -> None: + return None + + def cursor(self) -> object: + return MockedTURBODBCCursor() + + +class MockedTURBODBCCursor: + def execute(self, query) -> None: + return None + + def fetchall(self) -> list: + return list + + def close(self) -> None: + return None + +def test_connection_close(mocker: MockerFixture): + mocker.patch(TURBODBC_CONNECT, return_value=MockedTURBODBCConnection()) + mocked_close = mocker.spy(MockedTURBODBCConnection, "close") + + mocked_connection = TURBODBCSQLConnection(HOST_NAME, HTTP_PATH, ACCESS_TOKEN) + mocked_connection.close() + + mocked_close.assert_called_once() + +def test_connection_cursor(mocker: MockerFixture): + mocker.patch(TURBODBC_CONNECT, return_value=MockedTURBODBCConnection()) + mocked_cursor = mocker.spy(MockedTURBODBCConnection, "cursor") + + mocked_connection = TURBODBCSQLConnection(HOST_NAME, HTTP_PATH, ACCESS_TOKEN) + result = mocked_connection.cursor() + + assert isinstance(result, object) + mocked_cursor.assert_called_once() + +def test_cursor_execute(mocker: MockerFixture): + mocked_execute = mocker.spy(MockedTURBODBCCursor, "execute") + + mocked_cursor = TURBODBCSQLCursor(MockedTURBODBCCursor()) + mocked_cursor.execute("test") + + mocked_execute.assert_called_with(mocker.ANY, query="test") + +def test_cursor_fetch_all(mocker: MockerFixture): + mocker.patch.object(MockedTURBODBCCursor, "fetchall", return_value= [['1', '2', '3', '4']]) + + foo = MockedTURBODBCCursor() + foo.description = (('column_name_1', 0, 1, 2), ('column_name_2', 2, 3, 4), ('column_name_3', 4, 5, 6), ('column_name_4', 6, 7, 8)) + + mocked_cursor = TURBODBCSQLCursor(foo) + result = mocked_cursor.fetch_all() + + assert isinstance(result, pd.DataFrame) + assert len(result.columns) == 4 + +def test_cursor_close(mocker: MockerFixture): + mocked_close = mocker.spy(MockedTURBODBCCursor, "close") + + mocked_cursor = TURBODBCSQLCursor(MockedTURBODBCCursor()) + mocked_cursor.close() + + mocked_close.assert_called_once() + +def test_connection_close_fails(mocker: MockerFixture): + mocker.patch(TURBODBC_CONNECT, return_value=MockedTURBODBCConnection()) + mocker.patch.object(MockedTURBODBCConnection, "close", side_effect=Exception) + + mocked_connection = TURBODBCSQLConnection(HOST_NAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + assert mocked_connection.close() + +def test_connection_cursor_fails(mocker: MockerFixture): + mocker.patch(TURBODBC_CONNECT, return_value=MockedTURBODBCConnection()) + mocker.patch.object(MockedTURBODBCConnection, "cursor", side_effect=Exception) + + mocked_connection = TURBODBCSQLConnection(HOST_NAME, HTTP_PATH, ACCESS_TOKEN) + + with pytest.raises(Exception): + assert mocked_connection.cursor() + +def test_cursor_execute_fails(mocker: MockerFixture): + mocker.patch.object(MockedTURBODBCCursor, "execute", side_effect=Exception) + + mocked_cursor = TURBODBCSQLCursor(MockedTURBODBCCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.execute("test") + +def test_cursor_fetch_all_fails(mocker: MockerFixture): + mocker.patch.object(MockedTURBODBCCursor, "fetchall", side_effect=Exception) + + mocked_cursor = TURBODBCSQLCursor(MockedTURBODBCCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.fetch_all() + +def test_cursor_close_fails(mocker: MockerFixture): + mocker.patch.object(MockedTURBODBCCursor, "close", side_effect=Exception) + + mocked_cursor = TURBODBCSQLCursor(MockedTURBODBCCursor()) + + with pytest.raises(Exception): + assert mocked_cursor.close()