From 4917027717bb57373fcccd2bc004acb27e89b0fa Mon Sep 17 00:00:00 2001 From: GBBBAS <42962356+GBBBAS@users.noreply.github.com> Date: Wed, 1 May 2024 12:51:26 +0100 Subject: [PATCH] Add Plot Query to SDK and API (#731) * Additions for Plot Query Signed-off-by: GBBBAS * Updates for Plot Query Signed-off-by: GBBBAS * Updates for Python package Signed-off-by: GBBBAS * Updates for test errors Signed-off-by: GBBBAS * Update test name Signed-off-by: GBBBAS * Update plot tests Signed-off-by: GBBBAS --------- Signed-off-by: GBBBAS --- .vscode/settings.json | 6 +- .../query/functions/time_series/plot.md | 15 ++ docs/sdk/examples/query/Plot.md | 1 + docs/sdk/queries/functions.md | 12 + environment.yml | 13 +- mkdocs.yml | 6 +- setup.py | 4 +- src/api/requirements.txt | 2 +- src/api/v1/__init__.py | 1 + src/api/v1/common.py | 4 + src/api/v1/models.py | 35 +++ src/api/v1/plot.py | 142 ++++++++++++ .../time_series/_time_series_query_builder.py | 106 +++++++++ .../rtdip_sdk/queries/time_series/plot.py | 88 ++++++++ .../time_series/time_series_query_builder.py | 74 ++++++ tests/api/v1/api_test_objects.py | 21 ++ tests/api/v1/test_api_plot.py | 213 ++++++++++++++++++ .../queries/_test_utils/sdk_test_objects.py | 2 + .../queries/time_series/test_plot.py | 85 +++++++ .../queries/time_series/test_query_builder.py | 21 ++ 20 files changed, 835 insertions(+), 16 deletions(-) create mode 100644 docs/sdk/code-reference/query/functions/time_series/plot.md create mode 100644 docs/sdk/examples/query/Plot.md create mode 100644 src/api/v1/plot.py create mode 100644 src/sdk/python/rtdip_sdk/queries/time_series/plot.py create mode 100644 tests/api/v1/test_api_plot.py create mode 100644 tests/sdk/python/rtdip_sdk/queries/time_series/test_plot.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 2b688f678..334815bf7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,11 +5,7 @@ "azureFunctions.projectRuntime": "~4", "python.testing.pytestArgs": [ - "--cache-clear", - "--cov=.", - "--cov-report=xml:cov.xml", - "tests", - "-vv", + "tests" ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, diff --git a/docs/sdk/code-reference/query/functions/time_series/plot.md b/docs/sdk/code-reference/query/functions/time_series/plot.md new file mode 100644 index 000000000..9d4dd190f --- /dev/null +++ b/docs/sdk/code-reference/query/functions/time_series/plot.md @@ -0,0 +1,15 @@ +# Plot Function +::: src.sdk.python.rtdip_sdk.queries.time_series.plot + +## Example +```python +--8<-- "https://raw.githubusercontent.com/rtdip/samples/main/queries/Plot/plot.py" +``` + +This example is using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). + +!!! note "Note" + See [Samples Repository](https://github.com/rtdip/samples/tree/main/queries) for full list of examples. + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
\ No newline at end of file diff --git a/docs/sdk/examples/query/Plot.md b/docs/sdk/examples/query/Plot.md new file mode 100644 index 000000000..a87aaa508 --- /dev/null +++ b/docs/sdk/examples/query/Plot.md @@ -0,0 +1 @@ +--8<-- "https://raw.githubusercontent.com/rtdip/samples/main/queries/Plot/README.md" \ No newline at end of file diff --git a/docs/sdk/queries/functions.md b/docs/sdk/queries/functions.md index af516be02..c01072c2d 100644 --- a/docs/sdk/queries/functions.md +++ b/docs/sdk/queries/functions.md @@ -23,6 +23,18 @@ The RTDIP SDK enables users to perform complex queries, including aggregation on - Time Interval Unit - The time interval unit (second, minute, day, hour) - Aggregation Method - Aggregations including first, last, avg, min, max +!!! note "Note" + Sample Rate and Sample Unit parameters are deprecated and will be removed in v1.0.0. Please use Time Interval Rate and Time Interval Unit instead.
+ +### Plot + +[Plot](../code-reference/query/functions/time_series/plot.md) enables changing the frequency of time series observations and performing Average, Min, Max, First, Last and StdDev aggregations. This is achieved by providing the following parameters: + +- Sample Rate - (deprecated) +- Sample Unit - (deprecated) +- Time Interval Rate - The time interval rate +- Time Interval Unit - The time interval unit (second, minute, day, hour) + !!! note "Note" Sample Rate and Sample Unit parameters are deprecated and will be removed in v1.0.0. Please use Time Interval Rate and Time Interval Unit instead.
diff --git a/environment.yml b/environment.yml index 2532d1a9a..cec84b295 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,6 @@ channels: - defaults dependencies: - python>=3.9,<3.12 - - mkdocs-material-extensions==1.1.1 - jinja2==3.1.3 - pytest==7.4.0 - pytest-mock==3.11.1 @@ -46,19 +45,20 @@ dependencies: - googleapis-common-protos>=1.56.4 - openjdk==11.0.15 - openai==1.13.3 - - mkdocs-material==9.3.1 + - mkdocs-material==9.5.20 + - mkdocs-material-extensions==1.3.1 - mkdocstrings==0.22.0 - mkdocstrings-python==1.4.0 - mkdocs-macros-plugin==1.0.1 - pygments==2.16.1 - - pymdown-extensions==10.1.0 + - pymdown-extensions==10.8.1 - databricks-sql-connector==3.1.0 - semver==3.0.0 - xlrd==2.0.1 - pygithub==1.59.0 - pydantic==2.6.0 - pyjwt==2.8.0 - - web3==6.5.0 + - web3==6.16.0 - twine==4.0.2 - delta-sharing-python==1.0.0 - polars==0.18.8 @@ -75,9 +75,10 @@ dependencies: - azure-functions==1.15.0 - azure-mgmt-eventgrid==10.2.0 - hvac==1.1.1 - - langchain==0.1.11 + - langchain==0.1.17 - build==0.10.0 - deltalake==0.10.1 - trio==0.22.1 - sqlparams==5.1.0 - - entsoe-py==0.5.10 \ No newline at end of file + - entsoe-py==0.5.10 + - eth-typing==4.2.1 \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index dbeae52dc..cee646a19 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -113,8 +113,8 @@ markdown_extensions: - pymdownx.tabbed: alternate_style: true - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg # Page tree + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg # Page tree - pymdownx.snippets: url_download: true @@ -258,6 +258,7 @@ nav: - Raw: sdk/code-reference/query/functions/time_series/raw.md - Latest: sdk/code-reference/query/functions/time_series/latest.md - Resample: sdk/code-reference/query/functions/time_series/resample.md + - Plot: sdk/code-reference/query/functions/time_series/plot.md - Interpolate: sdk/code-reference/query/functions/time_series/interpolate.md - Interpolation at Time: sdk/code-reference/query/functions/time_series/interpolation-at-time.md - Time Weighted Average: sdk/code-reference/query/functions/time_series/time-weighted-average.md @@ -289,6 +290,7 @@ nav: - Metadata: sdk/examples/query/Metadata.md - Raw: sdk/examples/query/Raw.md - Resample: sdk/examples/query/Resample.md + - Plot: sdk/examples/query/Plot.md - Time Weighted Average: sdk/examples/query/Time-Weighted-Average.md - Circular Average: sdk/examples/query/Circular-Average.md - Circular Standard Deviation: sdk/examples/query/Circular-Standard-Deviation.md diff --git a/setup.py b/setup.py index bf6483891..9abecb1b7 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ "grpcio>=1.48.1", "grpcio-status>=1.48.1", "googleapis-common-protos>=1.56.4", - "langchain==0.1.11", + "langchain==0.1.17", "openai==1.13.3", "pydantic==2.6.0", ] @@ -58,7 +58,7 @@ "boto3==1.28.2", "hvac==1.1.1", "azure-keyvault-secrets==4.7.0", - "web3==6.5.0", + "web3==6.16.0", "polars[deltalake]==0.18.8", "delta-sharing==1.0.0", "xarray>=2023.1.0,<2023.8.0", diff --git a/src/api/requirements.txt b/src/api/requirements.txt index 6d80ad256..ceb0ce9e2 100644 --- a/src/api/requirements.txt +++ b/src/api/requirements.txt @@ -18,6 +18,6 @@ packaging==23.2 grpcio>=1.48.1 grpcio-status>=1.48.1 googleapis-common-protos>=1.56.4 -langchain==0.1.11 +langchain==0.1.17 openai==1.13.3 pyjwt==2.8.0 \ No newline at end of file diff --git a/src/api/v1/__init__.py b/src/api/v1/__init__.py index fa38d1eca..37f5865af 100644 --- a/src/api/v1/__init__.py +++ b/src/api/v1/__init__.py @@ -21,6 +21,7 @@ sql, latest, resample, + plot, interpolate, interpolation_at_time, circular_average, diff --git a/src/api/v1/common.py b/src/api/v1/common.py index 6c8d2fda0..1e509885b 100644 --- a/src/api/v1/common.py +++ b/src/api/v1/common.py @@ -36,6 +36,7 @@ def common_api_setup_tasks( # NOSONAR sql_query_parameters=None, tag_query_parameters=None, resample_query_parameters=None, + plot_query_parameters=None, interpolate_query_parameters=None, interpolation_at_time_query_parameters=None, time_weighted_average_query_parameters=None, @@ -100,6 +101,9 @@ def common_api_setup_tasks( # NOSONAR if resample_query_parameters != None: parameters = dict(parameters, **resample_query_parameters.__dict__) + if plot_query_parameters != None: + parameters = dict(parameters, **plot_query_parameters.__dict__) + if interpolate_query_parameters != None: parameters = dict(parameters, **interpolate_query_parameters.__dict__) diff --git a/src/api/v1/models.py b/src/api/v1/models.py index b85391f98..e7989610c 100644 --- a/src/api/v1/models.py +++ b/src/api/v1/models.py @@ -134,6 +134,17 @@ class ResampleInterpolateRow(BaseModel): Value: Union[float, int, str, None] +class PlotRow(BaseModel): + EventTime: datetime + TagName: str + Average: float + Min: float + Max: float + First: float + Last: float + StdDev: float + + class PivotRow(BaseModel): EventTime: datetime @@ -277,6 +288,30 @@ class TagsBodyParams(BaseModel): tag_name: List[str] +class PlotQueryParams: + def __init__( + self, + sample_rate: str = Query( + ..., + description="sample_rate is deprecated and will be removed in v1.0.0. Please use time_interval_rate instead.", + examples=[5], + deprecated=True, + ), + sample_unit: str = Query( + ..., + description="sample_unit is deprecated and will be removed in v1.0.0. Please use time_interval_unit instead.", + examples=["second", "minute", "hour", "day"], + deprecated=True, + ), + time_interval_rate: str = DuplicatedQueryParameters.time_interval_rate, + time_interval_unit: str = DuplicatedQueryParameters.time_interval_unit, + ): + self.sample_rate = sample_rate + self.sample_unit = sample_unit + self.time_interval_rate = time_interval_rate + self.time_interval_unit = time_interval_unit + + class ResampleQueryParams: def __init__( self, diff --git a/src/api/v1/plot.py b/src/api/v1/plot.py new file mode 100644 index 000000000..1d12158f2 --- /dev/null +++ b/src/api/v1/plot.py @@ -0,0 +1,142 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +from typing import Union +from src.api.FastAPIApp import api_v1_router +from fastapi import HTTPException, Depends, Body + +from src.sdk.python.rtdip_sdk.queries.time_series import plot +from src.api.v1.models import ( + BaseQueryParams, + BaseHeaders, + ResampleInterpolateRow, + HTTPError, + RawQueryParams, + TagsQueryParams, + TagsBodyParams, + PlotQueryParams, + PivotQueryParams, + LimitOffsetQueryParams, +) +from src.api.v1.common import common_api_setup_tasks, json_response + + +def plot_events_get( + base_query_parameters, + raw_query_parameters, + tag_query_parameters, + plot_query_parameters, + limit_offset_parameters, + base_headers, +): + try: + (connection, parameters) = common_api_setup_tasks( + base_query_parameters, + raw_query_parameters=raw_query_parameters, + tag_query_parameters=tag_query_parameters, + plot_query_parameters=plot_query_parameters, + limit_offset_query_parameters=limit_offset_parameters, + base_headers=base_headers, + ) + + data = plot.get(connection, parameters) + + return json_response(data, limit_offset_parameters) + except Exception as e: + logging.error(str(e)) + raise HTTPException(status_code=400, detail=str(e)) + + +get_description = """ +## Plot + +Plotting of resampled raw timeseries data and aggregated to Min, Max, First and Last and an Exception Value(Status = Bad) if it exists. +""" + + +@api_v1_router.get( + path="/events/plot", + name="Plot GET", + description=get_description, + tags=["Events"], + responses={ + 200: {"model": ResampleInterpolateRow}, + 400: {"model": HTTPError}, + }, + openapi_extra={ + "externalDocs": { + "description": "RTDIP Plot Query Documentation", + "url": "https://www.rtdip.io/sdk/code-reference/query/functions/time_series/plot/", + } + }, +) +async def plot_get( + base_query_parameters: BaseQueryParams = Depends(), + raw_query_parameters: RawQueryParams = Depends(), + tag_query_parameters: TagsQueryParams = Depends(), + plot_query_parameters: PlotQueryParams = Depends(), + limit_offset_parameters: LimitOffsetQueryParams = Depends(), + base_headers: BaseHeaders = Depends(), +): + return plot_events_get( + base_query_parameters, + raw_query_parameters, + tag_query_parameters, + plot_query_parameters, + limit_offset_parameters, + base_headers, + ) + + +post_description = """ +## Plot + +Plotting of resampled raw timeseries data and aggregated to Average, Min, Max, First and Last and an Exception Value(Status = Bad) if it exists, via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters. +""" + + +@api_v1_router.post( + path="/events/plot", + name="Plot POST", + description=post_description, + tags=["Events"], + responses={ + 200: {"model": ResampleInterpolateRow}, + 400: {"model": HTTPError}, + }, + openapi_extra={ + "externalDocs": { + "description": "RTDIP Resample Query Documentation", + "url": "https://www.rtdip.io/sdk/code-reference/query/functions/time_series/plot/", + } + }, +) +async def plot_post( + base_query_parameters: BaseQueryParams = Depends(), + raw_query_parameters: RawQueryParams = Depends(), + tag_query_parameters: TagsBodyParams = Body(default=...), + plot_query_parameters: PlotQueryParams = Depends(), + limit_offset_parameters: LimitOffsetQueryParams = Depends(), + base_headers: BaseHeaders = Depends(), +): + return plot_events_get( + base_query_parameters, + raw_query_parameters, + tag_query_parameters, + plot_query_parameters, + limit_offset_parameters, + base_headers, + ) diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py b/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py index 6311ca5ba..4d8f518d0 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py @@ -202,6 +202,103 @@ def _sample_query(parameters_dict: dict) -> tuple: return sql_query, sample_query, sample_parameters +def _plot_query(parameters_dict: dict) -> tuple: + plot_query = ( + "WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`{{ timestamp_column }}`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"{{ time_zone }}\") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} 'Good' AS `Status`, {% endif %} `{{ value_column }}` FROM " + "{% if source is defined and source is not none %}" + "`{{ source|lower }}` " + "{% else %}" + "`{{ business_unit|lower }}`.`sensors`.`{{ asset|lower }}_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " + "{% endif %}" + "{% if case_insensitivity_tag_search is defined and case_insensitivity_tag_search == true %}" + "WHERE `{{ timestamp_column }}` BETWEEN to_timestamp(\"{{ start_date }}\") AND to_timestamp(\"{{ end_date }}\") AND UPPER(`{{ tagname_column }}`) IN ('{{ tag_names | join('\\', \\'') | upper }}') " + "{% else %}" + "WHERE `{{ timestamp_column }}` BETWEEN to_timestamp(\"{{ start_date }}\") AND to_timestamp(\"{{ end_date }}\") AND `{{ tagname_column }}` IN ('{{ tag_names | join('\\', \\'') }}') " + "{% endif %}" + "{% if include_status is defined and include_status == true and include_bad_data is defined and include_bad_data == false %} AND `{{ status_column }}` IN ('Good', 'Good, Annotated', 'Substituted, Good, Annotated', 'Substituted, Good', 'Good, Questionable', 'Questionable, Good') {% endif %}) " + ',date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("{{ start_date }}"), "{{ time_zone }}"), from_utc_timestamp(to_timestamp("{{ end_date }}"), "{{ time_zone }}"), INTERVAL \'{{ time_interval_rate + \' \' + time_interval_unit }}\')) AS timestamp_array) ' + ",window_buckets AS (SELECT timestamp_array AS window_start, timestampadd({{time_interval_unit }}, {{ time_interval_rate }}, timestamp_array) AS window_end FROM date_array) " + ",plot AS (SELECT /*+ RANGE_JOIN(d, {{ range_join_seconds }} ) */ d.window_start, d.window_end, e.`{{ tagname_column }}`" + ", min(CASE WHEN `{{ status_column }}` = 'Bad' THEN null ELSE struct(e.`{{ value_column }}`, e.`{{ timestamp_column }}`) END) OVER (PARTITION BY e.`{{ tagname_column }}`, d.window_start ORDER BY e.`{{ timestamp_column }}` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `min_{{ value_column }}`" + ", max(CASE WHEN `{{ status_column }}` = 'Bad' THEN null ELSE struct(e.`{{ value_column }}`, e.`{{ timestamp_column }}`) END) OVER (PARTITION BY e.`{{ tagname_column }}`, d.window_start ORDER BY e.`{{ timestamp_column }}` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `max_{{ value_column }}`" + ", first(CASE WHEN `{{ status_column }}` = 'Bad' THEN null ELSE struct(e.`{{ value_column }}`, e.`{{ timestamp_column }}`) END, True) OVER (PARTITION BY e.`{{ tagname_column }}`, d.window_start ORDER BY e.`{{ timestamp_column }}` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `first_{{ value_column }}`" + ", last(CASE WHEN `{{ status_column }}` = 'Bad' THEN null ELSE struct(e.`{{ value_column }}`, e.`{{ timestamp_column }}`) END, True) OVER (PARTITION BY e.`{{ tagname_column }}`, d.window_start ORDER BY e.`{{ timestamp_column }}` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `last_{{ value_column }}`" + ", first(CASE WHEN `{{ status_column }}` = 'Bad' THEN struct(e.`{{ value_column }}`, e.`{{ timestamp_column }}`) ELSE null END, True) OVER (PARTITION BY e.`{{ tagname_column }}`, d.window_start ORDER BY e.`{{ timestamp_column }}` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `excp_{{ value_column }}` " + "FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`{{ timestamp_column }}` AND d.window_end > e.`{{ timestamp_column }}`) " + ",deduplicate AS (SELECT window_start AS `{{ timestamp_column }}`, `{{ tagname_column }}`, `min_{{ value_column }}` as `Min`, `max_{{ value_column }}` as `Max`, `first_{{ value_column }}` as `First`, `last_{{ value_column }}` as `Last`, `excp_{{ value_column }}` as `Exception` FROM plot GROUP BY window_start, `{{ tagname_column }}`, `min_{{ value_column }}`, `max_{{ value_column }}`, `first_{{ value_column }}`, `last_{{ value_column }}`, `excp_{{ value_column }}`) " + ",project AS (SELECT distinct Values.{{ timestamp_column }}, `{{ tagname_column }}`, Values.{{ value_column }} FROM (SELECT * FROM deduplicate UNPIVOT (`Values` for `Aggregation` IN (`Min`, `Max`, `First`, `Last`, `Exception`))) " + "{% if is_resample is defined and is_resample == true %}" + "ORDER BY `{{ tagname_column }}`, `{{ timestamp_column }}` " + "{% endif %}" + ") " + "{% if is_resample is defined and is_resample == true and pivot is defined and pivot == true %}" + "{% if case_insensitivity_tag_search is defined and case_insensitivity_tag_search == true %}" + ",pivot AS (SELECT * FROM (SELECT `{{ timestamp_column }}`, `{{ value_column }}`, UPPER(`{{ tagname_column }}`) AS `{{ tagname_column }}` FROM project) PIVOT (FIRST(`{{ value_column }}`) FOR `{{ tagname_column }}` IN (" + "{% for i in range(tag_names | length) %}" + "'{{ tag_names[i] | upper }}' AS `{{ tag_names[i] }}`{% if not loop.last %}, {% endif %}" + "{% endfor %}" + "{% else %}" + ",pivot AS (SELECT * FROM (SELECT `{{ timestamp_column }}`, `{{ value_column }}`, `{{ tagname_column }}` AS `{{ tagname_column }}` FROM project) PIVOT (FIRST(`{{ value_column }}`) FOR `{{ tagname_column }}` IN (" + "{% for i in range(tag_names | length) %}" + "'{{ tag_names[i] }}' AS `{{ tag_names[i] }}`{% if not loop.last %}, {% endif %}" + "{% endfor %}" + "{% endif %}" + "))) SELECT * FROM pivot ORDER BY `{{ timestamp_column }}` " + "{% else %}" + "SELECT * FROM project " + "{% endif %}" + "{% if is_resample is defined and is_resample == true and limit is defined and limit is not none %}" + "LIMIT {{ limit }} " + "{% endif %}" + "{% if is_resample is defined and is_resample == true and offset is defined and offset is not none %}" + "OFFSET {{ offset }} " + "{% endif %}" + ) + + plot_parameters = { + "source": parameters_dict.get("source", None), + "business_unit": parameters_dict.get("business_unit"), + "region": parameters_dict.get("region"), + "asset": parameters_dict.get("asset"), + "data_security_level": parameters_dict.get("data_security_level"), + "data_type": parameters_dict.get("data_type"), + "start_date": parameters_dict["start_date"], + "end_date": parameters_dict["end_date"], + "tag_names": list(dict.fromkeys(parameters_dict["tag_names"])), + "include_bad_data": True, + "time_interval_rate": parameters_dict["time_interval_rate"], + "time_interval_unit": parameters_dict["time_interval_unit"], + "time_zone": parameters_dict["time_zone"], + "pivot": False, + "limit": parameters_dict.get("limit", None), + "offset": parameters_dict.get("offset", None), + "is_resample": True, + "tagname_column": parameters_dict.get("tagname_column", "TagName"), + "timestamp_column": parameters_dict.get("timestamp_column", "EventTime"), + "include_status": ( + False + if "status_column" in parameters_dict + and parameters_dict.get("status_column") is None + else True + ), + "status_column": ( + "Status" + if "status_column" in parameters_dict + and parameters_dict.get("status_column") is None + else parameters_dict.get("status_column", "Status") + ), + "value_column": parameters_dict.get("value_column", "Value"), + "range_join_seconds": parameters_dict["range_join_seconds"], + "case_insensitivity_tag_search": parameters_dict.get( + "case_insensitivity_tag_search", False + ), + } + + sql_template = Template(plot_query) + sql_query = sql_template.render(plot_parameters) + return sql_query, plot_query, plot_parameters + + def _interpolation_query( parameters_dict: dict, sample_query: str, sample_parameters: dict ) -> str: @@ -830,6 +927,15 @@ def _query_builder(parameters_dict: dict, query_type: str) -> str: ) return sample_prepared_query + if query_type == "plot": + parameters_dict["range_join_seconds"] = _convert_to_seconds( + parameters_dict["time_interval_rate"] + + " " + + parameters_dict["time_interval_unit"][0] + ) + plot_prepared_query, _, _ = _plot_query(parameters_dict) + return plot_prepared_query + if query_type == "interpolate": parameters_dict["range_join_seconds"] = _convert_to_seconds( parameters_dict["time_interval_rate"] diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/plot.py b/src/sdk/python/rtdip_sdk/queries/time_series/plot.py new file mode 100644 index 000000000..78f10f0ac --- /dev/null +++ b/src/sdk/python/rtdip_sdk/queries/time_series/plot.py @@ -0,0 +1,88 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import pandas as pd +from ._time_series_query_builder import _query_builder + + +def get(connection: object, parameters_dict: dict) -> pd.DataFrame: + """ + An RTDIP Resampling plot function in spark to resample data to find the First, Last, Min, Max and Exception Value for the time interval by querying databricks SQL warehouses using a connection and authentication method specified by the user. This spark resample function will return a resampled dataframe. + + The available connectors by RTDIP are Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect. + + The available authentication methods are Certificate Authentication, Client Secret Authentication or Default Authentication. See documentation. + + This function requires the user to input a dictionary of parameters. (See Attributes table below) + + Args: + connection: Connection chosen by the user (Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect) + parameters_dict: A dictionary of parameters (see Attributes table below) + + Attributes: + business_unit (str): Business unit of the data + region (str): Region + asset (str): Asset + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) + tag_names (list): List of tagname or tagnames ["tag_1", "tag_2"] + start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + sample_rate (int): (deprecated) Please use time_interval_rate instead. See below. + sample_unit (str): (deprecated) Please use time_interval_unit instead. See below. + time_interval_rate (str): The time interval rate (numeric input) + time_interval_unit (str): The time interval unit (second, minute, day, hour) + limit (optional int): The number of rows to be returned + offset (optional int): The number of rows to skip before returning rows + case_insensitivity_tag_search (optional bool): Search for tags using case insensitivity with True or case sensitivity with False + + Returns: + DataFrame: A resampled dataframe. + + !!! warning + Setting `case_insensitivity_tag_search` to True will result in a longer query time. + """ + if isinstance(parameters_dict["tag_names"], list) is False: + raise ValueError("tag_names must be a list") + + if "sample_rate" in parameters_dict: + logging.warning( + "Parameter sample_rate is deprecated and will be removed in v1.0.0. Please use time_interval_rate instead." + ) + parameters_dict["time_interval_rate"] = parameters_dict["sample_rate"] + + if "sample_unit" in parameters_dict: + logging.warning( + "Parameter sample_unit is deprecated and will be removed in v1.0.0. Please use time_interval_unit instead." + ) + parameters_dict["time_interval_unit"] = parameters_dict["sample_unit"] + + try: + query = _query_builder(parameters_dict, "plot") + + try: + cursor = connection.cursor() + cursor.execute(query) + df = cursor.fetch_all() + cursor.close() + connection.close() + return df + except Exception as e: + logging.exception("error returning dataframe") + raise e + + except Exception as e: + logging.exception("error with resampling function") + raise e diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/time_series_query_builder.py b/src/sdk/python/rtdip_sdk/queries/time_series/time_series_query_builder.py index 0a47d5da0..2c67333e3 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/time_series_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/time_series_query_builder.py @@ -17,6 +17,7 @@ from . import ( raw, resample, + plot, interpolate, interpolation_at_time, time_weighted_average, @@ -260,6 +261,79 @@ def resample( return resample.get(self.connection, resample_parameters) + def plot( + self, + tagname_filter: [str], + start_date: str, + end_date: str, + time_interval_rate: str, + time_interval_unit: str, + include_bad_data: bool = False, + limit: int = None, + offset: int = None, + ) -> DataFrame: + """ + A query to plot the source data for a time interval for Min, Max, First, Last and an Exception Value(Status = Bad), if it exists. + + **Example:** + ```python + from rtdip_sdk.authentication.azure import DefaultAuth + from rtdip_sdk.connectors import DatabricksSQLConnection + from rtdip_sdk.queries import TimeSeriesQueryBuilder + + auth = DefaultAuth().authenticate() + token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + + data = ( + TimeSeriesQueryBuilder() + .connect(connection) + .source("{table_path}") + .plot( + tagname_filter=["{tag_name_1}", "{tag_name_2}"], + start_date="2023-01-01", + end_date="2023-01-31", + time_interval_rate="15", + time_interval_unit="minute", + ) + ) + + display(data) + + ``` + + Args: + tagname_filter (list str): List of tagnames to filter on the source + start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + time_interval_rate (str): The time interval rate (numeric input) + time_interval_unit (str): The time interval unit (second, minute, day, hour) + limit (optional int): The number of rows to be returned + offset (optional int): The number of rows to skip before returning rows + + Returns: + DataFrame: A dataframe of resampled timeseries data. + """ + + plot_parameters = { + "source": self.data_source, + "tag_names": tagname_filter, + "start_date": start_date, + "end_date": end_date, + "include_bad_data": include_bad_data, + "time_interval_rate": time_interval_rate, + "time_interval_unit": time_interval_unit, + "limit": limit, + "offset": offset, + "tagname_column": self.tagname_column, + "timestamp_column": self.timestamp_column, + "status_column": self.status_column, + "value_column": self.value_column, + "supress_warning": True, + } + + return plot.get(self.connection, plot_parameters) + def interpolate( self, tagname_filter: [str], diff --git a/tests/api/v1/api_test_objects.py b/tests/api/v1/api_test_objects.py index 883320d87..7c414fa2d 100644 --- a/tests/api/v1/api_test_objects.py +++ b/tests/api/v1/api_test_objects.py @@ -93,6 +93,27 @@ "MOCKED-TAGNAME2", ] +PLOT_MOCKED_PARAMETER_DICT = RAW_MOCKED_PARAMETER_DICT.copy() +PLOT_MOCKED_PARAMETER_ERROR_DICT = RAW_MOCKED_PARAMETER_ERROR_DICT.copy() + +PLOT_MOCKED_PARAMETER_DICT["sample_rate"] = "15" +PLOT_MOCKED_PARAMETER_DICT["sample_unit"] = "minute" +PLOT_MOCKED_PARAMETER_DICT["time_interval_rate"] = "15" +PLOT_MOCKED_PARAMETER_DICT["time_interval_unit"] = "minute" +PLOT_MOCKED_PARAMETER_ERROR_DICT["sample_rate"] = "15" +PLOT_MOCKED_PARAMETER_ERROR_DICT["sample_unit"] = "minute" +PLOT_MOCKED_PARAMETER_ERROR_DICT["time_interval_rate"] = "1" +PLOT_MOCKED_PARAMETER_ERROR_DICT["time_interval_unit"] = "minute" + +PLOT_POST_MOCKED_PARAMETER_DICT = RESAMPLE_MOCKED_PARAMETER_DICT.copy() +PLOT_POST_MOCKED_PARAMETER_DICT.pop("tag_name") + +PLOT_POST_BODY_MOCKED_PARAMETER_DICT = {} +PLOT_POST_BODY_MOCKED_PARAMETER_DICT["tag_name"] = [ + "MOCKED-TAGNAME1", + "MOCKED-TAGNAME2", +] + INTERPOLATE_MOCKED_PARAMETER_DICT = RESAMPLE_MOCKED_PARAMETER_DICT.copy() INTERPOLATE_MOCKED_PARAMETER_ERROR_DICT = RESAMPLE_MOCKED_PARAMETER_ERROR_DICT.copy() diff --git a/tests/api/v1/test_api_plot.py b/tests/api/v1/test_api_plot.py new file mode 100644 index 000000000..717316227 --- /dev/null +++ b/tests/api/v1/test_api_plot.py @@ -0,0 +1,213 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from pytest_mock import MockerFixture +import pandas as pd +from datetime import datetime +from tests.api.v1.api_test_objects import ( + PLOT_MOCKED_PARAMETER_DICT, + PLOT_MOCKED_PARAMETER_ERROR_DICT, + PLOT_POST_MOCKED_PARAMETER_DICT, + PLOT_POST_BODY_MOCKED_PARAMETER_DICT, + mocker_setup, + TEST_HEADERS, + BASE_URL, +) +from httpx import AsyncClient +from src.api.v1 import app + +MOCK_METHOD = "src.sdk.python.rtdip_sdk.queries.time_series.plot.get" +MOCK_API_NAME = "/api/v1/events/plot" + +pytestmark = pytest.mark.anyio + + +async def test_api_plot_get_success(mocker: MockerFixture): + test_data = pd.DataFrame( + { + "EventTime": [datetime.utcnow()], + "TagName": ["TestTag"], + "Average": [1.01], + "Min": [1.01], + "Max": [1.01], + "First": [1.01], + "Last": [1.01], + "StdDev": [1.01], + } + ) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + response = await ac.get( + MOCK_API_NAME, headers=TEST_HEADERS, params=PLOT_MOCKED_PARAMETER_DICT + ) + actual = response.text + expected = test_data.to_json(orient="table", index=False, date_unit="ns") + expected = ( + expected.rstrip("}") + ',"pagination":{"limit":null,"offset":null,"next":null}}' + ) + + assert response.status_code == 200 + assert actual == expected + + +async def test_api_plot_get_validation_error(mocker: MockerFixture): + test_data = pd.DataFrame( + { + "EventTime": [datetime.utcnow()], + "TagName": ["TestTag"], + "Average": [1.01], + "Min": [1.01], + "Max": [1.01], + "First": [1.01], + "Last": [1.01], + "StdDev": [1.01], + } + ) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + response = await ac.get( + MOCK_API_NAME, + headers=TEST_HEADERS, + params=PLOT_MOCKED_PARAMETER_ERROR_DICT, + ) + actual = response.text + + assert response.status_code == 422 + assert ( + actual + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + ) + + +async def test_api_pot_get_error(mocker: MockerFixture): + test_data = pd.DataFrame( + { + "EventTime": [datetime.utcnow()], + "TagName": ["TestTag"], + "Average": [1.01], + "Min": [1.01], + "Max": [1.01], + "First": [1.01], + "Last": [1.01], + "StdDev": [1.01], + } + ) + mocker = mocker_setup( + mocker, MOCK_METHOD, test_data, Exception("Error Connecting to Database") + ) + + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + response = await ac.get( + MOCK_API_NAME, headers=TEST_HEADERS, params=PLOT_MOCKED_PARAMETER_DICT + ) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' + + +async def test_api_plot_post_success(mocker: MockerFixture): + test_data = pd.DataFrame( + { + "EventTime": [datetime.utcnow()], + "TagName": ["TestTag"], + "Average": [1.01], + "Min": [1.01], + "Max": [1.01], + "First": [1.01], + "Last": [1.01], + "StdDev": [1.01], + } + ) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + response = await ac.post( + MOCK_API_NAME, + headers=TEST_HEADERS, + params=PLOT_POST_MOCKED_PARAMETER_DICT, + json=PLOT_POST_BODY_MOCKED_PARAMETER_DICT, + ) + actual = response.text + expected = test_data.to_json(orient="table", index=False, date_unit="ns") + expected = ( + expected.rstrip("}") + ',"pagination":{"limit":null,"offset":null,"next":null}}' + ) + + assert response.status_code == 200 + assert actual == expected + + +async def test_api_plot_post_validation_error(mocker: MockerFixture): + test_data = pd.DataFrame( + { + "EventTime": [datetime.utcnow()], + "TagName": ["TestTag"], + "Average": [1.01], + "Min": [1.01], + "Max": [1.01], + "First": [1.01], + "Last": [1.01], + "StdDev": [1.01], + } + ) + mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + response = await ac.post( + MOCK_API_NAME, + headers=TEST_HEADERS, + params=PLOT_MOCKED_PARAMETER_ERROR_DICT, + json=PLOT_POST_BODY_MOCKED_PARAMETER_DICT, + ) + actual = response.text + + assert response.status_code == 422 + assert ( + actual + == '{"detail":[{"type":"missing","loc":["query","start_date"],"msg":"Field required","input":null,"url":"https://errors.pydantic.dev/2.6/v/missing"}]}' + ) + + +async def test_api_plot_post_error(mocker: MockerFixture): + test_data = pd.DataFrame( + { + "EventTime": [datetime.utcnow()], + "TagName": ["TestTag"], + "Average": [1.01], + "Min": [1.01], + "Max": [1.01], + "First": [1.01], + "Last": [1.01], + "StdDev": [1.01], + } + ) + mocker = mocker_setup( + mocker, MOCK_METHOD, test_data, Exception("Error Connecting to Database") + ) + + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + response = await ac.post( + MOCK_API_NAME, + headers=TEST_HEADERS, + params=PLOT_MOCKED_PARAMETER_DICT, + json=PLOT_POST_BODY_MOCKED_PARAMETER_DICT, + ) + actual = response.text + + assert response.status_code == 400 + assert actual == '{"detail":"Error Connecting to Database"}' diff --git a/tests/sdk/python/rtdip_sdk/queries/_test_utils/sdk_test_objects.py b/tests/sdk/python/rtdip_sdk/queries/_test_utils/sdk_test_objects.py index f750b17f7..df1713175 100644 --- a/tests/sdk/python/rtdip_sdk/queries/_test_utils/sdk_test_objects.py +++ b/tests/sdk/python/rtdip_sdk/queries/_test_utils/sdk_test_objects.py @@ -36,6 +36,8 @@ RESAMPLE_MOCKED_QUERY = 'WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`EventTime`, \'yyyy-MM-dd HH:mm:ss.SSS\')), "+0000") AS `EventTime`, `TagName`, `Status`, `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp("2011-01-01T00:00:00+00:00") AND to_timestamp("2011-01-02T23:59:59+00:00") AND `TagName` IN (\'mocked-TAGNAME\') ) ,date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS timestamp_array) ,window_buckets AS (SELECT timestamp_array AS window_start, timestampadd(minute, 15, timestamp_array) AS window_end FROM date_array) ,resample AS (SELECT /*+ RANGE_JOIN(d, 900 ) */ d.window_start, d.window_end, e.`TagName`, avg(e.`Value`) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `Value` FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`EventTime` AND d.window_end > e.`EventTime`) ,project AS (SELECT window_start AS `EventTime`, `TagName`, `Value` FROM resample GROUP BY window_start, `TagName`, `Value` ORDER BY `TagName`, `EventTime` ) SELECT * FROM project ' RESAMPLE_MOCKED_QUERY_CHECK_TAGS = 'WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`EventTime`, \'yyyy-MM-dd HH:mm:ss.SSS\')), "+0000") AS `EventTime`, `TagName`, `Status`, `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp("2011-01-01T00:00:00+00:00") AND to_timestamp("2011-01-02T23:59:59+00:00") AND UPPER(`TagName`) IN (\'MOCKED-TAGNAME\') ) ,date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS timestamp_array) ,window_buckets AS (SELECT timestamp_array AS window_start, timestampadd(minute, 15, timestamp_array) AS window_end FROM date_array) ,resample AS (SELECT /*+ RANGE_JOIN(d, 900 ) */ d.window_start, d.window_end, e.`TagName`, avg(e.`Value`) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `Value` FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`EventTime` AND d.window_end > e.`EventTime`) ,project AS (SELECT window_start AS `EventTime`, `TagName`, `Value` FROM resample GROUP BY window_start, `TagName`, `Value` ORDER BY `TagName`, `EventTime` ) SELECT * FROM project ' RESAMPLE_MOCKED_QUERY_PIVOT = 'WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`EventTime`, \'yyyy-MM-dd HH:mm:ss.SSS\')), "+0000") AS `EventTime`, `TagName`, `Status`, `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp("2011-01-01T00:00:00+00:00") AND to_timestamp("2011-01-02T23:59:59+00:00") AND `TagName` IN (\'mocked-TAGNAME\') ) ,date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS timestamp_array) ,window_buckets AS (SELECT timestamp_array AS window_start, timestampadd(minute, 15, timestamp_array) AS window_end FROM date_array) ,resample AS (SELECT /*+ RANGE_JOIN(d, 900 ) */ d.window_start, d.window_end, e.`TagName`, avg(e.`Value`) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `Value` FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`EventTime` AND d.window_end > e.`EventTime`) ,project AS (SELECT window_start AS `EventTime`, `TagName`, `Value` FROM resample GROUP BY window_start, `TagName`, `Value` ORDER BY `TagName`, `EventTime` ) ,pivot AS (SELECT * FROM (SELECT `EventTime`, `Value`, `TagName` AS `TagName` FROM project) PIVOT (FIRST(`Value`) FOR `TagName` IN (\'mocked-TAGNAME\' AS `mocked-TAGNAME`))) SELECT * FROM pivot ORDER BY `EventTime` ' +PLOT_MOCKED_QUERY = "WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`EventTime`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"+0000\") AS `EventTime`, `TagName`, `Status`, `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp(\"2011-01-01T00:00:00+00:00\") AND to_timestamp(\"2011-01-02T23:59:59+00:00\") AND `TagName` IN ('mocked-TAGNAME') ) ,date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp(\"2011-01-01T00:00:00+00:00\"), \"+0000\"), from_utc_timestamp(to_timestamp(\"2011-01-02T23:59:59+00:00\"), \"+0000\"), INTERVAL '15 minute')) AS timestamp_array) ,window_buckets AS (SELECT timestamp_array AS window_start, timestampadd(minute, 15, timestamp_array) AS window_end FROM date_array) ,plot AS (SELECT /*+ RANGE_JOIN(d, 900 ) */ d.window_start, d.window_end, e.`TagName`, min(CASE WHEN `Status` = 'Bad' THEN null ELSE struct(e.`Value`, e.`EventTime`) END) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `min_Value`, max(CASE WHEN `Status` = 'Bad' THEN null ELSE struct(e.`Value`, e.`EventTime`) END) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `max_Value`, first(CASE WHEN `Status` = 'Bad' THEN null ELSE struct(e.`Value`, e.`EventTime`) END, True) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `first_Value`, last(CASE WHEN `Status` = 'Bad' THEN null ELSE struct(e.`Value`, e.`EventTime`) END, True) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `last_Value`, first(CASE WHEN `Status` = 'Bad' THEN struct(e.`Value`, e.`EventTime`) ELSE null END, True) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `excp_Value` FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`EventTime` AND d.window_end > e.`EventTime`) ,deduplicate AS (SELECT window_start AS `EventTime`, `TagName`, `min_Value` as `Min`, `max_Value` as `Max`, `first_Value` as `First`, `last_Value` as `Last`, `excp_Value` as `Exception` FROM plot GROUP BY window_start, `TagName`, `min_Value`, `max_Value`, `first_Value`, `last_Value`, `excp_Value`) ,project AS (SELECT distinct Values.EventTime, `TagName`, Values.Value FROM (SELECT * FROM deduplicate UNPIVOT (`Values` for `Aggregation` IN (`Min`, `Max`, `First`, `Last`, `Exception`))) ORDER BY `TagName`, `EventTime` ) SELECT * FROM project " +PLOT_MOCKED_QUERY_CHECK_TAGS = "WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`EventTime`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"+0000\") AS `EventTime`, `TagName`, `Status`, `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp(\"2011-01-01T00:00:00+00:00\") AND to_timestamp(\"2011-01-02T23:59:59+00:00\") AND UPPER(`TagName`) IN ('MOCKED-TAGNAME') ) ,date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp(\"2011-01-01T00:00:00+00:00\"), \"+0000\"), from_utc_timestamp(to_timestamp(\"2011-01-02T23:59:59+00:00\"), \"+0000\"), INTERVAL '15 minute')) AS timestamp_array) ,window_buckets AS (SELECT timestamp_array AS window_start, timestampadd(minute, 15, timestamp_array) AS window_end FROM date_array) ,plot AS (SELECT /*+ RANGE_JOIN(d, 900 ) */ d.window_start, d.window_end, e.`TagName`, min(CASE WHEN `Status` = 'Bad' THEN null ELSE struct(e.`Value`, e.`EventTime`) END) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `min_Value`, max(CASE WHEN `Status` = 'Bad' THEN null ELSE struct(e.`Value`, e.`EventTime`) END) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `max_Value`, first(CASE WHEN `Status` = 'Bad' THEN null ELSE struct(e.`Value`, e.`EventTime`) END, True) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `first_Value`, last(CASE WHEN `Status` = 'Bad' THEN null ELSE struct(e.`Value`, e.`EventTime`) END, True) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `last_Value`, first(CASE WHEN `Status` = 'Bad' THEN struct(e.`Value`, e.`EventTime`) ELSE null END, True) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `excp_Value` FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`EventTime` AND d.window_end > e.`EventTime`) ,deduplicate AS (SELECT window_start AS `EventTime`, `TagName`, `min_Value` as `Min`, `max_Value` as `Max`, `first_Value` as `First`, `last_Value` as `Last`, `excp_Value` as `Exception` FROM plot GROUP BY window_start, `TagName`, `min_Value`, `max_Value`, `first_Value`, `last_Value`, `excp_Value`) ,project AS (SELECT distinct Values.EventTime, `TagName`, Values.Value FROM (SELECT * FROM deduplicate UNPIVOT (`Values` for `Aggregation` IN (`Min`, `Max`, `First`, `Last`, `Exception`))) ORDER BY `TagName`, `EventTime` ) SELECT * FROM project " INTERPOLATE_MOCKED_QUERY = 'WITH resample AS (WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`EventTime`, \'yyyy-MM-dd HH:mm:ss.SSS\')), "+0000") AS `EventTime`, `TagName`, `Status`, `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp("2011-01-01T00:00:00+00:00") AND to_timestamp("2011-01-02T23:59:59+00:00") AND `TagName` IN (\'mocked-TAGNAME\') ) ,date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS timestamp_array) ,window_buckets AS (SELECT timestamp_array AS window_start, timestampadd(minute, 15, timestamp_array) AS window_end FROM date_array) ,resample AS (SELECT /*+ RANGE_JOIN(d, 900 ) */ d.window_start, d.window_end, e.`TagName`, avg(e.`Value`) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `Value` FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`EventTime` AND d.window_end > e.`EventTime`) ,project AS (SELECT window_start AS `EventTime`, `TagName`, `Value` FROM resample GROUP BY window_start, `TagName`, `Value` ) SELECT * FROM project ),date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS `EventTime`, explode(array(\'mocked-TAGNAME\')) AS `TagName`) ,project AS (SELECT a.`EventTime`, a.`TagName`, last_value(b.`Value`, true) OVER (PARTITION BY a.`TagName` ORDER BY a.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `Value` FROM date_array a LEFT OUTER JOIN resample b ON a.`EventTime` = b.`EventTime` AND a.`TagName` = b.`TagName`) SELECT * FROM project ORDER BY `TagName`, `EventTime` ' INTERPOLATE_MOCKED_QUERY_BACKWARD_FILL = 'WITH resample AS (WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`EventTime`, \'yyyy-MM-dd HH:mm:ss.SSS\')), "+0000") AS `EventTime`, `TagName`, `Status`, `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp("2011-01-01T00:00:00+00:00") AND to_timestamp("2011-01-02T23:59:59+00:00") AND `TagName` IN (\'mocked-TAGNAME\') ) ,date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS timestamp_array) ,window_buckets AS (SELECT timestamp_array AS window_start, timestampadd(minute, 15, timestamp_array) AS window_end FROM date_array) ,resample AS (SELECT /*+ RANGE_JOIN(d, 900 ) */ d.window_start, d.window_end, e.`TagName`, avg(e.`Value`) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `Value` FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`EventTime` AND d.window_end > e.`EventTime`) ,project AS (SELECT window_start AS `EventTime`, `TagName`, `Value` FROM resample GROUP BY window_start, `TagName`, `Value` ) SELECT * FROM project ),date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS `EventTime`, explode(array(\'mocked-TAGNAME\')) AS `TagName`) ,project AS (SELECT a.`EventTime`, a.`TagName`, first_value(b.`Value`, true) OVER (PARTITION BY a.`TagName` ORDER BY a.`EventTime` ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) AS `Value` FROM date_array a LEFT OUTER JOIN resample b ON a.`EventTime` = b.`EventTime` AND a.`TagName` = b.`TagName`) SELECT * FROM project ORDER BY `TagName`, `EventTime` ' INTERPOLATE_MOCKED_QUERY_CHECK_TAGS = 'WITH resample AS (WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`EventTime`, \'yyyy-MM-dd HH:mm:ss.SSS\')), "+0000") AS `EventTime`, `TagName`, `Status`, `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp("2011-01-01T00:00:00+00:00") AND to_timestamp("2011-01-02T23:59:59+00:00") AND UPPER(`TagName`) IN (\'MOCKED-TAGNAME\') ) ,date_array AS (SELECT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS timestamp_array) ,window_buckets AS (SELECT timestamp_array AS window_start, timestampadd(minute, 15, timestamp_array) AS window_end FROM date_array) ,resample AS (SELECT /*+ RANGE_JOIN(d, 900 ) */ d.window_start, d.window_end, e.`TagName`, avg(e.`Value`) OVER (PARTITION BY e.`TagName`, d.window_start ORDER BY e.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS `Value` FROM window_buckets d INNER JOIN raw_events e ON d.window_start <= e.`EventTime` AND d.window_end > e.`EventTime`) ,project AS (SELECT window_start AS `EventTime`, `TagName`, `Value` FROM resample GROUP BY window_start, `TagName`, `Value` ) SELECT * FROM project ),date_array AS (SELECT DISTINCT explode(sequence(from_utc_timestamp(to_timestamp("2011-01-01T00:00:00+00:00"), "+0000"), from_utc_timestamp(to_timestamp("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS `EventTime`, explode(array(`TagName`)) AS `TagName` FROM resample) ,project AS (SELECT a.`EventTime`, a.`TagName`, last_value(b.`Value`, true) OVER (PARTITION BY a.`TagName` ORDER BY a.`EventTime` ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS `Value` FROM date_array a LEFT OUTER JOIN resample b ON a.`EventTime` = b.`EventTime` AND a.`TagName` = b.`TagName`) SELECT * FROM project ORDER BY `TagName`, `EventTime` ' diff --git a/tests/sdk/python/rtdip_sdk/queries/time_series/test_plot.py b/tests/sdk/python/rtdip_sdk/queries/time_series/test_plot.py new file mode 100644 index 000000000..320728edc --- /dev/null +++ b/tests/sdk/python/rtdip_sdk/queries/time_series/test_plot.py @@ -0,0 +1,85 @@ +# Copyright 2022 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +sys.path.insert(0, ".") +from pytest_mock import MockerFixture +from src.sdk.python.rtdip_sdk.connectors import DatabricksSQLConnection +from src.sdk.python.rtdip_sdk.queries.time_series.plot import get as plot_get +from tests.sdk.python.rtdip_sdk.queries.time_series._test_base import ( + _test_base_succeed, + _test_base_fails, +) +from tests.sdk.python.rtdip_sdk.queries._test_utils.sdk_test_objects import ( + MOCKED_QUERY_OFFSET_LIMIT, + MOCKED_PARAMETER_DICT, + PLOT_MOCKED_QUERY, + PLOT_MOCKED_QUERY_CHECK_TAGS, +) + +MOCKED_PLOT_PARAMETER_DICT = MOCKED_PARAMETER_DICT.copy() +MOCKED_PLOT_PARAMETER_DICT["time_interval_rate"] = "15" +MOCKED_PLOT_PARAMETER_DICT["time_interval_unit"] = "minute" + + +def test_plot_success(mocker: MockerFixture): + _test_base_succeed( + mocker, + MOCKED_PLOT_PARAMETER_DICT, + PLOT_MOCKED_QUERY, + plot_get, + ) + + +def test_plot_check_tags(mocker: MockerFixture): + MOCKED_PLOT_PARAMETER_DICT["case_insensitivity_tag_search"] = True + _test_base_succeed( + mocker, + MOCKED_PLOT_PARAMETER_DICT, + PLOT_MOCKED_QUERY_CHECK_TAGS, + plot_get, + ) + + +def test_plot_sample_rate_unit(mocker: MockerFixture): + MOCKED_PLOT_PARAMETER_DICT["case_insensitivity_tag_search"] = False + MOCKED_PLOT_PARAMETER_DICT["sample_rate"] = "15" + MOCKED_PLOT_PARAMETER_DICT["sample_unit"] = "minute" + _test_base_succeed( + mocker, + MOCKED_PLOT_PARAMETER_DICT, + PLOT_MOCKED_QUERY, + plot_get, + ) + + +def test_plot_offset_limit(mocker: MockerFixture): + MOCKED_PLOT_PARAMETER_DICT["offset"] = 10 + MOCKED_PLOT_PARAMETER_DICT["limit"] = 10 + _test_base_succeed( + mocker, + MOCKED_PLOT_PARAMETER_DICT, + (PLOT_MOCKED_QUERY + MOCKED_QUERY_OFFSET_LIMIT), + plot_get, + ) + + +def test_plot_fails(mocker: MockerFixture): + _test_base_fails(mocker, MOCKED_PLOT_PARAMETER_DICT, plot_get) + + +def test_plot_tag_name_not_list_fails(mocker: MockerFixture): + MOCKED_PLOT_PARAMETER_DICT["tag_names"] = "abc" + _test_base_fails(mocker, MOCKED_PLOT_PARAMETER_DICT, plot_get) diff --git a/tests/sdk/python/rtdip_sdk/queries/time_series/test_query_builder.py b/tests/sdk/python/rtdip_sdk/queries/time_series/test_query_builder.py index ad1abf176..15e30ea3e 100644 --- a/tests/sdk/python/rtdip_sdk/queries/time_series/test_query_builder.py +++ b/tests/sdk/python/rtdip_sdk/queries/time_series/test_query_builder.py @@ -61,6 +61,27 @@ def test_query_builder_resample(mocker: MockerFixture): assert data == {"test": "data"} +def test_query_builder_plot(mocker: MockerFixture): + mocker.patch( + "src.sdk.python.rtdip_sdk.queries.time_series.time_series_query_builder.plot.get", + return_value={"test": "data"}, + ) + + data = ( + TimeSeriesQueryBuilder() + .connect(MOCK_CONNECTION) + .source(MOCK_TABLE) + .plot( + tagname_filter=["mock_tag"], + start_date="2021-01-01", + end_date="2021-01-02", + time_interval_rate="1", + time_interval_unit="hour", + ) + ) + assert data == {"test": "data"} + + def test_query_builder_interpolate(mocker: MockerFixture): mocker.patch( "src.sdk.python.rtdip_sdk.queries.time_series.time_series_query_builder.interpolate.get",