Thanks to visit codestin.com
Credit goes to github.com

Skip to content

v0.12.2 #823

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 2, 2024
Merged

v0.12.2 #823

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ec5c9fe
Query changes related to date_format, this will improve the performan…
srjhunjhunwalacorp Aug 22, 2024
3c4a1da
Revert "Query changes related to date_format, this will improve the p…
cching95 Aug 27, 2024
3ba9871
Changed Date Formatting for Performance (#817)
cching95 Aug 28, 2024
79838dc
fix interpolation at time bug (#819)
cching95 Aug 30, 2024
53f78a8
add to_json back into sql query (#821)
cching95 Sep 2, 2024
4a46098
API: Update batch api parsing to handle date columns correctly (#822)
ummer-shell Sep 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions environment.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ dependencies:
- mkdocstrings==0.25.0
- mkdocstrings-python==1.10.8
- mkdocs-macros-plugin==1.0.1
- mkdocs-autorefs>=1.0.0,<1.1.0
- pygments==2.16.1
- pymdown-extensions==10.8.1
- databricks-sql-connector>=3.1.0,<4.0.0
Expand Down
10 changes: 8 additions & 2 deletions src/api/v1/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

from datetime import datetime
import datetime as dt
import json
import os
import importlib.util
Expand Down Expand Up @@ -240,13 +241,15 @@ def get_as_dict(data):
def convert_value(x):
if isinstance(x, pd.Timestamp):
return x.isoformat(timespec="nanoseconds")
elif isinstance(x, dt.date):
return x.isoformat()
elif isinstance(x, pd.Timedelta):
return x.isoformat()
elif isinstance(x, Decimal):
return float(x)
return x

data_parsed = data.map(convert_value).replace({np.nan: None})
data_parsed = data.applymap(convert_value).replace({np.nan: None})
schema = build_table_schema(data_parsed, index=False, primary_key=False)
data_dict = data_parsed.to_dict(orient="records")

Expand Down Expand Up @@ -291,9 +294,12 @@ def lookup_before_get(
# run function with each parameters concurrently
results = batch.get(connection, request_list, threadpool_max_workers=max_workers)

# Check if pivot is required
should_pivot = parameters["pivot"] if "pivot" in parameters else False

# Append/concat results as required
data = concatenate_dfs_and_order(
dfs_arr=results, pivot=False, tags=parameters["tag_names"]
dfs_arr=results, pivot=should_pivot, tags=parameters["tag_names"]
)

return data
Expand Down
15 changes: 6 additions & 9 deletions src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

def _raw_query(parameters_dict: dict) -> str:
raw_query = (
"WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`{{ timestamp_column }}`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"{{ time_zone }}\") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% endif %} `{{ value_column }}` FROM "
'WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(date_trunc("millisecond",`{{ timestamp_column }}`), "{{ time_zone }}") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% endif %} `{{ value_column }}` FROM '
"{% if source is defined and source is not none %}"
"`{{ source|lower }}` "
"{% else %}"
Expand Down Expand Up @@ -129,7 +129,7 @@ def _sql_query(parameters_dict: dict) -> str:

def _sample_query(parameters_dict: dict) -> tuple:
sample_query = (
"WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`{{ timestamp_column }}`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"{{ time_zone }}\") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} 'Good' AS `Status`, {% endif %} `{{ value_column }}` FROM "
'WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(date_trunc("millisecond",`{{ timestamp_column }}`), "{{ time_zone }}") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} \'Good\' AS `Status`, {% endif %} `{{ value_column }}` FROM '
"{% if source is defined and source is not none %}"
"`{{ source|lower }}` "
"{% else %}"
Expand Down Expand Up @@ -227,7 +227,7 @@ def _sample_query(parameters_dict: dict) -> tuple:

def _plot_query(parameters_dict: dict) -> tuple:
plot_query = (
"WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`{{ timestamp_column }}`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"{{ time_zone }}\") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} 'Good' AS `Status`, {% endif %} `{{ value_column }}` FROM "
'WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(date_trunc("millisecond",`{{ timestamp_column }}`), "{{ time_zone }}") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} \'Good\' AS `Status`, {% endif %} `{{ value_column }}` FROM '
"{% if source is defined and source is not none %}"
"`{{ source|lower }}` "
"{% else %}"
Expand Down Expand Up @@ -416,7 +416,7 @@ def _interpolation_at_time(parameters_dict: dict) -> str:
parameters_dict["max_timestamp"] = max(timestamps_deduplicated)

interpolate_at_time_query = (
"WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`{{ timestamp_column }}`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"{{ time_zone }}\") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} 'Good' AS `Status`, {% endif %} `{{ value_column }}` FROM "
'WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(date_trunc("millisecond",`{{ timestamp_column }}`), "{{ time_zone }}") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} \'Good\' AS `Status`, {% endif %} `{{ value_column }}` FROM '
"{% if source is defined and source is not none %}"
"`{{ source|lower }}` "
"{% else %}"
Expand Down Expand Up @@ -629,7 +629,7 @@ def _time_weighted_average_query(parameters_dict: dict) -> str:
).strftime("%Y-%m-%dT%H:%M:%S")

time_weighted_average_query = (
"WITH raw_events AS (SELECT DISTINCT `{{ tagname_column }}`, from_utc_timestamp(to_timestamp(date_format(`{{ timestamp_column }}`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"{{ time_zone }}\") AS `{{ timestamp_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} 'Good' AS `Status`, {% endif %} `{{ value_column }}` FROM "
'WITH raw_events AS (SELECT DISTINCT `{{ tagname_column }}`, from_utc_timestamp(date_trunc("millisecond",`{{ timestamp_column }}`), "{{ time_zone }}") AS `{{ timestamp_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} \'Good\' AS `Status`, {% endif %} `{{ value_column }}` FROM '
"{% if source is defined and source is not none %}"
"`{{ source|lower }}` "
"{% else %}"
Expand Down Expand Up @@ -751,7 +751,7 @@ def _time_weighted_average_query(parameters_dict: dict) -> str:

def _circular_stats_query(parameters_dict: dict) -> str:
circular_base_query = (
"WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(to_timestamp(date_format(`{{ timestamp_column }}`, 'yyyy-MM-dd HH:mm:ss.SSS')), \"{{ time_zone }}\") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} 'Good' AS `Status`, {% endif %} `{{ value_column }}` FROM "
'WITH raw_events AS (SELECT DISTINCT from_utc_timestamp(date_trunc("millisecond",`{{ timestamp_column }}`), "{{ time_zone }}") AS `{{ timestamp_column }}`, `{{ tagname_column }}`, {% if include_status is defined and include_status == true %} `{{ status_column }}`, {% else %} \'Good\' AS `Status`, {% endif %} `{{ value_column }}` FROM '
"{% if source is defined and source is not none %}"
"`{{ source|lower }}` "
"{% else %}"
Expand Down Expand Up @@ -1020,13 +1020,10 @@ def _query_builder(parameters_dict: dict, query_type: str) -> str:
+ " "
+ parameters_dict["time_interval_unit"][0]
)
to_json = parameters_dict.get("to_json", False)
parameters_dict["to_json"] = False
sample_prepared_query, sample_query, sample_parameters = _sample_query(
parameters_dict
)
sample_parameters["is_resample"] = False
parameters_dict["to_json"] = to_json
return _interpolation_query(parameters_dict, sample_query, sample_parameters)

if query_type == "time_weighted_average":
Expand Down
Loading