From 657c9d9197458b050e6878e3b420404c22cb3abf Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 23 Jul 2024 12:34:07 +0100 Subject: [PATCH 1/8] API Memory Management Updates Signed-off-by: GBBBAS --- src/api/v1/common.py | 16 +- .../python/rtdip_sdk/connectors/__init__.py | 1 + src/sdk/python/rtdip_sdk/connectors/models.py | 22 + .../connectors/odbc/db_sql_connector.py | 50 +- .../connectors/odbc/turbodbc_sql_connector.py | 23 +- .../pipelines/_pipeline_utils/constants.py | 2 +- tests/api/v1/test_api_raw.py | 1035 +++++++++++++++++ .../spark/test_ssip_pi_binary_file_to_pcdm.py | 2 +- 8 files changed, 1131 insertions(+), 20 deletions(-) create mode 100644 src/sdk/python/rtdip_sdk/connectors/models.py diff --git a/src/api/v1/common.py b/src/api/v1/common.py index b50e75390..f59bb096c 100644 --- a/src/api/v1/common.py +++ b/src/api/v1/common.py @@ -21,8 +21,12 @@ from fastapi import Response import dateutil.parser from pandas import DataFrame +import pyarrow as pa from pandas.io.json import build_table_schema -from src.sdk.python.rtdip_sdk.connectors import DatabricksSQLConnection +from src.sdk.python.rtdip_sdk.connectors import ( + DatabricksSQLConnection, + ConnectionReturnType, +) if importlib.util.find_spec("turbodbc") != None: from src.sdk.python.rtdip_sdk.connectors import TURBODBCSQLConnection @@ -69,12 +73,14 @@ def common_api_setup_tasks( # NOSONAR databricks_server_host_name, databricks_http_path, token, + ConnectionReturnType.String, ) else: connection = DatabricksSQLConnection( databricks_server_host_name, databricks_http_path, token, + ConnectionReturnType.String, ) parameters = base_query_parameters.__dict__ @@ -178,11 +184,11 @@ def datetime_parser(json_dict): def json_response( - data: DataFrame, limit_offset_parameters: LimitOffsetQueryParams + data: str, limit_offset_parameters: LimitOffsetQueryParams ) -> Response: schema_df = pd.DataFrame() - if not data.empty: - json_str = data.loc[0, "Value"] + if data is not None and data != "": + json_str = data[0 : data.find("}") + 1] json_dict = json.loads(json_str, object_hook=datetime_parser) schema_df = pd.json_normalize(json_dict) @@ -192,7 +198,7 @@ def json_response( FieldSchema.model_validate( build_table_schema(schema_df, index=False, primary_key=False), ).model_dump_json(), - "[" + ",".join(data["Value"]) + "]", + "[" + data + "]", # data.replace({np.nan: None}).to_json( # orient="records", date_format="iso", date_unit="ns" # ), diff --git a/src/sdk/python/rtdip_sdk/connectors/__init__.py b/src/sdk/python/rtdip_sdk/connectors/__init__.py index e52897d02..824c69ba2 100644 --- a/src/sdk/python/rtdip_sdk/connectors/__init__.py +++ b/src/sdk/python/rtdip_sdk/connectors/__init__.py @@ -22,3 +22,4 @@ if importlib.util.find_spec("pyspark") != None: from .grpc.spark_connector import * from .llm.chatopenai_databricks_connector import * +from .models import * diff --git a/src/sdk/python/rtdip_sdk/connectors/models.py b/src/sdk/python/rtdip_sdk/connectors/models.py new file mode 100644 index 000000000..31c8e2f2c --- /dev/null +++ b/src/sdk/python/rtdip_sdk/connectors/models.py @@ -0,0 +1,22 @@ +# Copyright 2024 RTDIP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum + + +class ConnectionReturnType(str, Enum): + Pandas = "pandas" + Pyarrow = "pyarrow" + List = "list" + String = "string" diff --git a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py index c02031ff5..b26b8792c 100644 --- a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py +++ b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py @@ -14,8 +14,10 @@ from databricks import sql import pyarrow as pa +import polars as pl from ..connection_interface import ConnectionInterface from ..cursor_interface import CursorInterface +from ..models import ConnectionReturnType import logging @@ -32,10 +34,17 @@ class DatabricksSQLConnection(ConnectionInterface): access_token: Azure AD or Databricks PAT token """ - def __init__(self, server_hostname: str, http_path: str, access_token: str) -> None: + def __init__( + self, + server_hostname: str, + http_path: str, + access_token: str, + return_type=ConnectionReturnType.Pandas, + ) -> None: self.server_hostname = server_hostname self.http_path = http_path self.access_token = access_token + self.return_type = return_type # call auth method self.connection = self._connect() @@ -70,7 +79,7 @@ def cursor(self) -> object: try: if self.connection.open == False: self.connection = self._connect() - return DatabricksSQLCursor(self.connection.cursor()) + return DatabricksSQLCursor(self.connection.cursor(), self.return_type) except Exception as e: logging.exception("error with cursor object") raise e @@ -84,8 +93,9 @@ class DatabricksSQLCursor(CursorInterface): cursor: controls execution of commands on cluster or SQL Warehouse """ - def __init__(self, cursor: object) -> None: + def __init__(self, cursor: object, return_type=ConnectionReturnType.Pandas) -> None: self.cursor = cursor + self.return_type = return_type def execute(self, query: str) -> None: """ @@ -109,16 +119,40 @@ def fetch_all(self, fetch_size=5_000_000) -> list: """ try: get_next_result = True - results = [] + results = None if ConnectionReturnType.String else [] while get_next_result: result = self.cursor.fetchmany_arrow(fetch_size) - results.append(result) + if self.return_type == ConnectionReturnType.List: + column_list = [] + for column in result.columns: + column_list.append(column.to_pylist()) + results.extend(zip(*column_list)) + elif self.return_type == ConnectionReturnType.String: + column_list = [] + for column in result.columns: + column_list.append(column.to_pylist()) + + strings = ",".join([str(item[0]) for item in zip(*column_list)]) + if results is None: + results = strings + else: + results = ",".join([results, strings]) + else: + results.append(result) if result.num_rows < fetch_size: get_next_result = False - pyarrow_table = pa.concat_tables(results) - df = pyarrow_table.to_pandas() - return df + if self.return_type == ConnectionReturnType.Pandas: + pyarrow_table = pa.concat_tables(results) + return pyarrow_table.to_pandas() + elif self.return_type == ConnectionReturnType.Pyarrow: + pyarrow_table = pa.concat_tables(results) + return pyarrow_table + elif self.return_type in ( + ConnectionReturnType.List, + ConnectionReturnType.String, + ): + return results except Exception as e: logging.exception("error while fetching the rows of a query") raise e diff --git a/src/sdk/python/rtdip_sdk/connectors/odbc/turbodbc_sql_connector.py b/src/sdk/python/rtdip_sdk/connectors/odbc/turbodbc_sql_connector.py index b1da5b285..30608e420 100644 --- a/src/sdk/python/rtdip_sdk/connectors/odbc/turbodbc_sql_connector.py +++ b/src/sdk/python/rtdip_sdk/connectors/odbc/turbodbc_sql_connector.py @@ -17,6 +17,7 @@ from ..._sdk_utils.compare_versions import _package_version_meets_minimum from ..connection_interface import ConnectionInterface from ..cursor_interface import CursorInterface +from ..models import ConnectionReturnType import logging import os @@ -37,11 +38,18 @@ class TURBODBCSQLConnection(ConnectionInterface): More fields such as driver can be configured upon extension. """ - def __init__(self, server_hostname: str, http_path: str, access_token: str) -> None: + def __init__( + self, + server_hostname: str, + http_path: str, + access_token: str, + return_type=ConnectionReturnType.Pandas, + ) -> None: _package_version_meets_minimum("turbodbc", "4.0.0") self.server_hostname = server_hostname self.http_path = http_path self.access_token = access_token + self.return_type = return_type # call auth method self.connection = self._connect() self.open = True @@ -97,7 +105,9 @@ def cursor(self) -> object: try: if self.open == False: self.connection = self._connect() - return TURBODBCSQLCursor(self.connection.cursor()) + return TURBODBCSQLCursor( + self.connection.cursor(), return_type=self.return_type + ) except Exception as e: logging.exception("error with cursor object") raise e @@ -111,8 +121,9 @@ class TURBODBCSQLCursor(CursorInterface): cursor: controls execution of commands on cluster or SQL Warehouse """ - def __init__(self, cursor: object) -> None: + def __init__(self, cursor: object, return_type=ConnectionReturnType.Pandas) -> None: self.cursor = cursor + self.return_type = return_type def execute(self, query: str) -> None: """ @@ -136,8 +147,10 @@ def fetch_all(self) -> list: """ try: result = self.cursor.fetchallarrow() - df = result.to_pandas() - return df + if self.return_type == ConnectionReturnType.Pyarrow: + return result + elif self.return_type == ConnectionReturnType.Pandas: + return result.to_pandas() except Exception as e: logging.exception("error while fetching the rows from the query") raise e diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py index 5af5f2f46..7c0d32112 100644 --- a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py +++ b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/constants.py @@ -65,7 +65,7 @@ def get_default_package(package_name): "aws_boto3": PyPiLibrary(name="boto3", version="1.28.2"), "hashicorp_vault": PyPiLibrary(name="hvac", version="1.1.0"), "api_requests": PyPiLibrary(name="requests", version="2.30.0"), - "pyarrow": PyPiLibrary(name="pyarrow", version="12.0.0"), + "pyarrow": PyPiLibrary(name="pyarrow", version="14.0.2"), "pandas": PyPiLibrary(name="pandas", version="2.0.1"), } return DEFAULT_PACKAGES[package_name] diff --git a/tests/api/v1/test_api_raw.py b/tests/api/v1/test_api_raw.py index 07385fb88..805e4c858 100644 --- a/tests/api/v1/test_api_raw.py +++ b/tests/api/v1/test_api_raw.py @@ -17,6 +17,7 @@ import pandas as pd import numpy as np from datetime import datetime, timezone +from src.sdk.python.rtdip_sdk.authentication.azure import DefaultAuth from tests.api.v1.api_test_objects import ( RAW_MOCKED_PARAMETER_DICT, RAW_MOCKED_PARAMETER_ERROR_DICT, @@ -140,3 +141,1037 @@ async def test_api_raw_post_error(mocker: MockerFixture, api_test_data): assert response.status_code == 400 assert actual == '{"detail":"Error Connecting to Database"}' + + +async def test_api_raw_post_success_real(): + auth = DefaultAuth().authenticate() + token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + params = { + "business_unit": "downstream", + "region": "emea", + "asset": "pernis", + "data_security_level": "restricted", + "data_type": "float", + "include_bad_data": True, + "start_date": "2023-01-01", + "end_date": "2023-01-31", + } + body = { + "tag_name": [ + "HYCON:15TT115.PNT", + "HYCON:15TC303.SPT", + "HYCON:15TC309.MEAS", + "HYCON:15LW028.CUSUM", + "HYCON:15LC300.SPT", + "HYCON:15FT012.PNT", + "HYCON:15LT407.MEAS", + "HYCON:15HC063.OUT", + "HYCON:11TW675.UPTIME", + "HYCON:12XX150.HS", + "HYCON:12XX758.HS", + "HYCON:11TX215.DOW", + "HYCON:12TI727.CHK", + "HYCON:11TI735.CHK", + "HYCON:11TW688.CUSUM", + "HYCON:11TW336.CUSUM", + "HYCON:11TW678.CUSUM", + "HYCON:11TW338.CUSUM", + "HYCON:11TW653.CUSUM", + "HYCON:11TW361.CUSUM", + "HYCON:11XX107.PV.OUT", + "HYCON:12TD656.MEAS", + "HYCON:12XX138.PV", + "HYCON:11XX089.PV", + "HYCON:14XX120.PV", + "HYCON:12TD246.PNT", + "HYCON:11TY637.PNT", + "HYCON:12TT316.MEAS", + "HYCON:12X097.PNT", + "HYCON:14TT026.PNT", + "HYCON:14XY100.PNT", + "HYCON:11X106.PNT", + "HYCON:11X142.PNT", + "HYCON:11XX252.PV", + "HYCON:13PC009.OUT", + "HYCON:11XX030.PV", + "HYCON:11XX766.PV", + "HYCON:12XX013.PV", + "HYCON:12TY405.MEAS", + "HYCON:12XX332.PV", + "HYCON:11TI266.MEAS", + "HYCON:12TD221.PNT", + "HYCON:12TY304.PNT", + "HYCON:12GT009.PNT", + "HYCON:12TT198.MEAS", + "HYCON:12TT621_2.PNT", + "HYCON:11TT915.PNT", + "HYCON:12TD719.MEAS", + "HYCON:12XX324.PV", + "HYCON:11XX211.PV", + "HYCON:12XX104.PV", + "HYCON:12TT358.MEAS", + "HYCON:12TD622_2.PNT", + "HYCON:12TI678.PNT", + "HYCON:12TI448.MEAS", + "HYCON:11TI282.PNT", + "HYCON:11XX049.PV", + "HYCON:12TT886.PNT", + "HYCON:11XX019.PV", + "HYCON:12X155A.PV", + "HYCON:11X765D.PV", + "HYCON:11TY323.PNT", + "HYCON:11TT1197.PNT", + "HYCON:11TT312.MEAS", + "HYCON:11TT622_1.PNT", + "HYCON:11TT219.MEAS", + "HYCON:11TT803.PNT", + "HYCON:11TY373.PNT", + "HYCON:11TI740.MEAS", + "HYCON:11TT107.PNT", + "HYCON:12XX265.PV", + "HYCON:13LC048.OUT", + "HYCON:13XJ0019Y.PV", + "HYCON:12FC010.MEAS", + "HYCON:11TD419.MEAS", + "HYCON:11TD532.PNT", + "HYCON:11TD403.MEAS", + "HV8:008FC135.OP", + "HV8:008FC200.SP", + "HYCON:11TD244.PNT", + "HYCON:11PY013.PNT", + "HYCON:11FC028.MEAS", + "HYCON:10XI683_7.MEAS", + "HYCON:10XX160.PV", + "HYCON:10XX061.PV", + "HYCON:10XX042.PV", + "HYCON:10XJ0850Y.PV", + "HV8:008FC210.PV", + "HV8:008HC105.OP", + "HV8:008FY005.PV", + "HV8:008LC023.OP", + "HYCON:10PW084.CUSUM", + "HYCON:10LW020.UPTIME", + "HYCON:10PC203.MEAS", + "HYCON:10PI603A_2.MEAS", + "HYCON:10PI603A_4.MEAS", + "HYCON:10PW083.UPTIME", + "HYCON:10LT029.PNT", + "HYCON:10LI075.PNT", + "HYCON:10PT169.PNT", + "HYCON:10PY001.PNT", + "HYCON:10TI072.PNT", + "HYCON:10PI605B_3.MEAS", + "HYCON:10PW038.UPTIME", + "HYCON:10TI642.MEAS", + "HYCON:10LT035.PNT", + "HYCON:10PC106.MEAS", + "HYCON:10PI604A_4.MEAS", + "HYCON:10HC178.OUT", + "HYCON:10TT444.PNT", + "HYCON:10XI617_1.MEAS", + "HYCON:10TT261.PNT", + "HYCON:10XI643_3.MEAS", + "HYCON:10TT416.PNT", + "HYCON:10XI616_2.MEAS", + "HYCON:10XI626_3.MEAS", + "HV8:PAV_F801_Heartbeat", + "HV8:008XX032.PV", + "HV8:008XX440.PV", + "HV8:008XX107.PV", + "HV8:008XX809.PV", + "HV8:008XX263.PV", + "HV8:008TI075.PV", + "HV8:008PC1060.SP", + "HV8:008TI019.PV", + "HV8:008TI225.PV", + "HV8:008PC2058.OP", + "HV8:008TI295.PV", + "HV8:008TC220.PV", + "HV8:008TI162.PV", + "HYCON:15PGM052.CALC", + "HYCON:15FI026.MEAS", + "HYCON:15FT022.PNT", + "HYCON:15FT023.PNT", + "HYCON:15PC001.OUT", + "HYCON:15PY401.PNT", + "HYCON:12XX145.HS", + "HYCON:12TW647.UPTIME", + "HYCON:11TW287.UPTIME", + "HYCON:12TW681.UPTIME", + "HYCON:11XX435.MT", + "HYCON:13PW115.CUSUM", + "HYCON:12XX446.MT", + "HYCON:12TW267.CUSUM", + "HYCON:11TW373.CUSUM", + "HYCON:12TW315.CUSUM", + "HYCON:11TW677.CUSUM", + "HYCON:11XX451.MT", + "HYCON:12TW312.CUSUM", + "HYCON:12X046.PNT", + "HYCON:11XX201.PV", + "HYCON:11XX107.PV", + "HYCON:12XX402.PV", + "HYCON:12XX140.PV", + "HYCON:11XX280.PV", + "HYCON:12TY299.MEAS", + "HYCON:12XX742.PV", + "HYCON:13TT020.PNT", + "HYCON:12TY457.MEAS", + "HYCON:12TD060.MEAS", + "HYCON:12TT622_2.PNT", + "HYCON:12TD099.PNT", + "HYCON:12TY659.MEAS", + "HYCON:12XX060.PV", + "HYCON:12TT241.MEAS", + "HYCON:12TD080.PNT", + "HYCON:12XX306.PV.OUT", + "HYCON:11XX271.PV.OUT", + "HYCON:11XX753.PV", + "HYCON:12TY683.MEAS", + "HYCON:12XX017.PV", + "HYCON:11TI264.MEAS", + "HYCON:12XX204.PV", + "HYCON:12TY296.MEAS", + "HYCON:11TY457.MEAS", + "HYCON:11X209.PNT", + "HYCON:11XX060.PV", + "HYCON:12XX389.PV", + "HYCON:12TY466.MEAS", + "HYCON:11TT184.PNT", + "HYCON:11XX385.PV", + "HYCON:12X152A.PV", + "HYCON:12TD464.MEAS", + "HYCON:11XX388.PV", + "HYCON:11TI712.CHK", + "HYCON:11TT323.MEAS", + "HYCON:11TT049.PNT", + "HYCON:11TI722.MEAS", + "HYCON:11TY305.PNT", + "HYCON:11TT358.MEAS", + "HYCON:11TT197.MEAS", + "HYCON:11TT967.PNT", + "HYCON:11TI295.MEAS", + "HYCON:11TI657.PNT", + "HYCON:11TY290.MEAS", + "HYCON:11TY339.PNT", + "HYCON:12XX016.PV", + "HYCON:14FT010.PNT", + "HYCON:13PT026.PNT", + "HYCON:13LT029H.PNT", + "HYCON:13NROTA01.PNT", + "HYCON:13XJ0028Z.PV", + "HYCON:11TD706.MEAS", + "HYCON:11TD463.MEAS", + "HYCON:11TD073.MEAS", + "HYCON:11TC502.SPT", + "HYCON:10XI682_8.MEAS", + "HYCON:10XI686_4.MEAS", + "HYCON:10XJ0100XB.PV", + "HYCON:10XJ0828Z.PV", + "HV8:008FC206.PV", + "HV8:008HY107.OP", + "HV8:008FY155.PV", + "HYCON:10TW061.CUSUM", + "HYCON:10QT002_4.PNT", + "HYCON:10PI239.MEAS", + "HYCON:10PI088.PNT", + "HYCON:10PW073.UPTIME", + "HYCON:10PI073.PNT", + "HYCON:10PW148.UPTIME", + "HYCON:10TI006.PNT", + "HYCON:10QT002_1.PNT", + "HYCON:10TI602.MEAS", + "HYCON:10LI172.MEAS", + "HYCON:10HT042.PNT", + "HYCON:10PI646A_3.MEAS", + "HYCON:10PC224_MLM.out", + "HYCON:10TT463.PNT", + "HYCON:10XI622_8.MEAS", + "HYCON:10TT505.MEAS", + "HYCON:10TT424.PNT", + "HYCON:10XI626_8.MEAS", + "HYCON:10XI643_7.MEAS", + "HV8:008XX267.HS", + "HYCON:10FC054.MEAS", + "HV8:008XX112.PV", + "HV8:008XX237.PV", + "HV8:008X921T.PV", + "HV8:08LI0060.PV", + "HV8:008YP011.PV", + "HV8:08FC0146.PV", + "HV8:008TI029.PV", + "HV8:008TC116.OP", + "HV8:008TI091.PV", + "HV8:008TI084.PV", + "HV8:008XJ034Z.PV", + "HV8:008XX433.PV", + "HV8:008XX503.PV", + "HV8:008XX438.PV", + "HYCON:15TC321.SPT", + "HV8:008FC088.PV", + "HYCON:15TC018.SPT", + "HYCON:15PGM043.CALC", + "HYCON:15PGM029.CALC", + "HYCON:15LV026_1.OUT", + "HYCON:15LC003.MEAS", + "HYCON:12TW688.CUSUM", + "HYCON:11TW473.CUSUM", + "HYCON:11TW471.CUSUM", + "HYCON:12TD409.MEAS", + "HYCON:12TD198.PNT", + "HYCON:12TI077.MEAS", + "HYCON:12TI281.PNT", + "HYCON:12TT098.MEAS", + "HYCON:12TD425.MEAS", + "HYCON:12TD069.MEAS", + "HYCON:12TD079.PNT", + "HYCON:12TT372.MEAS", + "HYCON:12TI740.MEAS", + "HYCON:12TD465.MEAS", + "HYCON:12X007.PNT", + "HYCON:12TT912.PNT", + "HYCON:11TI265.MEAS", + "HYCON:12TT616.PNT", + "HYCON:13FC013.OUT", + "HYCON:11XX321.PV", + "HYCON:12TD427.MEAS", + "HYCON:11XX014.PV.OUT", + "HYCON:12TT633_3.PNT", + "HYCON:12TD301.MEAS", + "HYCON:12TY404.MEAS", + "HYCON:12PT018.PNT", + "HYCON:12XX035.PV.OUT", + "HYCON:12TY253.MEAS", + "HYCON:13PI092.MEAS", + "HYCON:12TT127.PNT", + "HYCON:12FC011.OUT", + "HYCON:12X060.PNT", + "HYCON:11XX381.PV", + "HYCON:12XX048.PV", + "HYCON:11X765C.PV", + "HYCON:11XX048.PV", + "HYCON:11XX224.PV", + "HYCON:12TI418.MEAS", + "HYCON:11XX229.PV", + "HYCON:11XX196.PV", + "HYCON:11XX197.PV", + "HYCON:11TY344.PNT", + "HYCON:11TT003.PNT", + "HYCON:11TT117.PNT", + "HYCON:11TY307.PNT", + "HYCON:11TY063.PNT", + "HYCON:11TT326.MEAS", + "HYCON:11TS1168.PNT", + "HYCON:11TY333.PNT", + "HYCON:11TI672.PNT", + "HYCON:11TI653.PNT", + "HYCON:11XX083.PNT", + "HYCON:12XX502.PNT", + "HYCON:14FC009.MEAS", + "HYCON:14FC013.OUT", + "HYCON:12XX285.PV", + "HYCON:13PC021.SPT", + "HYCON:13PC087.OUT", + "HYCON:12FT004.PNT", + "HYCON:11FC025.OUT", + "HYCON:10XX500.PV", + "HYCON:11TD292.MEAS", + "HYCON:11TD091.PNT", + "HYCON:11PT011.PNT", + "HYCON:11PT031.PNT", + "HYCON:11TD324.PNT", + "HYCON:10XI684_5.MEAS", + "HYCON:10XI682_3.MEAS", + "HYCON:10XJ0100YB.PV", + "HYCON:10XJ0858Z.PV", + "HV8:008HC202.OP", + "HV8:08FW0133.CUSUM", + "HYCON:10LT002.PNT", + "HYCON:10PI642A_2.MEAS", + "HYCON:10TI644.MEAS", + "HYCON:10TI604.MEAS", + "HYCON:10PT082.PNT", + "HYCON:10PC201.OUT", + "HYCON:10PI625A_4.MEAS", + "HYCON:10FY012.PNT", + "HYCON:10PT100.PNT", + "HYCON:10PT052.PNT", + "HYCON:10TT137.PNT", + "HYCON:10PT075.PNT", + "HYCON:10PC023.OUT", + "HYCON:10PW182.UPTIME", + "HYCON:10PW069_2.UPTIME", + "HYCON:10FY001.PNT", + "HYCON:10FY003.PNT", + "HYCON:10PI159.PNT", + "HYCON:10TI646.MEAS", + "HYCON:10PV010_2.OUT", + "HYCON:10LC109.SPT", + "HYCON:10HC021.MEAS", + "HYCON:10PC014.OUT", + "HYCON:10TT269.PNT", + "HYCON:10XI625_7.MEAS", + "HYCON:10TT426.PNT", + "HYCON:10XI663_6.MEAS", + "HYCON:10TT431.PNT", + "HV8:008TI246A.HS", + "HV8:08LI0045.PV", + "HV8:008XX850.PV", + "HYCON:10FI022.PNT", + "HV8:008XX026.PV", + "HV8:008XX051.PV", + "HV8:008XX176.PV", + "HV8:008PI018.PV", + "HV8:008TI066.PV", + "HV8:008Y0253.PV", + "HV8:008PI110.PV", + "HV8:008PC047.PV", + "HV8:008PI045.PV", + "HV8:008TI070.PV", + "HV8:008PI501.PV", + "HV8:008XX233D.PV", + "HV8:008XX171.PV", + "HV8:008XJ015Z.PV", + "HV8:008XJ010Z.PV", + "HV8:008XJ008Y.PV", + "HYCON:15TI418.MEAS", + "HV8:008FC090.OP", + "HYCON:15TC317.MEAS", + "HYCON:15PGM055.CALC", + "HYCON:15HC321.OUT", + "HYCON:15PT412.MEAS", + "HYCON:12TW676.UPTIME", + "HYCON:11TT622_3.CHK", + "HYCON:12XX457.MT", + "HYCON:12TW345.CUSUM", + "HYCON:11TW372.CUSUM", + "HYCON:11TW418.CUSUM", + "HYCON:11TW461.CUSUM", + "HYCON:11TW681.CUSUM", + "HYCON:11TW383.CUSUM", + "HYCON:12TT861.PNT", + "HYCON:13PT201.PNT", + "HYCON:11TY689.MEAS", + "HYCON:15FC005.SPT", + "HYCON:12TY074.MEAS", + "HYCON:12TD635.PNT", + "HYCON:12TD718.MEAS", + "HYCON:11XX010.PV.OUT", + "HYCON:11X026.PNT", + "HYCON:11TY429.MEAS", + "HYCON:12TI424.MEAS", + "HYCON:11TT129.PNT", + "HYCON:12XX093.PV", + "HYCON:11X049.PNT", + "HYCON:11TI263.MEAS", + "HYCON:11XX045.PV", + "HYCON:12TD623_3.PNT", + "HYCON:11XX262.PV", + "HYCON:12TD401.MEAS", + "HYCON:11TT925.PNT", + "HYCON:11TY458.MEAS", + "HYCON:12TT1200.PNT", + "HYCON:12X026.PNT", + "HYCON:12XX201.PV", + "HYCON:12TD342.PNT", + "HYCON:12TI682.PNT", + "HYCON:11XX227.PV", + "HYCON:12TY312.PNT", + "HYCON:12TD074.MEAS", + "HYCON:12TY334.PNT", + "HYCON:12TY418.MEAS", + "HYCON:11XX743.PV", + "HYCON:12X744A.PV", + "HYCON:11TY267.MEAS", + "HYCON:11TY062.PNT", + "HYCON:11TY374.PNT", + "HYCON:11TY295.MEAS", + "HYCON:11TS1167.PNT", + "HYCON:11TY335.PNT", + "HYCON:11TY382.PNT", + "HYCON:11TI432.MEAS", + "HYCON:11TI470.MEAS", + "HYCON:11TS1163.PNT", + "HYCON:11TT877.PNT", + "HYCON:11TY376.PNT", + "HYCON:12XX080.PNT", + "HYCON:12XX285.PV.OUT", + "HYCON:14PC040.SPT", + "HYCON:12XX274.PV", + "HYCON:15FC040.SPT", + "HYCON:13XJ0029Y.PV", + "HYCON:13XJ0018Z.PV", + "HYCON:15FC010.OUT", + "HYCON:13XJ0010Z.PV", + "HYCON:11TD474.MEAS", + "HV8:008FC201.SP", + "HYCON:11TC502.OUT", + "HYCON:11TD053.MEAS", + "HYCON:11FT026.PNT", + "HYCON:11TD293.MEAS", + "HYCON:11FW007.CUSUM", + "HYCON:11PT010.PNT", + "HYCON:10XX071.PV", + "HYCON:10XI676_6.MEAS", + "HYCON:10XI681_7.MEAS", + "HYCON:10XX130.PV", + "HYCON:10XX113.PV", + "HYCON:10XX034.PV", + "HYCON:10XJ0845Y.PV", + "HV8:008FI005.PV", + "HV8:008LC055.SP", + "HYCON:10LW026.CUSUM", + "HYCON:10TI613D.MEAS", + "HYCON:10PT109.MEAS", + "HYCON:10TI656C.MEAS", + "HYCON:10LT114.PNT", + "HYCON:10PT226.PNT", + "HYCON:10TI613A.MEAS", + "HYCON:10PI022.PNT", + "HYCON:10TI651A.MEAS", + "HYCON:10LC030.OUT", + "HYCON:10TT246.MEAS", + "HYCON:10PI621A_2.MEAS", + "HYCON:10PT132.MEAS", + "HYCON:10LT121.MEAS", + "HYCON:10PC106.OUT", + "HYCON:10PI626B_4.MEAS", + "HYCON:10PC106.SPT", + "HYCON:10TT344.MEAS", + "HYCON:10VC001.OUT", + "HYCON:10XI606_1.MEAS", + "HYCON:10XI605_4.MEAS", + "HYCON:10XI632_1.MEAS", + "HYCON:10FC047.OUT", + "HV8:008XX259.PV", + "HV8:008XX506.PV", + "HV8:008PC057.PV", + "HV8:008XX452.PV", + "HV8:008XX101.PV", + "HV8:008XJ064Y.PV", + "HV8:008PI108.PV", + "HV8:008PC060.OPLOLM", + "HV8:008PC016.PV", + "HV8:008TI052.PV", + "HV8:008TI296.PV", + "HV8:008TI173.PV", + "HV8:008TC114.PV", + "HV8:008XJ014Z.PV", + "HV8:008XX500.PV", + "HV8:008XX170.PV", + "HV8:008XJ011Z.PV", + "HV8:008XJ054Z.PV", + "HV8:008XJ032Y.PV", + "CD5:PACE_502FC096.INITMAN.VALVALID", + "CD5:PACE_502FC052.OP.POVSSVAL", + "CD5:PACE_502FC038.SP.MVMOVE", + "CD5:PACE_502FC047.SP.MVCALHL", + "CD5:514LT005.PV", + "CD5:514X910T.PV", + "CD5:515X895T.PV", + "CD5:513PV026A.OP", + "CD5:513PT066.PV", + "CD5:513QW010.CUSUM", + "CD5:513PW020.CUSUM", + "CD5:513XX110.PV", + "CD5:513XX010.PV", + "CD5:513PC003C.PV", + "CD5:513PC150.SP", + "CCR2:063TI051.PV", + "CC2:092XI010.PV", + "CC2:061TT0107.PV", + "CC2:061QI2105.PV", + "CC2:061FY106.PV", + "CC2:063TI031.PV", + "CC2:90TC0302.OP", + "CC2:91TI2302.PV", + "CC2:091X004.PV", + "CC2:063PI1931.PV", + "CC2:063TI020.PV", + "CC2:062PI0214.PV", + "CC2:092TI333.PV", + "CC2:092QY1022.PV", + "CC2:064PI056.PV", + "CC2:062FC001.OPHILM", + "CC2:061PC0201.SP", + "CC2:062FC304.OPHILM", + "CC2:062TC359.PV", + "CC2:062TC001.OPHILM", + "CC2:64LW0151.CUSUM", + "CC2:063PI0102.PV", + "CC2:062TC001.OPLOLM", + "CC2:062X060.PV", + "CC2:91TC0303.PV", + "CD5:$NM12B11.$UCNLSB(37)", + "CD5:$PRSTS04.CPUMIN", + "CD5:502XX914.PV", + "CD5:503XX910.PV", + "CD5:504PT012.PV", + "CD5:504XX761.PV", + "CD5:506XX101.PV", + "CD5:502XX078.PV", + "CD5:503TC028.PV", + "CD5:502XX930.PV", + "CD5:506LC004.OP", + "CD5:504PT212.PV", + "CD5:504PT079.PV", + "CD5:504TC016.OP", + "CD5:502TT408.PV", + "CD5:502TT534.PV", + "CD5:507LC006.OP", + "CD5:507FT014.PV", + "CD5:507FQ009.SP", + "CD5:507HC055.PV", + "C4TR:40TJ002.PV", + "CC2:61PW1031.CUSUM", + "CC2:64PW006 .CUSUM", + "CC2:64PW005 .CUSUM", + "C4TR:041FY011.PV", + "CC2:PACE_61PC0202.PV.POVPRV", + "CC2:061PI662.PV", + "CC2:PACE_072TY021.PV.CVHL", + "CC2:PACE_062TC355.SP.CVELL", + "CC2:PACE_062TC355.MODE.VALVALID", + "CC2:PACE_062FC051.SP.CVSSVAL", + "CC2:PACE_072PI003.PV.CVSSVAL", + "CC2:061FY011.PV", + "CC2:PACE_73QE0001.PV.POVPRERR", + "CC2:061PI1203.PV", + "CC2:061FY006.PV", + "CC2:PACE_073PC001.OP.POVSSVAL", + "CC2:PACE_072XC103.OP.VALVALID", + "CC2:060FY011.PV", + "CC2:PACE_062TC001.OP.CVVAL", + "CC2:060TI104.PV", + "CC2:PACE_072TC003.PV.VALVALID", + "CC2:PACE_062PI005.PV.CVEHL", + "CC2:PACE_062FC011.OP.CVLL", + "CC2:060TI103.PV", + "CC2:063PI0401.PV", + "CC2:62TJ110.PV", + "CC2:061XX040.PV", + "CC2:061XX141.PV", + "CC2:074XX913.PV", + "CC2:062XX036.PV", + "CC2:061XX060.PV", + "CC2:062X601A.PV", + "CC2:061XX038.PV", + "CC2:81XJ003ZD.PV", + "CC2:062XX305.PV", + "CC2:063XJ052Y.PV", + "CC2:62XJ003T.PV", + "C4TR:41XJ003Z.PV", + "CC2:083XJ053Y.PV", + "CC2:91XX111.PV", + "CC2:62LW013.UPTIME", + "CC2:060XJ053Z.PV", + "CC2:061XX139A.PV", + "CC2:61PW0018.UPTIME", + "CC2:62FW0301.UPTIME", + "CC2:61TW2202.UPTIME", + "CC2:PACE_EFC_C402_REB_DT.VALUE", + "CD5:502TT071.PV", + "CD5:502PI182.PV", + "CD5:502YY191.PV", + "CD5:507XC011.OP", + "CD5:506PT006.PV", + "CD5:502TT423.PV", + "CD5:507QT502.PV", + "CD5:502TC056.SP", + "CD5:502FT360.PV", + "CC2:PACE_EFC_020PC131.VALUE", + "CD5:502FC098.OP", + "CC2:PACE_QE_60QE0023_LAB_REF.POVLASTAMV", + "CD5:504PC004B.PV", + "CD5:502PT047.PV", + "CD5:502FC363.OP", + "CD5:502FZ068.PV", + "CC2:PACE_QE_60QE0023_LAB_REF.POVPRV", + "CD5:502PT219.PV", + "CD5:504FW003.CUSUM", + "CD5:502HC170.OP", + "CD5:506TC001.OP", + "CD5:502HC042.SP", + "CD5:505HC002.SP", + "CD5:502XJ150XA.PV", + "CD5:502XX651.MTD", + "CD5:502XX798.NEG", + "CD5:502XX772.PV", + "CD5:507XX006.PV", + "CD5:502XJ607Y.PV", + "CD5:502XJ153ZC.PV", + "CD5:502XJ152XC.PV", + "CD5:502XX172.PV", + "CD5:506XJ028Z.PV", + "CD5:505XJ028Y.PV", + "CD5:502XJ143ZD.PV", + "CD5:502XJ633Z.PV", + "CD5:504TW071.UPTIME", + "CD5:503TW029.UPTIME", + "CD5:504XJ020Z.PV", + "CD5:502FW071.UPTIME", + "CD5:506XJ031Y.PV", + "CD5:502XJ149XB.PV", + "CD5:504FW153.UPTIME", + "CD5:502X685B.PV", + "CD5:508FT022.PV", + "CD5:508TT065.PV", + "CD5:513FT096.PV", + "CD5:PACE_502FC096.SP.MVSSVAL", + "CD5:PACE_502FC038.SP.MVHL", + "CD5:PACE_502FC047.OP.VALVALID", + "CD5:PACE_502FC040.SP.POVSSVAL", + "CD5:PACE_502FC040.OP.POVPRERR", + "CD5:515QX012.PV", + "CD5:513YY026.PV", + "CD5:513YY009.PV", + "CD5:PACE_502FC029.OP.MVMOVE", + "CD5:PACE_502DT075.PV.CVEHL", + "CD5:513YY013.PV", + "CD5:513XX928.PV", + "CD5:513XJ267Y.PV", + "CD5:513XX100.HS", + "CD5:513X910B.PV", + "CD5:513XJ254Z.PV", + "CD5:513PC111.PV", + "CD5:513PC004.SP", + "CD5:513PC124.OP", + "CD5:513LC023.PV", + "CC2:92QI2322.PV", + "CC2:92TMR001.PV", + "CC2:92QA2320.PV", + "CC2:061X102.PV", + "CC2:062FC052.PV", + "CC2:064PC012.PV", + "CC2:064PY003.PV", + "CC2:061X017.PV", + "CC2:63XW3125.CUSUM", + "CC2:060FC012.OPHILM", + "CC2:083X001.PV", + "CC2:062FC024.PV", + "CC2:064GI1151.PV", + "CC2:064FI021.PV", + "CC2:062X023.PV", + "CC2:062TI011.PV", + "CC2:061UT002.PV", + "CC2:081FC003.PV", + "CC2:081PI001.PV", + "CC2:061X089.PV", + "CC2:063PC202.OPHILM", + "CC2:062TI391.PV", + "C4TR:41FY1002.PV", + "CD5:$PRSTS53.CPUFREE", + "CD5:502YY087.PV", + "CD5:504TT070.PV", + "CD5:502TT505.PV", + "CD5:502TT535.PV", + "CD5:502TT529.PV", + "CD5:503HC006.OP", + "CD5:502XC503.PV", + "CD5:502XX824.PV", + "CD5:504PI161.PV", + "CD5:502TT499.PV", + "CD5:506FT020.PV", + "CD5:505TT088.PV", + "CD5:502XI315.PV", + "CD5:503TC028.OP", + "CD5:502XX826.PV", + "CD5:503XX710.PV", + "CD5:502TT516.PV", + "CD5:504FC216.OP", + "CD5:503FY011.PV", + "CD5:504XX020.PV", + "CD5:504PT074.PV", + "CD5:504LC006.SP", + "CD5:504FC154.PV", + "CD5:506HC005.OP", + "CD5:504XX999.PV", + "CC2:PACE_073XC046.INITMAN.VALVALID", + "CC2:PACE_020TC138.OP.POVSSVAL", + "CC2:050XI085.PV", + "C4TR:040PC331.PV", + "CC2:PACE_062FC051.SP.POVPRV", + "CC2:PACE_062FC051.SP.POVPRERR", + "CC2:PACE_072FC006.SP.POVPRV", + "CC2:061FI133.PV", + "CC2:PACE_020PC131.SP.CVVAL", + "C4TR:041FY132.PV", + "CC2:PACE_061X035.PV.CVHL", + "CC2:PACE_072TC003.SP.MVCALHL", + "CC2:PACE_073Y0045.PV.POVSSVAL", + "CC2:PACE_61PC0201.PV.CVLL", + "CC2:061FC010.SP", + "CC2:PACE_062TC001.SP.CVVAL", + "CC2:PACE_020TC135.SP.MVCALLL", + "CC2:061PI028.PV", + "CC2:PACE_061QI031.PV.VALVALID", + "CC2:PACE_020TC138.PV.VALVALID", + "CC2:PACE_073PC001.SP.CVHL", + "CC2:061XX152.PV", + "CC2:062XX302.PV", + "CC2:061XX115.PV", + "CC2:060XX666.PV", + "CC2:062XJ160Z.PV", + "CC2:61PW0050.UPTIME", + "CC2:61TW0202.UPTIME", + "CC2:062TJ118.PV", + "CC2:81XJ002ZE.PV", + "CC2:62XJ004XA.PV", + "CC2:064XJ110Z.PV", + "C4TR:40XJ007Z.PV", + "CC2:62XJ005XE.PV", + "CC2:92TW1316.UPTIME", + "CC2:64PW0005.UPTIME", + "CC2:061XX193.PV", + "CC2:63XW1125.UPTIME", + "CC2:61TW0115.UPTIME", + "CD5:502TT368.PV", + "CC2:PACE_QE_073QI001.PV_RAWPRED.POVSSVAL", + "CD5:507PT007.PV", + "CD5:502X1003.PV", + "CD5:507XC011.PV", + "CC2:PACE_EFC_61TC0107.VALUE", + "CD5:507PT006.PV", + "CD5:502FC401.SP", + "CD5:502TT211.PV", + "CD5:501LT011.PV", + "CD5:501QT001W.PV", + "CD5:502FC364.PV", + "CC2:PACE_QE_60QE0023_LAB_REF.POVSSVAL", + "CD5:502PC411.PV", + "CC2:PACE_QE_20QE0003_LAB_REF.POVPRV", + "CD5:502TY714.PV", + "CD5:502TC410.OP", + "CD5:502FC024.SP", + "CD5:502FC117.OP", + "CD5:502XJ142ZB.PV", + "CD5:507PC004.OP", + "CD5:502FC119.OP", + "CD5:501FT008.PV", + "CD5:502XJ142YE.PV", + "CD5:502XX762.PV", + "CD5:502XX115A.PV", + "CD5:502XX817.MA", + "CD5:502XJ147ZA.PV", + "CD5:507XJ055Y.PV", + "CD5:507XJ071Z.PV", + "CD5:503XJ011Y.PV", + "CD5:502FW086.UPTIME", + "CD5:502XJ149T.PV", + "CD5:504XJ017Z.PV", + "CD5:502TW203.UPTIME", + "CD5:502LW014.UPTIME", + "CD5:507XY013.PV", + "CD5:508TT061.PV", + "CD5:513FC316.OP", + "CD5:513FC081.SP", + "CD5:508XC001.SP", + "CD5:PACE_502FC052.OP.CVLL", + "CD5:PACE_502FC052.OP.POVPRV", + "CD5:PACE_502DT075.PV.CVHL", + "CD5:PACE_502DT071.PV.CVEHL", + "CD5:513YY057.PV", + "CD5:513YY005.PV", + "CD5:LRMBP", + "CD5:513PT018.PV", + "CD5:513XX071.PV", + "CD5:513PC002.PV", + "CD5:513PC131.PV", + "CD5:513PC150.PV", + "CD5:513PC006B.OP", + "CD5:513PC022.PV", + "CD5:513PC094.OP", + "CD5:513LW204.UPTIME", + "CC2:63PW0401.CUSUM", + "CC2:063XI6141.PV", + "CC2:083FC001.PV", + "CC2:062X068.PV", + "CC2:91TW3303.CUSUM", + "CC2:061TI047.PV", + "CC2:090PI0034.PV", + "CC2:064TI1102.PV", + "CC2:062Y307.OP", + "CC2:081TI034.PV", + "CC2:064LI054.PV", + "CC2:063PI083.PV", + "CC2:091FY006.PV", + "CC2:064X001.PV", + "CC2:064PI065.PV", + "CC2:080LI017.PV", + "CC2:061TI0032.PV", + "CC2:062TI055.PV", + "CC2:062PI116.PV", + "CC2:092QI1011.PV", + "CC2:061UT001.PV", + "CC2:092PI0022.PV", + "CC2:081FY018.PV", + "CC2:061PC031.OPHILM", + "CC2:11YX0065.PV", + "CC2:062FC001.OP", + "CC2:080PC001.OP", + "CC2:63XW2125.CUSUM", + "CC2:081FC006.OP", + "CC2:064PI102.PV", + "CC2:062PC071.PV", + "CC2:061X084.PV", + "CC2:061X108.PV", + "CC2:062TC225.OPHILM", + "CD5:$PRSTS02.CPUFREE", + "CD5:$NM02B19.NPARAVG", + "CD5:$PRSTS06.CPUFREE", + "CD5:502YY047.PV", + "CD5:502YY118.PV", + "CD5:503LC004.PV", + "CD5:504PT070.PV", + "CD5:504TZ027.PV", + "CD5:507FC002.OP", + "CD5:502XX777.PV", + "CD5:506LC020.OP", + "CD5:503PZ022.PV", + "CD5:502XX798.PV", + "CD5:502XX039.PV", + "CD5:507FY003.PV", + "CD5:503XX789.PV", + "CD5:503XX813.PV", + "CD5:506HC012.SP", + "CD5:504XX021.PV", + "CD5:504PS072.PV", + "CC2:64FW002 .CUSUM", + "CC2:061TI0091.PV", + "CC2:PACE_20QE0002.PV.CVSSVAL", + "CC2:PACE_071FC502.SP.CVHL", + "CC2:PACE_062FC055.SP.CVELL", + "CC2:PACE_072TC003.SP.VALVALID", + "CC2:PACE_061X058.PV.CVHL", + "CC2:PACE_61TC0107.SP.MVCALHL", + "CC2:PACE_072Y0102.PV.POVPRERR", + "CC2:PACE_020TC138.PV.POVPRV", + "CC2:PACE_060FC012.SP.MVSSVAL", + "BDK_trigger_tag.Value", + "C4TR:041FC151.SP", + "CC2:PACE_062TI014.PV.CVEHL", + "CC2:PACE_062TC001.INITMAN.VALVALID", + "CC2:PACE_072FC006.SP.CVLL", + "CC2:PACE_061TI040.PV.CVHL", + "CC2:PACE_062PC003.OP.CVELL", + "CC2:PACE_062FC011.PV.VALVALID", + "CC2:PACE_061X003.PV.POVPRERR", + "CC2:PACE_61PC0201.PV.CVHL", + "CC2:PACE_62YY0016.PV.POVPRV", + "CC2:061LI2202.PV", + "CC2:PACE_061XS042.OP.POVPRERR", + "C4TR:041HY141.PV", + "CC2:PACE_73QE0013.PV.CVHL", + "CC2:PACE_020TC135.SP.MVOUT", + "CC2:PACE_062FC011.OP.POVPRV", + "CC2:PACE_062TI061.PV.POVPRERR", + "CC2:PACE_073XC045.OP.VALVALID", + "C4TR:41XJ016Z.PV", + "CC2:062X606A.PV", + "CC2:092XX011.PV", + "CC2:060XX609.PV", + "CC2:060X765B.PV", + "CC2:062XX308.PV", + "CC2:060XX612.PV", + "CC2:62XJ005ZB.PV", + "CC2:081XJ058Y.PV", + "CC2:060XX055.PV", + "C4TR:40TJ003.PV", + "CC2:062XJ151Y.PV", + "CC2:062XX812.CAL-EXC", + "CC2:062XX913.OUTSERV", + "CC2:062XJ157Y.PV", + "CCR2:99TJ001H.PV", + "CD5:502FW003.CUSUM", + "CD5:501PT005.PV", + "CD5:502FV029B.OP", + "CC2:PACE_QE_071TI046.PV.VALVALID", + "CD5:502FI609.PV", + "CD5:502T4422.PV", + "CD5:504LT021.PV", + "CD5:502FC102.SP", + "CD5:502TC374.PV", + "CC2:PACE_QE_20QE0015_LAB_REF.POVPRV", + "CD5:502XC002.OP", + "CD5:502FY033.PV", + "CD5:502FT043.PV", + "CC2:PACE_QE_C205_RFX_TOPS_.VALVALID", + "CD5:502TT372.PV", + "CC2:PACE_QE_60QE0024_LAB_REF.POVLASTAMT", + "CD5:502FZ089.PV", + "CD5:501LT030C.PV", + "CD5:502YY146.PV", + "CD5:502PC016A.PV", + "CD5:506TC001.PV", + "CC2:PACE_QE_20QE0005_LAB_REF.POVSSVAL", + "CC2:PACE_QE_073QI003.PV.POVSSVAL", + "CD5:502TC408.OP", + "CD5:502FC051.SP", + "CC2:PACE_EFC_062PC001.VALUE", + "CD5:502TI126.PV", + "CD5:502PC004B.PV", + "CC2:PACE_EFC_071PC001.VALUE", + "CD5:502FC059.SP", + "CD5:507TT110.PV", + "CD5:502PT216.PV", + "CD5:502PC413.SP", + "CD5:502FT142.PV", + "CD5:502DI164.PV", + "CD5:504YY014.PV", + "CD5:507XJ070Z.PV", + "CD5:504XJ016Y.PV", + "CD5:502TJ712.PV", + "CD5:502T1477.TV", + "CD5:502XX420.PV", + "CD5:502XX128C.PV", + "CD5:507X752A.PV", + "CD5:502XJ140XF.PV", + "CD5:502XJ632Z.PV", + "CD5:502XJ153YF.PV", + "CD5:503XJ010Y.PV", + "CD5:502XJ144YA.PV", + "CD5:502X680B.PV", + "CD5:502XJ164ZB.PV", + "CD5:502XJ170Z.PV", + "CD5:506LW008.UPTIME", + "CD5:502XJ164XA.PV", + "CD5:502XX158.PV", + "CD5:502FW067.UPTIME", + "CD5:502XX079.HS", + "CD5:513FT051.PV", + "CD5:508FC005.OP", + "CD5:PACE_502FC097.OP.MVLL", + "CD5:PACE_502FC037.OP.POVSSVAL", + "CD5:515TT003.PV", + "CD5:514LT001.PV", + "CD5:513XX922.PV", + "CD5:513PT022.PV", + "CD5:513TT183.PV", + "CD5:513XX084.PV", + "CD5:513XJ252Z.PV", + "CD5:513XJ250Y.PV", + "CD5:513XJ257Z.PV", + "CD5:513X920S.PV", + "CD5:513PC007B.SP", + "CD5:513LT171.PV", + "CD5:513HC012.PV", + "CC2:92TI0319.PV", + "CC2:061X087.PV", + "CC2:061X018.PV", + ], + } + headers = { + "Authorization": "Bearer " + token, + "Content-Type": "application/json", + "x-databricks-server-hostname": "adb-8969364155430721.1.azuredatabricks.net", + "x-databricks-http-path": "/sql/1.0/warehouses/9ecb6a8d6707260c", + } + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + response = await ac.post( + MOCK_API_NAME, + headers=headers, + params=params, + json=body, + ) + actual = response.text + print(actual) diff --git a/tests/sdk/python/rtdip_sdk/pipelines/transformers/spark/test_ssip_pi_binary_file_to_pcdm.py b/tests/sdk/python/rtdip_sdk/pipelines/transformers/spark/test_ssip_pi_binary_file_to_pcdm.py index 18d2cc249..a9a49e7c1 100644 --- a/tests/sdk/python/rtdip_sdk/pipelines/transformers/spark/test_ssip_pi_binary_file_to_pcdm.py +++ b/tests/sdk/python/rtdip_sdk/pipelines/transformers/spark/test_ssip_pi_binary_file_to_pcdm.py @@ -45,7 +45,7 @@ def test_ssip_binary_file_to_pcdm_setup(): assert ssip_pi_binary_file_to_pcdm.libraries() == Libraries( maven_libraries=[], pypi_libraries=[ - PyPiLibrary(name="pyarrow", version="12.0.0", repo=None), + PyPiLibrary(name="pyarrow", version="14.0.2", repo=None), PyPiLibrary(name="pandas", version="2.0.1", repo=None), ], pythonwheel_libraries=[], From ebceec470f97fbbcf577a1b1d8b47e3a43f06017 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 23 Jul 2024 12:35:56 +0100 Subject: [PATCH 2/8] Remove Unit Test Signed-off-by: GBBBAS --- tests/api/v1/test_api_raw.py | 1034 ---------------------------------- 1 file changed, 1034 deletions(-) diff --git a/tests/api/v1/test_api_raw.py b/tests/api/v1/test_api_raw.py index 805e4c858..bd4e64a5c 100644 --- a/tests/api/v1/test_api_raw.py +++ b/tests/api/v1/test_api_raw.py @@ -141,1037 +141,3 @@ async def test_api_raw_post_error(mocker: MockerFixture, api_test_data): assert response.status_code == 400 assert actual == '{"detail":"Error Connecting to Database"}' - - -async def test_api_raw_post_success_real(): - auth = DefaultAuth().authenticate() - token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token - params = { - "business_unit": "downstream", - "region": "emea", - "asset": "pernis", - "data_security_level": "restricted", - "data_type": "float", - "include_bad_data": True, - "start_date": "2023-01-01", - "end_date": "2023-01-31", - } - body = { - "tag_name": [ - "HYCON:15TT115.PNT", - "HYCON:15TC303.SPT", - "HYCON:15TC309.MEAS", - "HYCON:15LW028.CUSUM", - "HYCON:15LC300.SPT", - "HYCON:15FT012.PNT", - "HYCON:15LT407.MEAS", - "HYCON:15HC063.OUT", - "HYCON:11TW675.UPTIME", - "HYCON:12XX150.HS", - "HYCON:12XX758.HS", - "HYCON:11TX215.DOW", - "HYCON:12TI727.CHK", - "HYCON:11TI735.CHK", - "HYCON:11TW688.CUSUM", - "HYCON:11TW336.CUSUM", - "HYCON:11TW678.CUSUM", - "HYCON:11TW338.CUSUM", - "HYCON:11TW653.CUSUM", - "HYCON:11TW361.CUSUM", - "HYCON:11XX107.PV.OUT", - "HYCON:12TD656.MEAS", - "HYCON:12XX138.PV", - "HYCON:11XX089.PV", - "HYCON:14XX120.PV", - "HYCON:12TD246.PNT", - "HYCON:11TY637.PNT", - "HYCON:12TT316.MEAS", - "HYCON:12X097.PNT", - "HYCON:14TT026.PNT", - "HYCON:14XY100.PNT", - "HYCON:11X106.PNT", - "HYCON:11X142.PNT", - "HYCON:11XX252.PV", - "HYCON:13PC009.OUT", - "HYCON:11XX030.PV", - "HYCON:11XX766.PV", - "HYCON:12XX013.PV", - "HYCON:12TY405.MEAS", - "HYCON:12XX332.PV", - "HYCON:11TI266.MEAS", - "HYCON:12TD221.PNT", - "HYCON:12TY304.PNT", - "HYCON:12GT009.PNT", - "HYCON:12TT198.MEAS", - "HYCON:12TT621_2.PNT", - "HYCON:11TT915.PNT", - "HYCON:12TD719.MEAS", - "HYCON:12XX324.PV", - "HYCON:11XX211.PV", - "HYCON:12XX104.PV", - "HYCON:12TT358.MEAS", - "HYCON:12TD622_2.PNT", - "HYCON:12TI678.PNT", - "HYCON:12TI448.MEAS", - "HYCON:11TI282.PNT", - "HYCON:11XX049.PV", - "HYCON:12TT886.PNT", - "HYCON:11XX019.PV", - "HYCON:12X155A.PV", - "HYCON:11X765D.PV", - "HYCON:11TY323.PNT", - "HYCON:11TT1197.PNT", - "HYCON:11TT312.MEAS", - "HYCON:11TT622_1.PNT", - "HYCON:11TT219.MEAS", - "HYCON:11TT803.PNT", - "HYCON:11TY373.PNT", - "HYCON:11TI740.MEAS", - "HYCON:11TT107.PNT", - "HYCON:12XX265.PV", - "HYCON:13LC048.OUT", - "HYCON:13XJ0019Y.PV", - "HYCON:12FC010.MEAS", - "HYCON:11TD419.MEAS", - "HYCON:11TD532.PNT", - "HYCON:11TD403.MEAS", - "HV8:008FC135.OP", - "HV8:008FC200.SP", - "HYCON:11TD244.PNT", - "HYCON:11PY013.PNT", - "HYCON:11FC028.MEAS", - "HYCON:10XI683_7.MEAS", - "HYCON:10XX160.PV", - "HYCON:10XX061.PV", - "HYCON:10XX042.PV", - "HYCON:10XJ0850Y.PV", - "HV8:008FC210.PV", - "HV8:008HC105.OP", - "HV8:008FY005.PV", - "HV8:008LC023.OP", - "HYCON:10PW084.CUSUM", - "HYCON:10LW020.UPTIME", - "HYCON:10PC203.MEAS", - "HYCON:10PI603A_2.MEAS", - "HYCON:10PI603A_4.MEAS", - "HYCON:10PW083.UPTIME", - "HYCON:10LT029.PNT", - "HYCON:10LI075.PNT", - "HYCON:10PT169.PNT", - "HYCON:10PY001.PNT", - "HYCON:10TI072.PNT", - "HYCON:10PI605B_3.MEAS", - "HYCON:10PW038.UPTIME", - "HYCON:10TI642.MEAS", - "HYCON:10LT035.PNT", - "HYCON:10PC106.MEAS", - "HYCON:10PI604A_4.MEAS", - "HYCON:10HC178.OUT", - "HYCON:10TT444.PNT", - "HYCON:10XI617_1.MEAS", - "HYCON:10TT261.PNT", - "HYCON:10XI643_3.MEAS", - "HYCON:10TT416.PNT", - "HYCON:10XI616_2.MEAS", - "HYCON:10XI626_3.MEAS", - "HV8:PAV_F801_Heartbeat", - "HV8:008XX032.PV", - "HV8:008XX440.PV", - "HV8:008XX107.PV", - "HV8:008XX809.PV", - "HV8:008XX263.PV", - "HV8:008TI075.PV", - "HV8:008PC1060.SP", - "HV8:008TI019.PV", - "HV8:008TI225.PV", - "HV8:008PC2058.OP", - "HV8:008TI295.PV", - "HV8:008TC220.PV", - "HV8:008TI162.PV", - "HYCON:15PGM052.CALC", - "HYCON:15FI026.MEAS", - "HYCON:15FT022.PNT", - "HYCON:15FT023.PNT", - "HYCON:15PC001.OUT", - "HYCON:15PY401.PNT", - "HYCON:12XX145.HS", - "HYCON:12TW647.UPTIME", - "HYCON:11TW287.UPTIME", - "HYCON:12TW681.UPTIME", - "HYCON:11XX435.MT", - "HYCON:13PW115.CUSUM", - "HYCON:12XX446.MT", - "HYCON:12TW267.CUSUM", - "HYCON:11TW373.CUSUM", - "HYCON:12TW315.CUSUM", - "HYCON:11TW677.CUSUM", - "HYCON:11XX451.MT", - "HYCON:12TW312.CUSUM", - "HYCON:12X046.PNT", - "HYCON:11XX201.PV", - "HYCON:11XX107.PV", - "HYCON:12XX402.PV", - "HYCON:12XX140.PV", - "HYCON:11XX280.PV", - "HYCON:12TY299.MEAS", - "HYCON:12XX742.PV", - "HYCON:13TT020.PNT", - "HYCON:12TY457.MEAS", - "HYCON:12TD060.MEAS", - "HYCON:12TT622_2.PNT", - "HYCON:12TD099.PNT", - "HYCON:12TY659.MEAS", - "HYCON:12XX060.PV", - "HYCON:12TT241.MEAS", - "HYCON:12TD080.PNT", - "HYCON:12XX306.PV.OUT", - "HYCON:11XX271.PV.OUT", - "HYCON:11XX753.PV", - "HYCON:12TY683.MEAS", - "HYCON:12XX017.PV", - "HYCON:11TI264.MEAS", - "HYCON:12XX204.PV", - "HYCON:12TY296.MEAS", - "HYCON:11TY457.MEAS", - "HYCON:11X209.PNT", - "HYCON:11XX060.PV", - "HYCON:12XX389.PV", - "HYCON:12TY466.MEAS", - "HYCON:11TT184.PNT", - "HYCON:11XX385.PV", - "HYCON:12X152A.PV", - "HYCON:12TD464.MEAS", - "HYCON:11XX388.PV", - "HYCON:11TI712.CHK", - "HYCON:11TT323.MEAS", - "HYCON:11TT049.PNT", - "HYCON:11TI722.MEAS", - "HYCON:11TY305.PNT", - "HYCON:11TT358.MEAS", - "HYCON:11TT197.MEAS", - "HYCON:11TT967.PNT", - "HYCON:11TI295.MEAS", - "HYCON:11TI657.PNT", - "HYCON:11TY290.MEAS", - "HYCON:11TY339.PNT", - "HYCON:12XX016.PV", - "HYCON:14FT010.PNT", - "HYCON:13PT026.PNT", - "HYCON:13LT029H.PNT", - "HYCON:13NROTA01.PNT", - "HYCON:13XJ0028Z.PV", - "HYCON:11TD706.MEAS", - "HYCON:11TD463.MEAS", - "HYCON:11TD073.MEAS", - "HYCON:11TC502.SPT", - "HYCON:10XI682_8.MEAS", - "HYCON:10XI686_4.MEAS", - "HYCON:10XJ0100XB.PV", - "HYCON:10XJ0828Z.PV", - "HV8:008FC206.PV", - "HV8:008HY107.OP", - "HV8:008FY155.PV", - "HYCON:10TW061.CUSUM", - "HYCON:10QT002_4.PNT", - "HYCON:10PI239.MEAS", - "HYCON:10PI088.PNT", - "HYCON:10PW073.UPTIME", - "HYCON:10PI073.PNT", - "HYCON:10PW148.UPTIME", - "HYCON:10TI006.PNT", - "HYCON:10QT002_1.PNT", - "HYCON:10TI602.MEAS", - "HYCON:10LI172.MEAS", - "HYCON:10HT042.PNT", - "HYCON:10PI646A_3.MEAS", - "HYCON:10PC224_MLM.out", - "HYCON:10TT463.PNT", - "HYCON:10XI622_8.MEAS", - "HYCON:10TT505.MEAS", - "HYCON:10TT424.PNT", - "HYCON:10XI626_8.MEAS", - "HYCON:10XI643_7.MEAS", - "HV8:008XX267.HS", - "HYCON:10FC054.MEAS", - "HV8:008XX112.PV", - "HV8:008XX237.PV", - "HV8:008X921T.PV", - "HV8:08LI0060.PV", - "HV8:008YP011.PV", - "HV8:08FC0146.PV", - "HV8:008TI029.PV", - "HV8:008TC116.OP", - "HV8:008TI091.PV", - "HV8:008TI084.PV", - "HV8:008XJ034Z.PV", - "HV8:008XX433.PV", - "HV8:008XX503.PV", - "HV8:008XX438.PV", - "HYCON:15TC321.SPT", - "HV8:008FC088.PV", - "HYCON:15TC018.SPT", - "HYCON:15PGM043.CALC", - "HYCON:15PGM029.CALC", - "HYCON:15LV026_1.OUT", - "HYCON:15LC003.MEAS", - "HYCON:12TW688.CUSUM", - "HYCON:11TW473.CUSUM", - "HYCON:11TW471.CUSUM", - "HYCON:12TD409.MEAS", - "HYCON:12TD198.PNT", - "HYCON:12TI077.MEAS", - "HYCON:12TI281.PNT", - "HYCON:12TT098.MEAS", - "HYCON:12TD425.MEAS", - "HYCON:12TD069.MEAS", - "HYCON:12TD079.PNT", - "HYCON:12TT372.MEAS", - "HYCON:12TI740.MEAS", - "HYCON:12TD465.MEAS", - "HYCON:12X007.PNT", - "HYCON:12TT912.PNT", - "HYCON:11TI265.MEAS", - "HYCON:12TT616.PNT", - "HYCON:13FC013.OUT", - "HYCON:11XX321.PV", - "HYCON:12TD427.MEAS", - "HYCON:11XX014.PV.OUT", - "HYCON:12TT633_3.PNT", - "HYCON:12TD301.MEAS", - "HYCON:12TY404.MEAS", - "HYCON:12PT018.PNT", - "HYCON:12XX035.PV.OUT", - "HYCON:12TY253.MEAS", - "HYCON:13PI092.MEAS", - "HYCON:12TT127.PNT", - "HYCON:12FC011.OUT", - "HYCON:12X060.PNT", - "HYCON:11XX381.PV", - "HYCON:12XX048.PV", - "HYCON:11X765C.PV", - "HYCON:11XX048.PV", - "HYCON:11XX224.PV", - "HYCON:12TI418.MEAS", - "HYCON:11XX229.PV", - "HYCON:11XX196.PV", - "HYCON:11XX197.PV", - "HYCON:11TY344.PNT", - "HYCON:11TT003.PNT", - "HYCON:11TT117.PNT", - "HYCON:11TY307.PNT", - "HYCON:11TY063.PNT", - "HYCON:11TT326.MEAS", - "HYCON:11TS1168.PNT", - "HYCON:11TY333.PNT", - "HYCON:11TI672.PNT", - "HYCON:11TI653.PNT", - "HYCON:11XX083.PNT", - "HYCON:12XX502.PNT", - "HYCON:14FC009.MEAS", - "HYCON:14FC013.OUT", - "HYCON:12XX285.PV", - "HYCON:13PC021.SPT", - "HYCON:13PC087.OUT", - "HYCON:12FT004.PNT", - "HYCON:11FC025.OUT", - "HYCON:10XX500.PV", - "HYCON:11TD292.MEAS", - "HYCON:11TD091.PNT", - "HYCON:11PT011.PNT", - "HYCON:11PT031.PNT", - "HYCON:11TD324.PNT", - "HYCON:10XI684_5.MEAS", - "HYCON:10XI682_3.MEAS", - "HYCON:10XJ0100YB.PV", - "HYCON:10XJ0858Z.PV", - "HV8:008HC202.OP", - "HV8:08FW0133.CUSUM", - "HYCON:10LT002.PNT", - "HYCON:10PI642A_2.MEAS", - "HYCON:10TI644.MEAS", - "HYCON:10TI604.MEAS", - "HYCON:10PT082.PNT", - "HYCON:10PC201.OUT", - "HYCON:10PI625A_4.MEAS", - "HYCON:10FY012.PNT", - "HYCON:10PT100.PNT", - "HYCON:10PT052.PNT", - "HYCON:10TT137.PNT", - "HYCON:10PT075.PNT", - "HYCON:10PC023.OUT", - "HYCON:10PW182.UPTIME", - "HYCON:10PW069_2.UPTIME", - "HYCON:10FY001.PNT", - "HYCON:10FY003.PNT", - "HYCON:10PI159.PNT", - "HYCON:10TI646.MEAS", - "HYCON:10PV010_2.OUT", - "HYCON:10LC109.SPT", - "HYCON:10HC021.MEAS", - "HYCON:10PC014.OUT", - "HYCON:10TT269.PNT", - "HYCON:10XI625_7.MEAS", - "HYCON:10TT426.PNT", - "HYCON:10XI663_6.MEAS", - "HYCON:10TT431.PNT", - "HV8:008TI246A.HS", - "HV8:08LI0045.PV", - "HV8:008XX850.PV", - "HYCON:10FI022.PNT", - "HV8:008XX026.PV", - "HV8:008XX051.PV", - "HV8:008XX176.PV", - "HV8:008PI018.PV", - "HV8:008TI066.PV", - "HV8:008Y0253.PV", - "HV8:008PI110.PV", - "HV8:008PC047.PV", - "HV8:008PI045.PV", - "HV8:008TI070.PV", - "HV8:008PI501.PV", - "HV8:008XX233D.PV", - "HV8:008XX171.PV", - "HV8:008XJ015Z.PV", - "HV8:008XJ010Z.PV", - "HV8:008XJ008Y.PV", - "HYCON:15TI418.MEAS", - "HV8:008FC090.OP", - "HYCON:15TC317.MEAS", - "HYCON:15PGM055.CALC", - "HYCON:15HC321.OUT", - "HYCON:15PT412.MEAS", - "HYCON:12TW676.UPTIME", - "HYCON:11TT622_3.CHK", - "HYCON:12XX457.MT", - "HYCON:12TW345.CUSUM", - "HYCON:11TW372.CUSUM", - "HYCON:11TW418.CUSUM", - "HYCON:11TW461.CUSUM", - "HYCON:11TW681.CUSUM", - "HYCON:11TW383.CUSUM", - "HYCON:12TT861.PNT", - "HYCON:13PT201.PNT", - "HYCON:11TY689.MEAS", - "HYCON:15FC005.SPT", - "HYCON:12TY074.MEAS", - "HYCON:12TD635.PNT", - "HYCON:12TD718.MEAS", - "HYCON:11XX010.PV.OUT", - "HYCON:11X026.PNT", - "HYCON:11TY429.MEAS", - "HYCON:12TI424.MEAS", - "HYCON:11TT129.PNT", - "HYCON:12XX093.PV", - "HYCON:11X049.PNT", - "HYCON:11TI263.MEAS", - "HYCON:11XX045.PV", - "HYCON:12TD623_3.PNT", - "HYCON:11XX262.PV", - "HYCON:12TD401.MEAS", - "HYCON:11TT925.PNT", - "HYCON:11TY458.MEAS", - "HYCON:12TT1200.PNT", - "HYCON:12X026.PNT", - "HYCON:12XX201.PV", - "HYCON:12TD342.PNT", - "HYCON:12TI682.PNT", - "HYCON:11XX227.PV", - "HYCON:12TY312.PNT", - "HYCON:12TD074.MEAS", - "HYCON:12TY334.PNT", - "HYCON:12TY418.MEAS", - "HYCON:11XX743.PV", - "HYCON:12X744A.PV", - "HYCON:11TY267.MEAS", - "HYCON:11TY062.PNT", - "HYCON:11TY374.PNT", - "HYCON:11TY295.MEAS", - "HYCON:11TS1167.PNT", - "HYCON:11TY335.PNT", - "HYCON:11TY382.PNT", - "HYCON:11TI432.MEAS", - "HYCON:11TI470.MEAS", - "HYCON:11TS1163.PNT", - "HYCON:11TT877.PNT", - "HYCON:11TY376.PNT", - "HYCON:12XX080.PNT", - "HYCON:12XX285.PV.OUT", - "HYCON:14PC040.SPT", - "HYCON:12XX274.PV", - "HYCON:15FC040.SPT", - "HYCON:13XJ0029Y.PV", - "HYCON:13XJ0018Z.PV", - "HYCON:15FC010.OUT", - "HYCON:13XJ0010Z.PV", - "HYCON:11TD474.MEAS", - "HV8:008FC201.SP", - "HYCON:11TC502.OUT", - "HYCON:11TD053.MEAS", - "HYCON:11FT026.PNT", - "HYCON:11TD293.MEAS", - "HYCON:11FW007.CUSUM", - "HYCON:11PT010.PNT", - "HYCON:10XX071.PV", - "HYCON:10XI676_6.MEAS", - "HYCON:10XI681_7.MEAS", - "HYCON:10XX130.PV", - "HYCON:10XX113.PV", - "HYCON:10XX034.PV", - "HYCON:10XJ0845Y.PV", - "HV8:008FI005.PV", - "HV8:008LC055.SP", - "HYCON:10LW026.CUSUM", - "HYCON:10TI613D.MEAS", - "HYCON:10PT109.MEAS", - "HYCON:10TI656C.MEAS", - "HYCON:10LT114.PNT", - "HYCON:10PT226.PNT", - "HYCON:10TI613A.MEAS", - "HYCON:10PI022.PNT", - "HYCON:10TI651A.MEAS", - "HYCON:10LC030.OUT", - "HYCON:10TT246.MEAS", - "HYCON:10PI621A_2.MEAS", - "HYCON:10PT132.MEAS", - "HYCON:10LT121.MEAS", - "HYCON:10PC106.OUT", - "HYCON:10PI626B_4.MEAS", - "HYCON:10PC106.SPT", - "HYCON:10TT344.MEAS", - "HYCON:10VC001.OUT", - "HYCON:10XI606_1.MEAS", - "HYCON:10XI605_4.MEAS", - "HYCON:10XI632_1.MEAS", - "HYCON:10FC047.OUT", - "HV8:008XX259.PV", - "HV8:008XX506.PV", - "HV8:008PC057.PV", - "HV8:008XX452.PV", - "HV8:008XX101.PV", - "HV8:008XJ064Y.PV", - "HV8:008PI108.PV", - "HV8:008PC060.OPLOLM", - "HV8:008PC016.PV", - "HV8:008TI052.PV", - "HV8:008TI296.PV", - "HV8:008TI173.PV", - "HV8:008TC114.PV", - "HV8:008XJ014Z.PV", - "HV8:008XX500.PV", - "HV8:008XX170.PV", - "HV8:008XJ011Z.PV", - "HV8:008XJ054Z.PV", - "HV8:008XJ032Y.PV", - "CD5:PACE_502FC096.INITMAN.VALVALID", - "CD5:PACE_502FC052.OP.POVSSVAL", - "CD5:PACE_502FC038.SP.MVMOVE", - "CD5:PACE_502FC047.SP.MVCALHL", - "CD5:514LT005.PV", - "CD5:514X910T.PV", - "CD5:515X895T.PV", - "CD5:513PV026A.OP", - "CD5:513PT066.PV", - "CD5:513QW010.CUSUM", - "CD5:513PW020.CUSUM", - "CD5:513XX110.PV", - "CD5:513XX010.PV", - "CD5:513PC003C.PV", - "CD5:513PC150.SP", - "CCR2:063TI051.PV", - "CC2:092XI010.PV", - "CC2:061TT0107.PV", - "CC2:061QI2105.PV", - "CC2:061FY106.PV", - "CC2:063TI031.PV", - "CC2:90TC0302.OP", - "CC2:91TI2302.PV", - "CC2:091X004.PV", - "CC2:063PI1931.PV", - "CC2:063TI020.PV", - "CC2:062PI0214.PV", - "CC2:092TI333.PV", - "CC2:092QY1022.PV", - "CC2:064PI056.PV", - "CC2:062FC001.OPHILM", - "CC2:061PC0201.SP", - "CC2:062FC304.OPHILM", - "CC2:062TC359.PV", - "CC2:062TC001.OPHILM", - "CC2:64LW0151.CUSUM", - "CC2:063PI0102.PV", - "CC2:062TC001.OPLOLM", - "CC2:062X060.PV", - "CC2:91TC0303.PV", - "CD5:$NM12B11.$UCNLSB(37)", - "CD5:$PRSTS04.CPUMIN", - "CD5:502XX914.PV", - "CD5:503XX910.PV", - "CD5:504PT012.PV", - "CD5:504XX761.PV", - "CD5:506XX101.PV", - "CD5:502XX078.PV", - "CD5:503TC028.PV", - "CD5:502XX930.PV", - "CD5:506LC004.OP", - "CD5:504PT212.PV", - "CD5:504PT079.PV", - "CD5:504TC016.OP", - "CD5:502TT408.PV", - "CD5:502TT534.PV", - "CD5:507LC006.OP", - "CD5:507FT014.PV", - "CD5:507FQ009.SP", - "CD5:507HC055.PV", - "C4TR:40TJ002.PV", - "CC2:61PW1031.CUSUM", - "CC2:64PW006 .CUSUM", - "CC2:64PW005 .CUSUM", - "C4TR:041FY011.PV", - "CC2:PACE_61PC0202.PV.POVPRV", - "CC2:061PI662.PV", - "CC2:PACE_072TY021.PV.CVHL", - "CC2:PACE_062TC355.SP.CVELL", - "CC2:PACE_062TC355.MODE.VALVALID", - "CC2:PACE_062FC051.SP.CVSSVAL", - "CC2:PACE_072PI003.PV.CVSSVAL", - "CC2:061FY011.PV", - "CC2:PACE_73QE0001.PV.POVPRERR", - "CC2:061PI1203.PV", - "CC2:061FY006.PV", - "CC2:PACE_073PC001.OP.POVSSVAL", - "CC2:PACE_072XC103.OP.VALVALID", - "CC2:060FY011.PV", - "CC2:PACE_062TC001.OP.CVVAL", - "CC2:060TI104.PV", - "CC2:PACE_072TC003.PV.VALVALID", - "CC2:PACE_062PI005.PV.CVEHL", - "CC2:PACE_062FC011.OP.CVLL", - "CC2:060TI103.PV", - "CC2:063PI0401.PV", - "CC2:62TJ110.PV", - "CC2:061XX040.PV", - "CC2:061XX141.PV", - "CC2:074XX913.PV", - "CC2:062XX036.PV", - "CC2:061XX060.PV", - "CC2:062X601A.PV", - "CC2:061XX038.PV", - "CC2:81XJ003ZD.PV", - "CC2:062XX305.PV", - "CC2:063XJ052Y.PV", - "CC2:62XJ003T.PV", - "C4TR:41XJ003Z.PV", - "CC2:083XJ053Y.PV", - "CC2:91XX111.PV", - "CC2:62LW013.UPTIME", - "CC2:060XJ053Z.PV", - "CC2:061XX139A.PV", - "CC2:61PW0018.UPTIME", - "CC2:62FW0301.UPTIME", - "CC2:61TW2202.UPTIME", - "CC2:PACE_EFC_C402_REB_DT.VALUE", - "CD5:502TT071.PV", - "CD5:502PI182.PV", - "CD5:502YY191.PV", - "CD5:507XC011.OP", - "CD5:506PT006.PV", - "CD5:502TT423.PV", - "CD5:507QT502.PV", - "CD5:502TC056.SP", - "CD5:502FT360.PV", - "CC2:PACE_EFC_020PC131.VALUE", - "CD5:502FC098.OP", - "CC2:PACE_QE_60QE0023_LAB_REF.POVLASTAMV", - "CD5:504PC004B.PV", - "CD5:502PT047.PV", - "CD5:502FC363.OP", - "CD5:502FZ068.PV", - "CC2:PACE_QE_60QE0023_LAB_REF.POVPRV", - "CD5:502PT219.PV", - "CD5:504FW003.CUSUM", - "CD5:502HC170.OP", - "CD5:506TC001.OP", - "CD5:502HC042.SP", - "CD5:505HC002.SP", - "CD5:502XJ150XA.PV", - "CD5:502XX651.MTD", - "CD5:502XX798.NEG", - "CD5:502XX772.PV", - "CD5:507XX006.PV", - "CD5:502XJ607Y.PV", - "CD5:502XJ153ZC.PV", - "CD5:502XJ152XC.PV", - "CD5:502XX172.PV", - "CD5:506XJ028Z.PV", - "CD5:505XJ028Y.PV", - "CD5:502XJ143ZD.PV", - "CD5:502XJ633Z.PV", - "CD5:504TW071.UPTIME", - "CD5:503TW029.UPTIME", - "CD5:504XJ020Z.PV", - "CD5:502FW071.UPTIME", - "CD5:506XJ031Y.PV", - "CD5:502XJ149XB.PV", - "CD5:504FW153.UPTIME", - "CD5:502X685B.PV", - "CD5:508FT022.PV", - "CD5:508TT065.PV", - "CD5:513FT096.PV", - "CD5:PACE_502FC096.SP.MVSSVAL", - "CD5:PACE_502FC038.SP.MVHL", - "CD5:PACE_502FC047.OP.VALVALID", - "CD5:PACE_502FC040.SP.POVSSVAL", - "CD5:PACE_502FC040.OP.POVPRERR", - "CD5:515QX012.PV", - "CD5:513YY026.PV", - "CD5:513YY009.PV", - "CD5:PACE_502FC029.OP.MVMOVE", - "CD5:PACE_502DT075.PV.CVEHL", - "CD5:513YY013.PV", - "CD5:513XX928.PV", - "CD5:513XJ267Y.PV", - "CD5:513XX100.HS", - "CD5:513X910B.PV", - "CD5:513XJ254Z.PV", - "CD5:513PC111.PV", - "CD5:513PC004.SP", - "CD5:513PC124.OP", - "CD5:513LC023.PV", - "CC2:92QI2322.PV", - "CC2:92TMR001.PV", - "CC2:92QA2320.PV", - "CC2:061X102.PV", - "CC2:062FC052.PV", - "CC2:064PC012.PV", - "CC2:064PY003.PV", - "CC2:061X017.PV", - "CC2:63XW3125.CUSUM", - "CC2:060FC012.OPHILM", - "CC2:083X001.PV", - "CC2:062FC024.PV", - "CC2:064GI1151.PV", - "CC2:064FI021.PV", - "CC2:062X023.PV", - "CC2:062TI011.PV", - "CC2:061UT002.PV", - "CC2:081FC003.PV", - "CC2:081PI001.PV", - "CC2:061X089.PV", - "CC2:063PC202.OPHILM", - "CC2:062TI391.PV", - "C4TR:41FY1002.PV", - "CD5:$PRSTS53.CPUFREE", - "CD5:502YY087.PV", - "CD5:504TT070.PV", - "CD5:502TT505.PV", - "CD5:502TT535.PV", - "CD5:502TT529.PV", - "CD5:503HC006.OP", - "CD5:502XC503.PV", - "CD5:502XX824.PV", - "CD5:504PI161.PV", - "CD5:502TT499.PV", - "CD5:506FT020.PV", - "CD5:505TT088.PV", - "CD5:502XI315.PV", - "CD5:503TC028.OP", - "CD5:502XX826.PV", - "CD5:503XX710.PV", - "CD5:502TT516.PV", - "CD5:504FC216.OP", - "CD5:503FY011.PV", - "CD5:504XX020.PV", - "CD5:504PT074.PV", - "CD5:504LC006.SP", - "CD5:504FC154.PV", - "CD5:506HC005.OP", - "CD5:504XX999.PV", - "CC2:PACE_073XC046.INITMAN.VALVALID", - "CC2:PACE_020TC138.OP.POVSSVAL", - "CC2:050XI085.PV", - "C4TR:040PC331.PV", - "CC2:PACE_062FC051.SP.POVPRV", - "CC2:PACE_062FC051.SP.POVPRERR", - "CC2:PACE_072FC006.SP.POVPRV", - "CC2:061FI133.PV", - "CC2:PACE_020PC131.SP.CVVAL", - "C4TR:041FY132.PV", - "CC2:PACE_061X035.PV.CVHL", - "CC2:PACE_072TC003.SP.MVCALHL", - "CC2:PACE_073Y0045.PV.POVSSVAL", - "CC2:PACE_61PC0201.PV.CVLL", - "CC2:061FC010.SP", - "CC2:PACE_062TC001.SP.CVVAL", - "CC2:PACE_020TC135.SP.MVCALLL", - "CC2:061PI028.PV", - "CC2:PACE_061QI031.PV.VALVALID", - "CC2:PACE_020TC138.PV.VALVALID", - "CC2:PACE_073PC001.SP.CVHL", - "CC2:061XX152.PV", - "CC2:062XX302.PV", - "CC2:061XX115.PV", - "CC2:060XX666.PV", - "CC2:062XJ160Z.PV", - "CC2:61PW0050.UPTIME", - "CC2:61TW0202.UPTIME", - "CC2:062TJ118.PV", - "CC2:81XJ002ZE.PV", - "CC2:62XJ004XA.PV", - "CC2:064XJ110Z.PV", - "C4TR:40XJ007Z.PV", - "CC2:62XJ005XE.PV", - "CC2:92TW1316.UPTIME", - "CC2:64PW0005.UPTIME", - "CC2:061XX193.PV", - "CC2:63XW1125.UPTIME", - "CC2:61TW0115.UPTIME", - "CD5:502TT368.PV", - "CC2:PACE_QE_073QI001.PV_RAWPRED.POVSSVAL", - "CD5:507PT007.PV", - "CD5:502X1003.PV", - "CD5:507XC011.PV", - "CC2:PACE_EFC_61TC0107.VALUE", - "CD5:507PT006.PV", - "CD5:502FC401.SP", - "CD5:502TT211.PV", - "CD5:501LT011.PV", - "CD5:501QT001W.PV", - "CD5:502FC364.PV", - "CC2:PACE_QE_60QE0023_LAB_REF.POVSSVAL", - "CD5:502PC411.PV", - "CC2:PACE_QE_20QE0003_LAB_REF.POVPRV", - "CD5:502TY714.PV", - "CD5:502TC410.OP", - "CD5:502FC024.SP", - "CD5:502FC117.OP", - "CD5:502XJ142ZB.PV", - "CD5:507PC004.OP", - "CD5:502FC119.OP", - "CD5:501FT008.PV", - "CD5:502XJ142YE.PV", - "CD5:502XX762.PV", - "CD5:502XX115A.PV", - "CD5:502XX817.MA", - "CD5:502XJ147ZA.PV", - "CD5:507XJ055Y.PV", - "CD5:507XJ071Z.PV", - "CD5:503XJ011Y.PV", - "CD5:502FW086.UPTIME", - "CD5:502XJ149T.PV", - "CD5:504XJ017Z.PV", - "CD5:502TW203.UPTIME", - "CD5:502LW014.UPTIME", - "CD5:507XY013.PV", - "CD5:508TT061.PV", - "CD5:513FC316.OP", - "CD5:513FC081.SP", - "CD5:508XC001.SP", - "CD5:PACE_502FC052.OP.CVLL", - "CD5:PACE_502FC052.OP.POVPRV", - "CD5:PACE_502DT075.PV.CVHL", - "CD5:PACE_502DT071.PV.CVEHL", - "CD5:513YY057.PV", - "CD5:513YY005.PV", - "CD5:LRMBP", - "CD5:513PT018.PV", - "CD5:513XX071.PV", - "CD5:513PC002.PV", - "CD5:513PC131.PV", - "CD5:513PC150.PV", - "CD5:513PC006B.OP", - "CD5:513PC022.PV", - "CD5:513PC094.OP", - "CD5:513LW204.UPTIME", - "CC2:63PW0401.CUSUM", - "CC2:063XI6141.PV", - "CC2:083FC001.PV", - "CC2:062X068.PV", - "CC2:91TW3303.CUSUM", - "CC2:061TI047.PV", - "CC2:090PI0034.PV", - "CC2:064TI1102.PV", - "CC2:062Y307.OP", - "CC2:081TI034.PV", - "CC2:064LI054.PV", - "CC2:063PI083.PV", - "CC2:091FY006.PV", - "CC2:064X001.PV", - "CC2:064PI065.PV", - "CC2:080LI017.PV", - "CC2:061TI0032.PV", - "CC2:062TI055.PV", - "CC2:062PI116.PV", - "CC2:092QI1011.PV", - "CC2:061UT001.PV", - "CC2:092PI0022.PV", - "CC2:081FY018.PV", - "CC2:061PC031.OPHILM", - "CC2:11YX0065.PV", - "CC2:062FC001.OP", - "CC2:080PC001.OP", - "CC2:63XW2125.CUSUM", - "CC2:081FC006.OP", - "CC2:064PI102.PV", - "CC2:062PC071.PV", - "CC2:061X084.PV", - "CC2:061X108.PV", - "CC2:062TC225.OPHILM", - "CD5:$PRSTS02.CPUFREE", - "CD5:$NM02B19.NPARAVG", - "CD5:$PRSTS06.CPUFREE", - "CD5:502YY047.PV", - "CD5:502YY118.PV", - "CD5:503LC004.PV", - "CD5:504PT070.PV", - "CD5:504TZ027.PV", - "CD5:507FC002.OP", - "CD5:502XX777.PV", - "CD5:506LC020.OP", - "CD5:503PZ022.PV", - "CD5:502XX798.PV", - "CD5:502XX039.PV", - "CD5:507FY003.PV", - "CD5:503XX789.PV", - "CD5:503XX813.PV", - "CD5:506HC012.SP", - "CD5:504XX021.PV", - "CD5:504PS072.PV", - "CC2:64FW002 .CUSUM", - "CC2:061TI0091.PV", - "CC2:PACE_20QE0002.PV.CVSSVAL", - "CC2:PACE_071FC502.SP.CVHL", - "CC2:PACE_062FC055.SP.CVELL", - "CC2:PACE_072TC003.SP.VALVALID", - "CC2:PACE_061X058.PV.CVHL", - "CC2:PACE_61TC0107.SP.MVCALHL", - "CC2:PACE_072Y0102.PV.POVPRERR", - "CC2:PACE_020TC138.PV.POVPRV", - "CC2:PACE_060FC012.SP.MVSSVAL", - "BDK_trigger_tag.Value", - "C4TR:041FC151.SP", - "CC2:PACE_062TI014.PV.CVEHL", - "CC2:PACE_062TC001.INITMAN.VALVALID", - "CC2:PACE_072FC006.SP.CVLL", - "CC2:PACE_061TI040.PV.CVHL", - "CC2:PACE_062PC003.OP.CVELL", - "CC2:PACE_062FC011.PV.VALVALID", - "CC2:PACE_061X003.PV.POVPRERR", - "CC2:PACE_61PC0201.PV.CVHL", - "CC2:PACE_62YY0016.PV.POVPRV", - "CC2:061LI2202.PV", - "CC2:PACE_061XS042.OP.POVPRERR", - "C4TR:041HY141.PV", - "CC2:PACE_73QE0013.PV.CVHL", - "CC2:PACE_020TC135.SP.MVOUT", - "CC2:PACE_062FC011.OP.POVPRV", - "CC2:PACE_062TI061.PV.POVPRERR", - "CC2:PACE_073XC045.OP.VALVALID", - "C4TR:41XJ016Z.PV", - "CC2:062X606A.PV", - "CC2:092XX011.PV", - "CC2:060XX609.PV", - "CC2:060X765B.PV", - "CC2:062XX308.PV", - "CC2:060XX612.PV", - "CC2:62XJ005ZB.PV", - "CC2:081XJ058Y.PV", - "CC2:060XX055.PV", - "C4TR:40TJ003.PV", - "CC2:062XJ151Y.PV", - "CC2:062XX812.CAL-EXC", - "CC2:062XX913.OUTSERV", - "CC2:062XJ157Y.PV", - "CCR2:99TJ001H.PV", - "CD5:502FW003.CUSUM", - "CD5:501PT005.PV", - "CD5:502FV029B.OP", - "CC2:PACE_QE_071TI046.PV.VALVALID", - "CD5:502FI609.PV", - "CD5:502T4422.PV", - "CD5:504LT021.PV", - "CD5:502FC102.SP", - "CD5:502TC374.PV", - "CC2:PACE_QE_20QE0015_LAB_REF.POVPRV", - "CD5:502XC002.OP", - "CD5:502FY033.PV", - "CD5:502FT043.PV", - "CC2:PACE_QE_C205_RFX_TOPS_.VALVALID", - "CD5:502TT372.PV", - "CC2:PACE_QE_60QE0024_LAB_REF.POVLASTAMT", - "CD5:502FZ089.PV", - "CD5:501LT030C.PV", - "CD5:502YY146.PV", - "CD5:502PC016A.PV", - "CD5:506TC001.PV", - "CC2:PACE_QE_20QE0005_LAB_REF.POVSSVAL", - "CC2:PACE_QE_073QI003.PV.POVSSVAL", - "CD5:502TC408.OP", - "CD5:502FC051.SP", - "CC2:PACE_EFC_062PC001.VALUE", - "CD5:502TI126.PV", - "CD5:502PC004B.PV", - "CC2:PACE_EFC_071PC001.VALUE", - "CD5:502FC059.SP", - "CD5:507TT110.PV", - "CD5:502PT216.PV", - "CD5:502PC413.SP", - "CD5:502FT142.PV", - "CD5:502DI164.PV", - "CD5:504YY014.PV", - "CD5:507XJ070Z.PV", - "CD5:504XJ016Y.PV", - "CD5:502TJ712.PV", - "CD5:502T1477.TV", - "CD5:502XX420.PV", - "CD5:502XX128C.PV", - "CD5:507X752A.PV", - "CD5:502XJ140XF.PV", - "CD5:502XJ632Z.PV", - "CD5:502XJ153YF.PV", - "CD5:503XJ010Y.PV", - "CD5:502XJ144YA.PV", - "CD5:502X680B.PV", - "CD5:502XJ164ZB.PV", - "CD5:502XJ170Z.PV", - "CD5:506LW008.UPTIME", - "CD5:502XJ164XA.PV", - "CD5:502XX158.PV", - "CD5:502FW067.UPTIME", - "CD5:502XX079.HS", - "CD5:513FT051.PV", - "CD5:508FC005.OP", - "CD5:PACE_502FC097.OP.MVLL", - "CD5:PACE_502FC037.OP.POVSSVAL", - "CD5:515TT003.PV", - "CD5:514LT001.PV", - "CD5:513XX922.PV", - "CD5:513PT022.PV", - "CD5:513TT183.PV", - "CD5:513XX084.PV", - "CD5:513XJ252Z.PV", - "CD5:513XJ250Y.PV", - "CD5:513XJ257Z.PV", - "CD5:513X920S.PV", - "CD5:513PC007B.SP", - "CD5:513LT171.PV", - "CD5:513HC012.PV", - "CC2:92TI0319.PV", - "CC2:061X087.PV", - "CC2:061X018.PV", - ], - } - headers = { - "Authorization": "Bearer " + token, - "Content-Type": "application/json", - "x-databricks-server-hostname": "adb-8969364155430721.1.azuredatabricks.net", - "x-databricks-http-path": "/sql/1.0/warehouses/9ecb6a8d6707260c", - } - async with AsyncClient(app=app, base_url=BASE_URL) as ac: - response = await ac.post( - MOCK_API_NAME, - headers=headers, - params=params, - json=body, - ) - actual = response.text - print(actual) From cee86e71bbd4af8fd24febb940405174f31c245b Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 23 Jul 2024 12:50:34 +0100 Subject: [PATCH 3/8] Test Updates Signed-off-by: GBBBAS --- src/api/v1/common.py | 4 ++-- src/api/v1/time_weighted_average.py | 1 - tests/conftest.py | 24 ++++++------------------ 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/api/v1/common.py b/src/api/v1/common.py index f59bb096c..3686f1eeb 100644 --- a/src/api/v1/common.py +++ b/src/api/v1/common.py @@ -142,7 +142,7 @@ def common_api_setup_tasks( # NOSONAR return connection, parameters -def pagination(limit_offset_parameters: LimitOffsetQueryParams, data: DataFrame): +def pagination(limit_offset_parameters: LimitOffsetQueryParams, data: str): pagination = PaginationRow( limit=None, offset=None, @@ -156,7 +156,7 @@ def pagination(limit_offset_parameters: LimitOffsetQueryParams, data: DataFrame) next_offset = None if ( - len(data.index) == limit_offset_parameters.limit + data.count("}") == limit_offset_parameters.limit and limit_offset_parameters.offset is not None ): next_offset = limit_offset_parameters.offset + limit_offset_parameters.limit diff --git a/src/api/v1/time_weighted_average.py b/src/api/v1/time_weighted_average.py index a9fbdb611..ac1893d11 100644 --- a/src/api/v1/time_weighted_average.py +++ b/src/api/v1/time_weighted_average.py @@ -54,7 +54,6 @@ def time_weighted_average_events_get( ) data = time_weighted_average.get(connection, parameters) - data = data.reset_index() return json_response(data, limit_offset_parameters) except Exception as e: diff --git a/tests/conftest.py b/tests/conftest.py index d09a0ea5e..67471b8fc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,9 +99,7 @@ def api_test_data(): } mock_raw_data = test_raw_data.copy() mock_raw_data["EventTime"] = mock_raw_data["EventTime"].strftime(datetime_format) - mock_raw_df = pd.DataFrame( - {"Value": [json.dumps(mock_raw_data, separators=(",", ":"))]} - ) + mock_raw_df = json.dumps(mock_raw_data, separators=(",", ":")) expected_raw = expected_result(test_raw_data) # Mock Aggregated Data @@ -112,9 +110,7 @@ def api_test_data(): } mock_agg_data = test_agg_data.copy() mock_agg_data["EventTime"] = mock_agg_data["EventTime"].strftime(datetime_format) - mock_agg_df = pd.DataFrame( - {"Value": [json.dumps(mock_agg_data, separators=(",", ":"))]} - ) + mock_agg_df = json.dumps(mock_agg_data, separators=(",", ":")) expected_agg = expected_result(test_agg_data) # Summary Data @@ -131,9 +127,7 @@ def api_test_data(): mock_plot_data = test_plot_data.copy() mock_plot_data["EventTime"] = mock_plot_data["EventTime"].strftime(datetime_format) - mock_plot_df = pd.DataFrame( - {"Value": [json.dumps(mock_plot_data, separators=(",", ":"))]} - ) + mock_plot_df = json.dumps(mock_plot_data, separators=(",", ":")) expected_plot = expected_result(test_plot_data) test_summary_data = { @@ -147,9 +141,7 @@ def api_test_data(): "Var": 0.0, } - mock_summary_df = pd.DataFrame( - {"Value": [json.dumps(test_summary_data, separators=(",", ":"))]} - ) + mock_summary_df = json.dumps(test_summary_data, separators=(",", ":")) expected_summary = expected_result(test_summary_data) test_metadata = { @@ -158,9 +150,7 @@ def api_test_data(): "Description": "Test Description", } - mock_metadata_df = pd.DataFrame( - {"Value": [json.dumps(test_metadata, separators=(",", ":"))]} - ) + mock_metadata_df = json.dumps(test_metadata, separators=(",", ":")) expected_metadata = expected_result(test_metadata) test_latest_data = { @@ -181,9 +171,7 @@ def api_test_data(): mock_latest_data["GoodEventTime"] = mock_latest_data["GoodEventTime"].strftime( datetime_format ) - mock_latest_df = pd.DataFrame( - {"Value": [json.dumps(mock_latest_data, separators=(",", ":"))]} - ) + mock_latest_df = json.dumps(mock_latest_data, separators=(",", ":")) expected_latest = expected_result(test_latest_data) expected_sql = expected_result(test_raw_data, "100", "100") From c3a89fe3c40dbf10101ff2794164a43455b3de55 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 23 Jul 2024 13:04:36 +0100 Subject: [PATCH 4/8] Updates for String data return type Signed-off-by: GBBBAS --- src/api/v1/common.py | 17 ++++++------- .../connectors/odbc/db_sql_connector.py | 14 +++++++---- tests/conftest.py | 24 ++++++++++++++----- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/api/v1/common.py b/src/api/v1/common.py index 3686f1eeb..714c66e73 100644 --- a/src/api/v1/common.py +++ b/src/api/v1/common.py @@ -142,7 +142,7 @@ def common_api_setup_tasks( # NOSONAR return connection, parameters -def pagination(limit_offset_parameters: LimitOffsetQueryParams, data: str): +def pagination(limit_offset_parameters: LimitOffsetQueryParams, rows: int): pagination = PaginationRow( limit=None, offset=None, @@ -156,7 +156,7 @@ def pagination(limit_offset_parameters: LimitOffsetQueryParams, data: str): next_offset = None if ( - data.count("}") == limit_offset_parameters.limit + rows == limit_offset_parameters.limit and limit_offset_parameters.offset is not None ): next_offset = limit_offset_parameters.offset + limit_offset_parameters.limit @@ -184,11 +184,11 @@ def datetime_parser(json_dict): def json_response( - data: str, limit_offset_parameters: LimitOffsetQueryParams + data: dict, limit_offset_parameters: LimitOffsetQueryParams ) -> Response: schema_df = pd.DataFrame() - if data is not None and data != "": - json_str = data[0 : data.find("}") + 1] + if data["data"] is not None and data["data"] != "": + json_str = data["data"][0 : data["data"].find("}") + 1] json_dict = json.loads(json_str, object_hook=datetime_parser) schema_df = pd.json_normalize(json_dict) @@ -198,11 +198,8 @@ def json_response( FieldSchema.model_validate( build_table_schema(schema_df, index=False, primary_key=False), ).model_dump_json(), - "[" + data + "]", - # data.replace({np.nan: None}).to_json( - # orient="records", date_format="iso", date_unit="ns" - # ), - pagination(limit_offset_parameters, data).model_dump_json(), + "[" + data["data"] + "]", + pagination(limit_offset_parameters, data["count"]).model_dump_json(), ) + "}", media_type="application/json", diff --git a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py index b26b8792c..166a90eee 100644 --- a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py +++ b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py @@ -110,7 +110,7 @@ def execute(self, query: str) -> None: logging.exception("error while executing the query") raise e - def fetch_all(self, fetch_size=5_000_000) -> list: + def fetch_all(self, fetch_size=5_000_000): """ Gets all rows of a query. @@ -120,8 +120,10 @@ def fetch_all(self, fetch_size=5_000_000) -> list: try: get_next_result = True results = None if ConnectionReturnType.String else [] + count = 0 while get_next_result: result = self.cursor.fetchmany_arrow(fetch_size) + count += result.num_rows if self.return_type == ConnectionReturnType.List: column_list = [] for column in result.columns: @@ -148,11 +150,13 @@ def fetch_all(self, fetch_size=5_000_000) -> list: elif self.return_type == ConnectionReturnType.Pyarrow: pyarrow_table = pa.concat_tables(results) return pyarrow_table - elif self.return_type in ( - ConnectionReturnType.List, - ConnectionReturnType.String, - ): + elif self.return_type == ConnectionReturnType.List: return results + elif self.return_type == ConnectionReturnType.String: + return { + "data": results, + "count": count, + } except Exception as e: logging.exception("error while fetching the rows of a query") raise e diff --git a/tests/conftest.py b/tests/conftest.py index 67471b8fc..7b0f7c624 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,7 +99,7 @@ def api_test_data(): } mock_raw_data = test_raw_data.copy() mock_raw_data["EventTime"] = mock_raw_data["EventTime"].strftime(datetime_format) - mock_raw_df = json.dumps(mock_raw_data, separators=(",", ":")) + mock_raw_df = {"data": json.dumps(mock_raw_data, separators=(",", ":")), "count": 1} expected_raw = expected_result(test_raw_data) # Mock Aggregated Data @@ -110,7 +110,7 @@ def api_test_data(): } mock_agg_data = test_agg_data.copy() mock_agg_data["EventTime"] = mock_agg_data["EventTime"].strftime(datetime_format) - mock_agg_df = json.dumps(mock_agg_data, separators=(",", ":")) + mock_agg_df = {"data": json.dumps(mock_agg_data, separators=(",", ":")), "count": 1} expected_agg = expected_result(test_agg_data) # Summary Data @@ -127,7 +127,10 @@ def api_test_data(): mock_plot_data = test_plot_data.copy() mock_plot_data["EventTime"] = mock_plot_data["EventTime"].strftime(datetime_format) - mock_plot_df = json.dumps(mock_plot_data, separators=(",", ":")) + mock_plot_df = { + "data": json.dumps(mock_plot_data, separators=(",", ":")), + "count": 1, + } expected_plot = expected_result(test_plot_data) test_summary_data = { @@ -141,7 +144,10 @@ def api_test_data(): "Var": 0.0, } - mock_summary_df = json.dumps(test_summary_data, separators=(",", ":")) + mock_summary_df = { + "data": json.dumps(test_summary_data, separators=(",", ":")), + "count": 1, + } expected_summary = expected_result(test_summary_data) test_metadata = { @@ -150,7 +156,10 @@ def api_test_data(): "Description": "Test Description", } - mock_metadata_df = json.dumps(test_metadata, separators=(",", ":")) + mock_metadata_df = { + "data": json.dumps(test_metadata, separators=(",", ":")), + "count": 1, + } expected_metadata = expected_result(test_metadata) test_latest_data = { @@ -171,7 +180,10 @@ def api_test_data(): mock_latest_data["GoodEventTime"] = mock_latest_data["GoodEventTime"].strftime( datetime_format ) - mock_latest_df = json.dumps(mock_latest_data, separators=(",", ":")) + mock_latest_df = { + "data": json.dumps(mock_latest_data, separators=(",", ":")), + "count": 1, + } expected_latest = expected_result(test_latest_data) expected_sql = expected_result(test_raw_data, "100", "100") From 9090b547b5bf69a7ea3ce189860f813687471fd0 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 23 Jul 2024 13:06:48 +0100 Subject: [PATCH 5/8] Update for docs Signed-off-by: GBBBAS --- src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py index 166a90eee..696b12819 100644 --- a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py +++ b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py @@ -110,7 +110,7 @@ def execute(self, query: str) -> None: logging.exception("error while executing the query") raise e - def fetch_all(self, fetch_size=5_000_000): + def fetch_all(self, fetch_size=5_000_000) -> list | dict: """ Gets all rows of a query. From da76c9b7884d811b5476f92b44efe3089883b4d0 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 23 Jul 2024 13:08:30 +0100 Subject: [PATCH 6/8] Update to docs type Signed-off-by: GBBBAS --- src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py index 696b12819..099583296 100644 --- a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py +++ b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py @@ -110,7 +110,7 @@ def execute(self, query: str) -> None: logging.exception("error while executing the query") raise e - def fetch_all(self, fetch_size=5_000_000) -> list | dict: + def fetch_all(self, fetch_size=5_000_000) -> list: """ Gets all rows of a query. From dcc4e84959ce67cac2578f8b63529e3676e85ad6 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 23 Jul 2024 13:08:59 +0100 Subject: [PATCH 7/8] Update to type Signed-off-by: GBBBAS --- src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py index 099583296..bd881cd23 100644 --- a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py +++ b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Union from databricks import sql import pyarrow as pa import polars as pl @@ -110,7 +111,7 @@ def execute(self, query: str) -> None: logging.exception("error while executing the query") raise e - def fetch_all(self, fetch_size=5_000_000) -> list: + def fetch_all(self, fetch_size=5_000_000) -> Union[list, dict]: """ Gets all rows of a query. From f7c943938d44e5e349976276fca56b8cd645d57c Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 23 Jul 2024 13:35:36 +0100 Subject: [PATCH 8/8] Update for query tests Signed-off-by: GBBBAS --- src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py index bd881cd23..0a4182f51 100644 --- a/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py +++ b/src/sdk/python/rtdip_sdk/connectors/odbc/db_sql_connector.py @@ -120,7 +120,7 @@ def fetch_all(self, fetch_size=5_000_000) -> Union[list, dict]: """ try: get_next_result = True - results = None if ConnectionReturnType.String else [] + results = None if self.return_type == ConnectionReturnType.String else [] count = 0 while get_next_result: result = self.cursor.fetchmany_arrow(fetch_size)