From 1d0a49524501dbc8ee29163dc23a9f24f86d427c Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Fri, 3 May 2024 16:50:09 +0100 Subject: [PATCH 1/2] Updates for Packages and API dates Signed-off-by: GBBBAS --- environment.yml | 6 ++-- setup.py | 1 + src/api/requirements.txt | 2 +- src/api/v1/models.py | 24 ++++++++++++--- .../queries/_utilities_query_builder.py | 2 +- tests/api/v1/test_api_raw.py | 30 +++++++++++++++++++ 6 files changed, 56 insertions(+), 9 deletions(-) diff --git a/environment.yml b/environment.yml index cec84b295..ec71b703d 100644 --- a/environment.yml +++ b/environment.yml @@ -39,7 +39,7 @@ dependencies: - fastapi==0.110.0 - httpx==0.24.1 - pyspark>=3.3.0,<3.6.0 - - delta-spark>=2.2.0,<3.2.0 + - delta-spark>=2.2.0,<3.3.0 - grpcio>=1.48.1 - grpcio-status>=1.48.1 - googleapis-common-protos>=1.56.4 @@ -52,7 +52,7 @@ dependencies: - mkdocs-macros-plugin==1.0.1 - pygments==2.16.1 - pymdown-extensions==10.8.1 - - databricks-sql-connector==3.1.0 + - databricks-sql-connector==2.9.3 - semver==3.0.0 - xlrd==2.0.1 - pygithub==1.59.0 @@ -81,4 +81,4 @@ dependencies: - trio==0.22.1 - sqlparams==5.1.0 - entsoe-py==0.5.10 - - eth-typing==4.2.1 \ No newline at end of file + - eth-typing>=4.2.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 9abecb1b7..1df183744 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ "hvac==1.1.1", "azure-keyvault-secrets==4.7.0", "web3==6.16.0", + "eth-typing>=4.2.2", "polars[deltalake]==0.18.8", "delta-sharing==1.0.0", "xarray>=2023.1.0,<2023.8.0", diff --git a/src/api/requirements.txt b/src/api/requirements.txt index ceb0ce9e2..420335190 100644 --- a/src/api/requirements.txt +++ b/src/api/requirements.txt @@ -5,7 +5,7 @@ pydantic==2.6.0 # turbodbc==4.11.0 pyodbc==4.0.39 importlib_metadata>=1.0.0 -databricks-sql-connector==3.1.0 +databricks-sql-connector==2.9.3 azure-identity==1.15.0 oauthlib>=3.2.2 pandas>=2.0.1,<3.0.0 diff --git a/src/api/v1/models.py b/src/api/v1/models.py index e7989610c..61f0dd2b4 100644 --- a/src/api/v1/models.py +++ b/src/api/v1/models.py @@ -15,8 +15,15 @@ import os from datetime import datetime from tracemalloc import start -from pydantic import BaseModel, ConfigDict, Field, field_serializer -from typing import List, Union, Dict, Any +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + Field, + Strict, + field_serializer, +) +from typing import Annotated, List, Union, Dict, Any from fastapi import Query, Header, Depends from datetime import date from src.api.auth.azuread import oauth2_scheme @@ -244,6 +251,13 @@ def __init__( self.tag_name = tag_name +def check_date(v: str) -> str: + assert ( + len(v) == 10 or len(v) == 15 or len(v) == 16 + ) # "Date must be in format YYYY-MM-DD, YYYY-MM-DD+zzzz or YYYY-MM-DD+zz:zz" + return v + + class RawQueryParams: def __init__( self, @@ -255,12 +269,14 @@ def __init__( include_bad_data: bool = Query( ..., description="Include or remove Bad data points" ), - start_date: Union[datetime, date] = Query( + start_date: Union[ + Annotated[date, BeforeValidator(check_date)], datetime + ] = Query( ..., description="Start Date in format YYYY-MM-DD or YYYY-MM-DDTHH:mm:ss or YYYY-MM-DDTHH:mm:ss+zz:zz", examples=[EXAMPLE_DATE, EXAMPLE_DATETIME, EXAMPLE_DATETIME_TIMEZOME], ), - end_date: Union[datetime, date] = Query( + end_date: Union[Annotated[date, BeforeValidator(check_date)], datetime] = Query( ..., description="End Date in format YYYY-MM-DD or YYYY-MM-DDTHH:mm:ss or YYYY-MM-DDTHH:mm:ss+zz:zz", examples=[EXAMPLE_DATE, EXAMPLE_DATETIME, EXAMPLE_DATETIME_TIMEZOME], diff --git a/src/sdk/python/rtdip_sdk/queries/_utilities_query_builder.py b/src/sdk/python/rtdip_sdk/queries/_utilities_query_builder.py index 467243e65..55babde8e 100644 --- a/src/sdk/python/rtdip_sdk/queries/_utilities_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/_utilities_query_builder.py @@ -29,7 +29,7 @@ def _is_date_format(dt, format): def _parse_date(dt, is_end_date=False, exclude_date_format=False): # NOSONAR if isinstance(dt, datetime): - if dt.time() == time.min: + if dt.time() == time.min and not is_end_date: if dt.tzinfo is not None: dt = datetime.strftime(dt, "%Y-%m-%d%z") else: diff --git a/tests/api/v1/test_api_raw.py b/tests/api/v1/test_api_raw.py index 674e93c20..411aac527 100644 --- a/tests/api/v1/test_api_raw.py +++ b/tests/api/v1/test_api_raw.py @@ -190,3 +190,33 @@ async def test_api_raw_post_error(mocker: MockerFixture): assert response.status_code == 400 assert actual == '{"detail":"Error Connecting to Database"}' + + +# async def test_api_raw_get_success_dates(mocker: MockerFixture): +# test_data = pd.DataFrame( +# { +# "EventTime": [datetime.utcnow()], +# "TagName": ["TestTag"], +# "Status": ["Good"], +# "Value": [1.01], +# } +# ) +# mocker = mocker_setup(mocker, MOCK_METHOD, test_data) + +# raw_parameters_with_updated_dates = RAW_MOCKED_PARAMETER_DICT.copy() +# raw_parameters_with_updated_dates["start_date"] = "2023-03-10" +# raw_parameters_with_updated_dates["end_date"] = "2023-03-11T00:00:00" +# async with AsyncClient(app=app, base_url=BASE_URL) as ac: +# response = await ac.get( +# MOCK_API_NAME, +# headers=TEST_HEADERS, +# params=raw_parameters_with_updated_dates, +# ) +# actual = response.text +# expected = test_data.to_json(orient="table", index=False, date_unit="ns") +# expected = ( +# expected.rstrip("}") + ',"pagination":{"limit":null,"offset":null,"next":null}}' +# ) + +# assert response.status_code == 200 +# assert actual == expected From 37a4413f6a92da1e1c461034599a2f0cf623b189 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Fri, 3 May 2024 16:51:04 +0100 Subject: [PATCH 2/2] Remove test Signed-off-by: GBBBAS --- tests/api/v1/test_api_raw.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/tests/api/v1/test_api_raw.py b/tests/api/v1/test_api_raw.py index 411aac527..674e93c20 100644 --- a/tests/api/v1/test_api_raw.py +++ b/tests/api/v1/test_api_raw.py @@ -190,33 +190,3 @@ async def test_api_raw_post_error(mocker: MockerFixture): assert response.status_code == 400 assert actual == '{"detail":"Error Connecting to Database"}' - - -# async def test_api_raw_get_success_dates(mocker: MockerFixture): -# test_data = pd.DataFrame( -# { -# "EventTime": [datetime.utcnow()], -# "TagName": ["TestTag"], -# "Status": ["Good"], -# "Value": [1.01], -# } -# ) -# mocker = mocker_setup(mocker, MOCK_METHOD, test_data) - -# raw_parameters_with_updated_dates = RAW_MOCKED_PARAMETER_DICT.copy() -# raw_parameters_with_updated_dates["start_date"] = "2023-03-10" -# raw_parameters_with_updated_dates["end_date"] = "2023-03-11T00:00:00" -# async with AsyncClient(app=app, base_url=BASE_URL) as ac: -# response = await ac.get( -# MOCK_API_NAME, -# headers=TEST_HEADERS, -# params=raw_parameters_with_updated_dates, -# ) -# actual = response.text -# expected = test_data.to_json(orient="table", index=False, date_unit="ns") -# expected = ( -# expected.rstrip("}") + ',"pagination":{"limit":null,"offset":null,"next":null}}' -# ) - -# assert response.status_code == 200 -# assert actual == expected