From 7cce44d4d1e6c8fc2c053b65f61e99a9f2854dce Mon Sep 17 00:00:00 2001 From: Ummer Taahir Date: Tue, 6 Aug 2024 10:25:43 +0100 Subject: [PATCH 1/3] API: Update and refactor batch route to only use lookup when parameters not fully defined Signed-off-by: Ummer Taahir --- src/api/v1/batch.py | 96 +++++++++++++------ tests/api/v1/api_test_objects.py | 49 ++++++++-- tests/api/v1/test_api_batch.py | 156 ++++++++++++++++++++++++++++++- 3 files changed, 265 insertions(+), 36 deletions(-) diff --git a/src/api/v1/batch.py b/src/api/v1/batch.py index 2e42388d8..db2d2ff38 100755 --- a/src/api/v1/batch.py +++ b/src/api/v1/batch.py @@ -15,6 +15,7 @@ import numpy as np import os from fastapi import HTTPException, Depends, Body # , JSONResponse +from src.sdk.python.rtdip_sdk.queries.time_series import batch from src.api.v1.models import ( BaseQueryParams, @@ -33,6 +34,7 @@ from src.api.FastAPIApp import api_v1_router from src.api.v1.common import lookup_before_get from concurrent.futures import * +import pandas as pd ROUTE_FUNCTION_MAPPING = { @@ -51,41 +53,80 @@ } +def parse_batch_requests(requests): + """ + Parse requests into dict of required format of sdk function + - Unpack request body if post request + - Map the url to the sdk function + - Rename tag_name parameter to tag_names + """ + + parsed_requests = [] + for request in requests: + + # If required, combine request body and parameters: + parameters = request["params"] + if request["method"] == "POST": + if request["body"] == None: + raise Exception( + "Incorrectly formatted request provided: All POST requests require a body" + ) + parameters = {**parameters, **request["body"]} + + # Map the url to a specific function + try: + func = ROUTE_FUNCTION_MAPPING[request["url"]] + except: + raise Exception( + "Unsupported url: Only relative base urls are supported, for example '/events/raw'. Please provide any parameters under the params key in the same format as the sdk" + ) + + # Rename tag_name to tag_names, if required + if "tag_name" in parameters.keys(): + parameters["tag_names"] = parameters.pop("tag_name") + + # Append to array + parsed_requests.append({"func": func, "parameters": parameters}) + + return parsed_requests + + +def run_direct_or_lookup(func_name, connection, parameters): + """ + Runs directly if all params (or SQL function) provided, otherwise uses lookup table + """ + try: + if func_name == "sql" or all( + (key in parameters and parameters[key] != None) + for key in ["business_unit", "asset"] + ): + # Run batch get for single table query if table name provided, or SQL function + params_list = [{"type": func_name, "parameters_dict": parameters}] + batch_results = batch.get(connection, params_list, threadpool_max_workers=1) + + # Extract 0th from generator object since only one result + result = [result for result in batch_results][0] + return result + else: + return lookup_before_get(func_name, connection, parameters) + except Exception as e: + # Return a dataframe with an error message if any of requests fail + return pd.DataFrame([{"Error": str(e)}]) + + async def batch_events_get( base_query_parameters, base_headers, batch_query_parameters, limit_offset_parameters ): + try: + # Set up connection (connection, parameters) = common_api_setup_tasks( base_query_parameters=base_query_parameters, base_headers=base_headers, ) - # Validate the parameters - parsed_requests = [] - for request in batch_query_parameters.requests: - # If required, combine request body and parameters: - parameters = request["params"] - if request["method"] == "POST": - if request["body"] == None: - raise Exception( - "Incorrectly formatted request provided: All POST requests require a body" - ) - parameters = {**parameters, **request["body"]} - - # Map the url to a specific function - try: - func = ROUTE_FUNCTION_MAPPING[request["url"]] - except: - raise Exception( - "Unsupported url: Only relative base urls are supported. Please provide any parameters in the params key" - ) - - # Rename tag_name to tag_names, if required - if "tag_name" in parameters.keys(): - parameters["tag_names"] = parameters.pop("tag_name") - - # Append to array - parsed_requests.append({"func": func, "parameters": parameters}) + # Parse requests into dicts required by sdk + parsed_requests = parse_batch_requests(batch_query_parameters.requests) # Obtain max workers from environment var, otherwise default to one less than cpu count max_workers = os.environ.get("BATCH_THREADPOOL_WORKERS", os.cpu_count() - 1) @@ -94,7 +135,8 @@ async def batch_events_get( with ThreadPoolExecutor(max_workers=max_workers) as executor: # Use executor.map to preserve order results = executor.map( - lambda arguments: lookup_before_get(*arguments), + # lambda arguments: lookup_before_get(*arguments), + lambda arguments: run_direct_or_lookup(*arguments), [ (parsed_request["func"], connection, parsed_request["parameters"]) for parsed_request in parsed_requests diff --git a/tests/api/v1/api_test_objects.py b/tests/api/v1/api_test_objects.py index 098855a78..bfd7bdf92 100644 --- a/tests/api/v1/api_test_objects.py +++ b/tests/api/v1/api_test_objects.py @@ -239,18 +239,33 @@ BATCH_POST_PAYLOAD_SINGLE_WITH_GET = { "requests": [ { - "url": "/api/v1/events/summary", + "url": "/events/summary", "method": "GET", "headers": TEST_HEADERS, - "params": SUMMARY_MOCKED_PARAMETER_DICT, + "params": SUMMARY_MOCKED_PARAMETER_DICT.copy(), + } + ] +} + +BATCH_POST_PAYLOAD_SINGLE_WITH_MISSING_BUSINESS_UNIT = { + "requests": [ + { + "url": "/events/summary", + "method": "GET", + "headers": TEST_HEADERS, + "params": SUMMARY_MOCKED_PARAMETER_DICT.copy(), } ] } +BATCH_POST_PAYLOAD_SINGLE_WITH_MISSING_BUSINESS_UNIT["requests"][0]["params"].pop( + "business_unit" +) + BATCH_POST_PAYLOAD_SINGLE_WITH_POST = { "requests": [ { - "url": "/api/v1/events/raw", + "url": "/events/raw", "method": "POST", "headers": TEST_HEADERS, "params": RAW_MOCKED_PARAMETER_DICT, @@ -262,7 +277,7 @@ BATCH_POST_PAYLOAD_SINGLE_WITH_GET_ERROR_DICT = { "requests": [ { - "url": "an_unsupported_route", + "url": "/api/v1/events/raw", # Invalid URL since it should be /events/raw "method": "GET", "headers": TEST_HEADERS, "params": SUMMARY_MOCKED_PARAMETER_DICT, @@ -273,7 +288,7 @@ BATCH_POST_PAYLOAD_SINGLE_WITH_POST_ERROR_DICT = { "requests": [ { - "url": "/api/v1/events/raw", + "url": "/events/raw", "method": "POST", "headers": TEST_HEADERS, "params": RAW_MOCKED_PARAMETER_DICT, @@ -285,13 +300,13 @@ BATCH_POST_PAYLOAD_MULTIPLE = { "requests": [ { - "url": "/api/v1/events/summary", + "url": "/events/summary", "method": "GET", "headers": TEST_HEADERS, "params": SUMMARY_MOCKED_PARAMETER_DICT, }, { - "url": "/api/v1/events/raw", + "url": "/events/raw", "method": "POST", "headers": TEST_HEADERS, "params": RAW_MOCKED_PARAMETER_DICT, @@ -300,6 +315,26 @@ ] } +BATCH_POST_PAYLOAD_ONE_SUCCESS_ONE_FAIL = { + "requests": [ + { + "url": "/sql/execute", + "method": "POST", + "headers": TEST_HEADERS, + "params": {}, + "body": { + "sql_statement": "SELECT * FROM 1", + }, + }, + { + "url": "/events/raw", + "method": "GET", + "headers": TEST_HEADERS, + "params": {}, + }, + ] +} + # Tag mapping test parameters MOCK_TAG_MAPPING_SINGLE = { diff --git a/tests/api/v1/test_api_batch.py b/tests/api/v1/test_api_batch.py index 48f679117..dab425c5a 100644 --- a/tests/api/v1/test_api_batch.py +++ b/tests/api/v1/test_api_batch.py @@ -22,10 +22,12 @@ from tests.api.v1.api_test_objects import ( BATCH_MOCKED_PARAMETER_DICT, BATCH_POST_PAYLOAD_SINGLE_WITH_GET, + BATCH_POST_PAYLOAD_SINGLE_WITH_MISSING_BUSINESS_UNIT, BATCH_POST_PAYLOAD_SINGLE_WITH_POST, BATCH_POST_PAYLOAD_SINGLE_WITH_GET_ERROR_DICT, BATCH_POST_PAYLOAD_SINGLE_WITH_POST_ERROR_DICT, BATCH_POST_PAYLOAD_MULTIPLE, + BATCH_POST_PAYLOAD_ONE_SUCCESS_ONE_FAIL, mocker_setup, TEST_HEADERS, BASE_URL, @@ -48,7 +50,8 @@ async def test_api_batch_single_get_success(mocker: MockerFixture): """ - Case when single get request supplied in array of correct format + Case when single get request supplied in array of correct format, + fully defined parameters so no lookup required """ test_data = pd.DataFrame( @@ -73,10 +76,16 @@ async def test_api_batch_single_get_success(mocker: MockerFixture): mock_method_return_data, tag_mapping_data=MOCK_TAG_MAPPING_SINGLE, ) + + # Mock the mapping endpoint variable mocker.patch.dict( os.environ, {"DATABRICKS_SERVING_ENDPOINT": MOCK_MAPPING_ENDPOINT_URL} ) + # Mock the lookup_before_get function, so we can check if called + mock_lookup = "src.api.v1.batch.lookup_before_get" + mocked_lookup_before_get = mocker.patch(mock_lookup, return_value=None) + async with AsyncClient(app=app, base_url=BASE_URL) as ac: actual = await ac.post( MOCK_API_NAME, @@ -119,6 +128,98 @@ async def test_api_batch_single_get_success(mocker: MockerFixture): ] } + # Check lookup_before_get function not called - since parameters fully defined + assert mocked_lookup_before_get.call_count == 0 + + # Check response + assert actual.json() == expected + assert actual.status_code == 200 + + +async def test_api_batch_single_get_success_with_lookup(mocker: MockerFixture): + """ + Case when single get request supplied in array of correct format, + but with missing business unit, so lookup is required + """ + + test_data = pd.DataFrame( + { + "TagName": ["TestTag"], + "Count": [10.0], + "Avg": [5.05], + "Min": [1.0], + "Max": [10.0], + "StDev": [3.02], + "Sum": [25.0], + "Var": [0.0], + } + ) + + # Mock the batch method, which outputs test data in the form of an array of dfs + mock_method = "src.sdk.python.rtdip_sdk.queries.time_series.batch.get" + mock_method_return_data = [test_data] + mocker = mocker_setup( + mocker, + mock_method, + mock_method_return_data, + tag_mapping_data=MOCK_TAG_MAPPING_SINGLE, + ) + + # Mock the mapping endpoint variable + mocker.patch.dict( + os.environ, {"DATABRICKS_SERVING_ENDPOINT": MOCK_MAPPING_ENDPOINT_URL} + ) + + # Mock the lookup_before_get function + mock_lookup = "src.api.v1.batch.lookup_before_get" + mocked_lookup_before_get = mocker.patch(mock_lookup, return_value=test_data) + + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + actual = await ac.post( + MOCK_API_NAME, + headers=TEST_HEADERS, + params=BATCH_MOCKED_PARAMETER_DICT, + json=BATCH_POST_PAYLOAD_SINGLE_WITH_MISSING_BUSINESS_UNIT, + ) + + # Define full expected structure for one test - for remainder use json_response_batch as already tested in common + expected = { + "data": [ + { + "schema": { + "fields": [ + {"name": "TagName", "type": "string"}, + {"name": "Count", "type": "number"}, + {"name": "Avg", "type": "number"}, + {"name": "Min", "type": "number"}, + {"name": "Max", "type": "number"}, + {"name": "StDev", "type": "number"}, + {"name": "Sum", "type": "number"}, + {"name": "Var", "type": "number"}, + ], + "primaryKey": False, + "pandas_version": "1.4.0", + }, + "data": [ + { + "TagName": "TestTag", + "Count": 10.0, + "Avg": 5.05, + "Min": 1.0, + "Max": 10.0, + "StDev": 3.02, + "Sum": 25.0, + "Var": 0.0, + } + ], + } + ] + } + + # Check lookup_before_get function was called + assert mocked_lookup_before_get.call_count == 1 + + # Check response assert actual.json() == expected assert actual.status_code == 200 @@ -200,7 +301,7 @@ async def test_api_batch_single_get_unsupported_route_error(mocker: MockerFixtur ) expected = { - "detail": "Unsupported url: Only relative base urls are supported. Please provide any parameters in the params key" + "detail": "Unsupported url: Only relative base urls are supported, for example '/events/raw'. Please provide any parameters under the params key in the same format as the sdk" } assert actual.json() == expected @@ -308,3 +409,54 @@ async def test_api_batch_multiple_success(mocker: MockerFixture): assert actual.json() == expected assert actual.status_code == 200 + + +# Test where one fails and one passes, including +async def test_api_batch_one_success_one_fail(mocker: MockerFixture): + """ + Case when single post request supplied in overall array of + correct format, but one passes and one fails due to missing parameters + """ + + sql_test_data = pd.DataFrame( + { + "EventTime": [datetime.now(timezone.utc)], + "TagName": ["TestTag"], + "Status": ["Good"], + "Value": [1.01], + } + ) + + raw_test_data_fail = pd.DataFrame([{"Error": "'tag_names'"}]) + + # Mock the batch method, which outputs test data in the form of an array of dfs + mock_method = "src.sdk.python.rtdip_sdk.queries.time_series.batch.get" + mock_method_return_data = None + # add side effect since require batch to return different data after each call + # batch.get return value is array of dfs, so must patch with nested array + mock_patch_side_effect = [[sql_test_data], [raw_test_data_fail]] + mocker = mocker_setup( + mocker, + mock_method, + mock_method_return_data, + patch_side_effect=mock_patch_side_effect, + tag_mapping_data=MOCK_TAG_MAPPING_SINGLE, + ) + mocker.patch.dict( + os.environ, {"DATABRICKS_SERVING_ENDPOINT": MOCK_MAPPING_ENDPOINT_URL} + ) + + async with AsyncClient(app=app, base_url=BASE_URL) as ac: + actual = await ac.post( + MOCK_API_NAME, + headers=TEST_HEADERS, + params=BATCH_MOCKED_PARAMETER_DICT, + json=BATCH_POST_PAYLOAD_ONE_SUCCESS_ONE_FAIL, + ) + + expected = json.loads( + json_response_batch([sql_test_data, raw_test_data_fail]).body.decode("utf-8") + ) + + assert actual.json() == expected + assert actual.status_code == 200 From cdc9317f72a899d774865b136d46a9cd659006a9 Mon Sep 17 00:00:00 2001 From: Ummer Taahir Date: Tue, 6 Aug 2024 10:27:35 +0100 Subject: [PATCH 2/3] API: Update batch route function mapping, including sql route misnaming fix Signed-off-by: Ummer Taahir --- src/api/v1/batch.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/api/v1/batch.py b/src/api/v1/batch.py index db2d2ff38..3a970e585 100755 --- a/src/api/v1/batch.py +++ b/src/api/v1/batch.py @@ -38,18 +38,18 @@ ROUTE_FUNCTION_MAPPING = { - "/api/v1/events/raw": "raw", - "/api/v1/events/latest": "latest", - "/api/v1/events/resample": "resample", - "/api/v1/events/plot": "plot", - "/api/v1/events/interpolate": "interpolate", - "/api/v1/events/interpolationattime": "interpolationattime", - "/api/v1/events/circularaverage": "circularaverage", - "/api/v1/events/circularstandarddeviation": "circularstandarddeviation", - "/api/v1/events/timeweightedaverage": "timeweightedaverage", - "/api/v1/events/summary": "summary", - "/api/v1/events/metadata": "metadata", - "/api/v1/sql/execute": "execute", + "/events/raw": "raw", + "/events/latest": "latest", + "/events/resample": "resample", + "/events/plot": "plot", + "/events/interpolate": "interpolate", + "/events/interpolationattime": "interpolationattime", + "/events/circularaverage": "circularaverage", + "/events/circularstandarddeviation": "circularstandarddeviation", + "/events/timeweightedaverage": "timeweightedaverage", + "/events/summary": "summary", + "/events/metadata": "metadata", + "/sql/execute": "sql", } From 8768fd7276e42c85030566fb4982f13798e5f36d Mon Sep 17 00:00:00 2001 From: Ummer Taahir Date: Thu, 8 Aug 2024 16:04:41 +0100 Subject: [PATCH 3/3] API: Update batch route mapping - include underscores for required functions Signed-off-by: Ummer Taahir --- src/api/v1/batch.py | 8 ++++---- tests/api/v1/api_test_objects.py | 16 ++++++++-------- tests/api/v1/test_api_batch.py | 20 +++++++++++++++++++- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/api/v1/batch.py b/src/api/v1/batch.py index 3a970e585..8048baa24 100755 --- a/src/api/v1/batch.py +++ b/src/api/v1/batch.py @@ -43,10 +43,10 @@ "/events/resample": "resample", "/events/plot": "plot", "/events/interpolate": "interpolate", - "/events/interpolationattime": "interpolationattime", - "/events/circularaverage": "circularaverage", - "/events/circularstandarddeviation": "circularstandarddeviation", - "/events/timeweightedaverage": "timeweightedaverage", + "/events/interpolationattime": "interpolation_at_time", + "/events/circularaverage": "circular_average", + "/events/circularstandarddeviation": "circular_standard_deviation", + "/events/timeweightedaverage": "time_weighted_average", "/events/summary": "summary", "/events/metadata": "metadata", "/sql/execute": "sql", diff --git a/tests/api/v1/api_test_objects.py b/tests/api/v1/api_test_objects.py index bfd7bdf92..2ebf01116 100644 --- a/tests/api/v1/api_test_objects.py +++ b/tests/api/v1/api_test_objects.py @@ -265,11 +265,11 @@ BATCH_POST_PAYLOAD_SINGLE_WITH_POST = { "requests": [ { - "url": "/events/raw", + "url": "/events/timeweightedaverage", "method": "POST", "headers": TEST_HEADERS, - "params": RAW_MOCKED_PARAMETER_DICT, - "body": RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT, + "params": TIME_WEIGHTED_AVERAGE_MOCKED_PARAMETER_DICT, + "body": TIME_WEIGHTED_AVERAGE_POST_BODY_MOCKED_PARAMETER_DICT, } ] } @@ -300,17 +300,17 @@ BATCH_POST_PAYLOAD_MULTIPLE = { "requests": [ { - "url": "/events/summary", + "url": "/events/interpolationattime", "method": "GET", "headers": TEST_HEADERS, - "params": SUMMARY_MOCKED_PARAMETER_DICT, + "params": INTERPOLATION_AT_TIME_MOCKED_PARAMETER_DICT, }, { - "url": "/events/raw", + "url": "/events/circularaverage", "method": "POST", "headers": TEST_HEADERS, - "params": RAW_MOCKED_PARAMETER_DICT, - "body": RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT, + "params": CIRCULAR_AVERAGE_MOCKED_PARAMETER_DICT, + "body": CIRCULAR_AVERAGE_POST_BODY_MOCKED_PARAMETER_DICT, }, ] } diff --git a/tests/api/v1/test_api_batch.py b/tests/api/v1/test_api_batch.py index dab425c5a..395806055 100644 --- a/tests/api/v1/test_api_batch.py +++ b/tests/api/v1/test_api_batch.py @@ -41,8 +41,9 @@ from httpx import AsyncClient from src.api.v1 import app from src.api.v1.common import json_response_batch +from src.sdk.python.rtdip_sdk.queries.time_series import batch -MOCK_METHOD = "src.sdk.python.rtdip_sdk.queries.time_series.raw.get" +MOCK_METHOD = "src.sdk.python.rtdip_sdk.queries.time_series.batch.get" MOCK_API_NAME = "/api/v1/events/batch" pytestmark = pytest.mark.anyio @@ -251,6 +252,9 @@ async def test_api_batch_single_post_success(mocker: MockerFixture): os.environ, {"DATABRICKS_SERVING_ENDPOINT": MOCK_MAPPING_ENDPOINT_URL} ) + # Make a surveillance batch method reference to check if called and what args with + surveillance_batch = mocker.patch(mock_method, return_value=mock_method_return_data) + async with AsyncClient(app=app, base_url=BASE_URL) as ac: actual = await ac.post( MOCK_API_NAME, @@ -261,6 +265,10 @@ async def test_api_batch_single_post_success(mocker: MockerFixture): expected = json.loads(json_response_batch([test_data]).body.decode("utf-8")) + # Check batch method called with correct parameters, specifically the right function mapping + assert surveillance_batch.call_count == 1 + assert surveillance_batch.call_args[0][1][0]["type"] == "time_weighted_average" + assert actual.json() == expected assert actual.status_code == 200 @@ -395,6 +403,9 @@ async def test_api_batch_multiple_success(mocker: MockerFixture): os.environ, {"DATABRICKS_SERVING_ENDPOINT": MOCK_MAPPING_ENDPOINT_URL} ) + # Make a surveillance batch method reference to check if called and what args with + surveillance_batch = mocker.patch(mock_method, side_effect=mock_patch_side_effect) + async with AsyncClient(app=app, base_url=BASE_URL) as ac: actual = await ac.post( MOCK_API_NAME, @@ -407,6 +418,13 @@ async def test_api_batch_multiple_success(mocker: MockerFixture): json_response_batch([summary_test_data, raw_test_data]).body.decode("utf-8") ) + # Check batch method called with correct parameters, specifically the right function mappings + assert surveillance_batch.call_count == 2 + assert ( + surveillance_batch.call_args_list[0][0][1][0]["type"] == "interpolation_at_time" + ) + assert surveillance_batch.call_args_list[1][0][1][0]["type"] == "circular_average" + assert actual.json() == expected assert actual.status_code == 200