Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Batch API Updates #797

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 8, 2024
Merged

Batch API Updates #797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7cce44d
API: Update and refactor batch route to only use lookup when paramete…
ummer-shell Aug 6, 2024
cdc9317
API: Update batch route function mapping, including sql route misnami…
ummer-shell Aug 6, 2024
8768fd7
API: Update batch route mapping - include underscores for required fu…
ummer-shell Aug 8, 2024
279bd3a
Merge branch 'develop' into feature/batch-api-updates
ummer-shell Aug 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions src/api/v1/batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@
"/events/resample": "resample",
"/events/plot": "plot",
"/events/interpolate": "interpolate",
"/events/interpolationattime": "interpolationattime",
"/events/circularaverage": "circularaverage",
"/events/circularstandarddeviation": "circularstandarddeviation",
"/events/timeweightedaverage": "timeweightedaverage",
"/events/interpolationattime": "interpolation_at_time",
"/events/circularaverage": "circular_average",
"/events/circularstandarddeviation": "circular_standard_deviation",
"/events/timeweightedaverage": "time_weighted_average",
"/events/summary": "summary",
"/events/metadata": "metadata",
"/sql/execute": "sql",
Expand Down Expand Up @@ -135,7 +135,6 @@ async def batch_events_get(
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Use executor.map to preserve order
results = executor.map(
# lambda arguments: lookup_before_get(*arguments),
lambda arguments: run_direct_or_lookup(*arguments),
[
(parsed_request["func"], connection, parsed_request["parameters"])
Expand Down
51 changes: 43 additions & 8 deletions tests/api/v1/api_test_objects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,14 +262,29 @@
)


BATCH_POST_PAYLOAD_SINGLE_WITH_MISSING_BUSINESS_UNIT = {
"requests": [
{
"url": "/events/summary",
"method": "GET",
"headers": TEST_HEADERS,
"params": SUMMARY_MOCKED_PARAMETER_DICT.copy(),
}
]
}
BATCH_POST_PAYLOAD_SINGLE_WITH_MISSING_BUSINESS_UNIT["requests"][0]["params"].pop(
"business_unit"
)


BATCH_POST_PAYLOAD_SINGLE_WITH_POST = {
"requests": [
{
"url": "/events/raw",
"url": "/events/timeweightedaverage",
"method": "POST",
"headers": TEST_HEADERS,
"params": RAW_MOCKED_PARAMETER_DICT,
"body": RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT,
"params": TIME_WEIGHTED_AVERAGE_MOCKED_PARAMETER_DICT,
"body": TIME_WEIGHTED_AVERAGE_POST_BODY_MOCKED_PARAMETER_DICT,
}
]
}
Expand Down Expand Up @@ -300,17 +315,37 @@
BATCH_POST_PAYLOAD_MULTIPLE = {
"requests": [
{
"url": "/events/summary",
"url": "/events/interpolationattime",
"method": "GET",
"headers": TEST_HEADERS,
"params": SUMMARY_MOCKED_PARAMETER_DICT,
"params": INTERPOLATION_AT_TIME_MOCKED_PARAMETER_DICT,
},
{
"url": "/events/raw",
"url": "/events/circularaverage",
"method": "POST",
"headers": TEST_HEADERS,
"params": RAW_MOCKED_PARAMETER_DICT,
"body": RESAMPLE_POST_BODY_MOCKED_PARAMETER_DICT,
"params": CIRCULAR_AVERAGE_MOCKED_PARAMETER_DICT,
"body": CIRCULAR_AVERAGE_POST_BODY_MOCKED_PARAMETER_DICT,
},
]
}

BATCH_POST_PAYLOAD_ONE_SUCCESS_ONE_FAIL = {
"requests": [
{
"url": "/sql/execute",
"method": "POST",
"headers": TEST_HEADERS,
"params": {},
"body": {
"sql_statement": "SELECT * FROM 1",
},
},
{
"url": "/events/raw",
"method": "GET",
"headers": TEST_HEADERS,
"params": {},
},
]
}
Expand Down
71 changes: 70 additions & 1 deletion tests/api/v1/test_api_batch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,9 @@
from httpx import AsyncClient
from src.api.v1 import app
from src.api.v1.common import json_response_batch
from src.sdk.python.rtdip_sdk.queries.time_series import batch

MOCK_METHOD = "src.sdk.python.rtdip_sdk.queries.time_series.raw.get"
MOCK_METHOD = "src.sdk.python.rtdip_sdk.queries.time_series.batch.get"
MOCK_API_NAME = "/api/v1/events/batch"

pytestmark = pytest.mark.anyio
Expand Down Expand Up @@ -251,6 +252,9 @@ async def test_api_batch_single_post_success(mocker: MockerFixture):
os.environ, {"DATABRICKS_SERVING_ENDPOINT": MOCK_MAPPING_ENDPOINT_URL}
)

# Make a surveillance batch method reference to check if called and what args with
surveillance_batch = mocker.patch(mock_method, return_value=mock_method_return_data)

async with AsyncClient(app=app, base_url=BASE_URL) as ac:
actual = await ac.post(
MOCK_API_NAME,
Expand All @@ -261,6 +265,10 @@ async def test_api_batch_single_post_success(mocker: MockerFixture):

expected = json.loads(json_response_batch([test_data]).body.decode("utf-8"))

# Check batch method called with correct parameters, specifically the right function mapping
assert surveillance_batch.call_count == 1
assert surveillance_batch.call_args[0][1][0]["type"] == "time_weighted_average"

assert actual.json() == expected
assert actual.status_code == 200

Expand Down Expand Up @@ -395,6 +403,9 @@ async def test_api_batch_multiple_success(mocker: MockerFixture):
os.environ, {"DATABRICKS_SERVING_ENDPOINT": MOCK_MAPPING_ENDPOINT_URL}
)

# Make a surveillance batch method reference to check if called and what args with
surveillance_batch = mocker.patch(mock_method, side_effect=mock_patch_side_effect)

async with AsyncClient(app=app, base_url=BASE_URL) as ac:
actual = await ac.post(
MOCK_API_NAME,
Expand All @@ -407,6 +418,64 @@ async def test_api_batch_multiple_success(mocker: MockerFixture):
json_response_batch([summary_test_data, raw_test_data]).body.decode("utf-8")
)

# Check batch method called with correct parameters, specifically the right function mappings
assert surveillance_batch.call_count == 2
assert (
surveillance_batch.call_args_list[0][0][1][0]["type"] == "interpolation_at_time"
)
assert surveillance_batch.call_args_list[1][0][1][0]["type"] == "circular_average"

assert actual.json() == expected
assert actual.status_code == 200


# Test where one fails and one passes, including
async def test_api_batch_one_success_one_fail(mocker: MockerFixture):
"""
Case when single post request supplied in overall array of
correct format, but one passes and one fails due to missing parameters
"""

sql_test_data = pd.DataFrame(
{
"EventTime": [datetime.now(timezone.utc)],
"TagName": ["TestTag"],
"Status": ["Good"],
"Value": [1.01],
}
)

raw_test_data_fail = pd.DataFrame([{"Error": "'tag_names'"}])

# Mock the batch method, which outputs test data in the form of an array of dfs
mock_method = "src.sdk.python.rtdip_sdk.queries.time_series.batch.get"
mock_method_return_data = None
# add side effect since require batch to return different data after each call
# batch.get return value is array of dfs, so must patch with nested array
mock_patch_side_effect = [[sql_test_data], [raw_test_data_fail]]
mocker = mocker_setup(
mocker,
mock_method,
mock_method_return_data,
patch_side_effect=mock_patch_side_effect,
tag_mapping_data=MOCK_TAG_MAPPING_SINGLE,
)
mocker.patch.dict(
os.environ, {"DATABRICKS_SERVING_ENDPOINT": MOCK_MAPPING_ENDPOINT_URL}
)

async with AsyncClient(app=app, base_url=BASE_URL) as ac:
actual = await ac.post(
MOCK_API_NAME,
headers=TEST_HEADERS,
params=BATCH_MOCKED_PARAMETER_DICT,
json=BATCH_POST_PAYLOAD_ONE_SUCCESS_ONE_FAIL,
)

expected = json.loads(
json_response_batch([sql_test_data, raw_test_data_fail]).body.decode("utf-8")
)

assert actual.json() == expected
assert actual.status_code == 200

Expand Down