From 7cfc6c5bc32ffc462ba3d63e4d4105afb8ab15c4 Mon Sep 17 00:00:00 2001 From: ummer-shell <124354705+ummer-shell@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:02:41 +0100 Subject: [PATCH 1/2] Batch API updates (#808) * API FIX: Ensure threadpool workers parsed as integers Signed-off-by: Ummer Taahir * API FIX: Add batch base query params rather than using normal base Signed-off-by: Ummer Taahir * Update mkdocstrings versions Signed-off-by: Ummer Taahir --------- Signed-off-by: Ummer Taahir --- environment.yml | 4 ++-- src/api/v1/batch.py | 11 +++++++---- src/api/v1/common.py | 3 +++ src/api/v1/models.py | 10 ++++++++++ 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/environment.yml b/environment.yml index aad477b70..a382d33e7 100644 --- a/environment.yml +++ b/environment.yml @@ -47,8 +47,8 @@ dependencies: - openai>=1.13.3,<2.0.0 - mkdocs-material==9.5.20 - mkdocs-material-extensions==1.3.1 - - mkdocstrings==0.22.0 - - mkdocstrings-python==1.4.0 + - mkdocstrings==0.25.0 + - mkdocstrings-python==1.10.8 - mkdocs-macros-plugin==1.0.1 - pygments==2.16.1 - pymdown-extensions==10.8.1 diff --git a/src/api/v1/batch.py b/src/api/v1/batch.py index 861b60986..4b17c772e 100755 --- a/src/api/v1/batch.py +++ b/src/api/v1/batch.py @@ -18,7 +18,7 @@ from src.sdk.python.rtdip_sdk.queries.time_series import batch from src.api.v1.models import ( - BaseQueryParams, + BatchBaseQueryParams, BaseHeaders, BatchBodyParams, BatchResponse, @@ -128,8 +128,11 @@ async def batch_events_get( # Parse requests into dicts required by sdk parsed_requests = parse_batch_requests(batch_query_parameters.requests) - # Obtain max workers from environment var, otherwise default to one less than cpu count - max_workers = os.environ.get("BATCH_THREADPOOL_WORKERS", os.cpu_count() - 1) + # Obtain max workers from environment var, otherwise default to 10 + max_workers = os.environ.get("BATCH_THREADPOOL_WORKERS", 10) + + # ensure max_workers is an integer + max_workers = int(max_workers) # Request the data for each concurrently with threadpool with ThreadPoolExecutor(max_workers=max_workers) as executor: @@ -172,7 +175,7 @@ async def batch_events_get( }, ) async def batch_post( - base_query_parameters: BaseQueryParams = Depends(), + base_query_parameters: BatchBaseQueryParams = Depends(), batch_query_parameters: BatchBodyParams = Body(default=...), base_headers: BaseHeaders = Depends(), limit_offset_query_parameters: LimitOffsetQueryParams = Depends(), diff --git a/src/api/v1/common.py b/src/api/v1/common.py index 04e657a5b..42b27e487 100644 --- a/src/api/v1/common.py +++ b/src/api/v1/common.py @@ -285,6 +285,9 @@ def lookup_before_get( # make default workers 3 as within one query typically will request from only a few tables at once max_workers = os.environ.get("LOOKUP_THREADPOOL_WORKERS", 3) + # ensure max_workers is an integer + max_workers = int(max_workers) + # run function with each parameters concurrently results = batch.get(connection, request_list, threadpool_max_workers=max_workers) diff --git a/src/api/v1/models.py b/src/api/v1/models.py index 34461f48b..d07fc9b4c 100644 --- a/src/api/v1/models.py +++ b/src/api/v1/models.py @@ -253,6 +253,16 @@ def __init__( self.authorization = authorization +class BatchBaseQueryParams: + def __init__( + self, + region: str = Query(..., description="Region"), + authorization: str = Depends(oauth2_scheme), + ): + self.region = region + self.authorization = authorization + + class MetadataQueryParams: def __init__( self, From 6454573b3442fdb06d95318a48548029cde393ea Mon Sep 17 00:00:00 2001 From: GBBBAS <42962356+GBBBAS@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:29:16 +0100 Subject: [PATCH 2/2] Authentication docs updates (#809) Signed-off-by: GBBBAS --- docs/sdk/authentication/azure.md | 10 +++++----- docs/university/essentials/api/exercise.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/sdk/authentication/azure.md b/docs/sdk/authentication/azure.md index 231587a1b..60f99d3d3 100644 --- a/docs/sdk/authentication/azure.md +++ b/docs/sdk/authentication/azure.md @@ -23,18 +23,18 @@ The following section describes authentication using [Azure Active Directory.](h 1\. Import **rtdip-sdk** authentication methods with the following: - from rtdip_sdk.authentication import authenticate as auth + from rtdip_sdk.authentication import azure as auth 2\. Use any of the following authentication methods. Replace **tenant_id** , **client_id**, **certificate_path** or **client_secret** with your own details. === "Default Authentication" - DefaultAzureCredential = auth.DefaultAuth().authenticate() + credential = auth.DefaultAuth().authenticate() === "Certificate Authentication" - CertificateCredential = auth.CertificateAuth(tenant_id, client_id, certificate_path).authenticate() + credential = auth.CertificateAuth(tenant_id, client_id, certificate_path).authenticate() === "Client Secret Authentication" - ClientSecretCredential = auth.ClientSecretAuth(tenant_id, client_id, client_secret).authenticate() + credential = auth.ClientSecretAuth(tenant_id, client_id, client_secret).authenticate() 3\. The methods above will return back a Client Object. The following example will show you how to retrieve the access_token from a credential object. The access token will be used in later steps to connect to RTDIP via the three options (Databricks SQL Connect, PYODBC SQL Connect, TURBODBC SQL Connect). @@ -43,7 +43,7 @@ The following section describes authentication using [Azure Active Directory.](h Once authenticated, it is possible to retrieve tokens for specific Azure Resources by providing scopes when retrieving tokens. Please see below for examples of how to retrieve tokens for Azure resources regularly used in RTDIP. === "Databricks" - access_token = DefaultAzureCredential.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + access_token = credential.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token diff --git a/docs/university/essentials/api/exercise.md b/docs/university/essentials/api/exercise.md index 86755ff10..5012929fa 100644 --- a/docs/university/essentials/api/exercise.md +++ b/docs/university/essentials/api/exercise.md @@ -2,7 +2,7 @@ In this exercise, you will learn how to run APIs on Swagger and Postman. -1. Go to [Swagger](https://ssip-api.shell.com/docs) and click the green Authorize button on the right. +1. Go to your RTDIP Swagger page and click the green Authorize button on the right. 2. Call a `Raw Get` query to retrieve some data from your time series data source.