Thanks to visit codestin.com
Credit goes to github.com

Skip to content

API Json Serialization Performance Updates #671

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 28, 2024
Merged

API Json Serialization Performance Updates #671

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
31d8d4f
Updates to fetchmany on Databricks SQL Connector
GBBBAS Feb 21, 2024
871b162
API Updates for JSON Serialization
GBBBAS Feb 28, 2024
ad82817
Merge branch 'develop' into feature/api_json
GBBBAS Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 14 additions & 12 deletions src/api/v1/circular_average.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@
import numpy as np
from requests import request
from src.api.FastAPIApp import api_v1_router
from fastapi import HTTPException, Depends, Body
from fastapi import HTTPException, Depends, Body, Response
import nest_asyncio
from pandas.io.json import build_table_schema
from src.sdk.python.rtdip_sdk.queries.time_series import circular_average
from src.api.v1.models import (
BaseQueryParams,
BaseHeaders,
FieldSchema,
ResampleInterpolateResponse,
PivotResponse,
HTTPError,
Expand Down Expand Up @@ -62,18 +63,19 @@ def circular_average_events_get(

data = circular_average.get(connection, parameters)

if parameters.get("pivot") == True:
return PivotResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
)
else:
return ResampleInterpolateResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
return Response(
"{"
+ '"schema":{},"data":{},"pagination":{}'.format(
FieldSchema.model_validate(
build_table_schema(data, index=False, primary_key=False),
).model_dump_json(),
data.replace({np.nan: None}).to_json(
orient="records", date_format="iso", date_unit="us"
),
pagination(limit_offset_parameters, data).model_dump_json(),
)
+ "}",
)
except Exception as e:
logging.error(str(e))
raise HTTPException(status_code=400, detail=str(e))
Expand Down
27 changes: 14 additions & 13 deletions src/api/v1/circular_standard_deviation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@
import numpy as np
from requests import request
from src.api.FastAPIApp import api_v1_router
from fastapi import HTTPException, Depends, Body
from fastapi import HTTPException, Depends, Body, Response
import nest_asyncio
from pandas.io.json import build_table_schema
from src.sdk.python.rtdip_sdk.queries.time_series import circular_standard_deviation
from src.api.v1.models import (
BaseQueryParams,
BaseHeaders,
FieldSchema,
ResampleInterpolateResponse,
PivotResponse,
HTTPError,
Expand Down Expand Up @@ -62,19 +63,19 @@ def circular_standard_deviation_events_get(

data = circular_standard_deviation.get(connection, parameters)

if parameters.get("pivot") == True:
return PivotResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
return Response(
"{"
+ '"schema":{},"data":{},"pagination":{}'.format(
FieldSchema.model_validate(
build_table_schema(data, index=False, primary_key=False),
).model_dump_json(),
data.replace({np.nan: None}).to_json(
orient="records", date_format="iso", date_unit="us"
),
pagination(limit_offset_parameters, data).model_dump_json(),
)
else:
return ResampleInterpolateResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
)

+ "}",
)
except Exception as e:
logging.error(str(e))
raise HTTPException(status_code=400, detail=str(e))
Expand Down
21 changes: 15 additions & 6 deletions src/api/v1/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

import os
import importlib.util
from typing import Any
from fastapi import Response

from pandas import DataFrame
from src.sdk.python.rtdip_sdk.connectors import DatabricksSQLConnection
Expand Down Expand Up @@ -126,21 +128,28 @@ def common_api_setup_tasks( # NOSONAR


def pagination(limit_offset_parameters: LimitOffsetQueryParams, data: DataFrame):
pagination = None
pagination = PaginationRow(
limit=None,
offset=None,
next=None,
)

if (
limit_offset_parameters.limit is not None
and limit_offset_parameters.offset is not None
or limit_offset_parameters.offset is not None
):
next = None
next_offset = None

if len(data.index) == limit_offset_parameters.limit:
next = limit_offset_parameters.offset + limit_offset_parameters.limit
if (
len(data.index) == limit_offset_parameters.limit
and limit_offset_parameters.offset is not None
):
next_offset = limit_offset_parameters.offset + limit_offset_parameters.limit

pagination = PaginationRow(
limit=limit_offset_parameters.limit,
offset=limit_offset_parameters.offset,
next=next,
next=next_offset,
)

return pagination
27 changes: 15 additions & 12 deletions src/api/v1/interpolate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,17 @@

import logging
from typing import Union

from src.api.FastAPIApp import api_v1_router
from fastapi import HTTPException, Depends, Body
from fastapi import HTTPException, Depends, Body, Response
import nest_asyncio
from pandas.io.json import build_table_schema
import numpy as np
from src.sdk.python.rtdip_sdk.queries.time_series import interpolate
from src.api.v1.models import (
BaseQueryParams,
BaseHeaders,
FieldSchema,
ResampleInterpolateResponse,
PivotResponse,
HTTPError,
Expand Down Expand Up @@ -63,18 +65,19 @@ def interpolate_events_get(

data = interpolate.get(connection, parameters)

if parameters.get("pivot") == True:
return PivotResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
)
else:
return ResampleInterpolateResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
return Response(
"{"
+ '"schema":{},"data":{},"pagination":{}'.format(
FieldSchema.model_validate(
build_table_schema(data, index=False, primary_key=False),
).model_dump_json(),
data.replace({np.nan: None}).to_json(
orient="records", date_format="iso", date_unit="us"
),
pagination(limit_offset_parameters, data).model_dump_json(),
)
+ "}",
)
except Exception as e:
logging.error(str(e))
raise HTTPException(status_code=400, detail=str(e))
Expand Down
26 changes: 14 additions & 12 deletions src/api/v1/interpolation_at_time.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,14 @@
from typing import Union
import numpy as np
from src.api.FastAPIApp import api_v1_router
from fastapi import HTTPException, Depends, Body
from fastapi import HTTPException, Depends, Body, Response
import nest_asyncio
from pandas.io.json import build_table_schema
from src.sdk.python.rtdip_sdk.queries.time_series import interpolation_at_time
from src.api.v1.models import (
BaseQueryParams,
BaseHeaders,
FieldSchema,
ResampleInterpolateResponse,
PivotResponse,
HTTPError,
Expand Down Expand Up @@ -57,18 +58,19 @@ def interpolation_at_time_events_get(

data = interpolation_at_time.get(connection, parameters)

if parameters.get("pivot") == True:
return PivotResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
)
else:
return ResampleInterpolateResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
return Response(
"{"
+ '"schema":{},"data":{},"pagination":{}'.format(
FieldSchema.model_validate(
build_table_schema(data, index=False, primary_key=False),
).model_dump_json(),
data.replace({np.nan: None}).to_json(
orient="records", date_format="iso", date_unit="us"
),
pagination(limit_offset_parameters, data).model_dump_json(),
)
+ "}",
)
except Exception as e:
logging.error(str(e))
raise HTTPException(status_code=400, detail=str(e))
Expand Down
19 changes: 14 additions & 5 deletions src/api/v1/latest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,13 @@
import logging
import numpy as np
from pandas.io.json import build_table_schema
from fastapi import Query, HTTPException, Depends, Body
from fastapi import Query, HTTPException, Depends, Body, Response
import nest_asyncio
from src.sdk.python.rtdip_sdk.queries.time_series import latest
from src.api.v1.models import (
BaseQueryParams,
BaseHeaders,
FieldSchema,
MetadataQueryParams,
TagsBodyParams,
LatestResponse,
Expand All @@ -48,10 +49,18 @@ def latest_retrieval_get(

data = latest.get(connection, parameters)

return LatestResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
return Response(
"{"
+ '"schema":{},"data":{},"pagination":{}'.format(
FieldSchema.model_validate(
build_table_schema(data, index=False, primary_key=False),
).model_dump_json(),
data.replace({np.nan: None}).to_json(
orient="records", date_format="iso", date_unit="us"
),
pagination(limit_offset_parameters, data).model_dump_json(),
)
+ "}",
)
except Exception as e:
logging.error(str(e))
Expand Down
22 changes: 14 additions & 8 deletions src/api/v1/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import logging
import numpy as np
from pandas.io.json import build_table_schema
from fastapi import Query, HTTPException, Depends, Body
from fastapi import Query, HTTPException, Depends, Body, Response
import nest_asyncio
from src.sdk.python.rtdip_sdk.queries import metadata
from src.api.v1.models import (
BaseQueryParams,
BaseHeaders,
FieldSchema,
MetadataQueryParams,
TagsBodyParams,
MetadataResponse,
Expand All @@ -48,12 +47,19 @@ def metadata_retrieval_get(

data = metadata.get(connection, parameters)

return MetadataResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
return Response(
"{"
+ '"schema":{},"data":{},"pagination":{}'.format(
FieldSchema.model_validate(
build_table_schema(data, index=False, primary_key=False),
).model_dump_json(),
data.replace({np.nan: None}).to_json(
orient="records", date_format="iso", date_unit="us"
),
pagination(limit_offset_parameters, data).model_dump_json(),
)
+ "}",
)

except Exception as e:
logging.error(str(e))
raise HTTPException(status_code=400, detail=str(e))
Expand Down
22 changes: 16 additions & 6 deletions src/api/v1/raw.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,17 @@
# limitations under the License.


import json
import logging
import numpy as np
from pandas.io.json import build_table_schema
from fastapi import Query, HTTPException, Depends, Body
from fastapi import Query, HTTPException, Depends, Body, Response
import nest_asyncio
from src.sdk.python.rtdip_sdk.queries.time_series import raw, summary
from src.sdk.python.rtdip_sdk.queries.time_series import raw
from src.api.v1.models import (
BaseHeaders,
BaseQueryParams,
FieldSchema,
RawResponse,
RawQueryParams,
TagsQueryParams,
Expand Down Expand Up @@ -54,10 +56,18 @@ def raw_events_get(

data = raw.get(connection, parameters)

return RawResponse(
schema=build_table_schema(data, index=False, primary_key=False),
data=data.replace({np.nan: None}).to_dict(orient="records"),
pagination=pagination(limit_offset_parameters, data),
return Response(
"{"
+ '"schema":{},"data":{},"pagination":{}'.format(
FieldSchema.model_validate(
build_table_schema(data, index=False, primary_key=False),
).model_dump_json(),
data.replace({np.nan: None}).to_json(
orient="records", date_format="iso", date_unit="us"
),
pagination(limit_offset_parameters, data).model_dump_json(),
)
+ "}",
)
except Exception as e:
logging.error(str(e))
Expand Down
Loading