rtdip · GBBBAS · Aug 17, 2023 · Aug 17, 2023 · Aug 17, 2023 · Aug 17, 2023
diff --git a/docs/domains/smart_meter/data_model.md b/docs/domains/smart_meter/data_model.md
@@ -1,7 +1,7 @@
 # Meters Data Model
 
-Base Raw To Meters Data Model: [here](/rtdip/core/sdk/code-reference/pipelines/transformers/spark/base_raw_to_mdm/)
+Base Raw To Meters Data Model: [here](../../sdk/code-reference/pipelines/transformers/spark/base_raw_to_mdm.md) 
 
 * ISO:
-    * MISO To Meters Data Model: [here](/rtdip/core/sdk/code-reference/pipelines/transformers/spark/iso/miso_to_mdm/)
-    * PJM  To Meters Data Model: [here](/rtdip/core/sdk/code-reference/pipelines/transformers/spark/iso/pjm_to_mdm/)
+    * MISO To Meters Data Model: [here](../../sdk/code-reference/pipelines/transformers/spark/iso/miso_to_mdm.md)
+    * PJM  To Meters Data Model: [here](../../sdk/code-reference/pipelines/transformers/spark/iso/pjm_to_mdm.md)
diff --git a/docs/domains/smart_meter/overview.md b/docs/domains/smart_meter/overview.md
@@ -17,7 +17,7 @@ At a high level, the electricity system (US example) works as follows:
 An **Independent System Operator (ISO)** sometimes called the Regional Transmission Organisation (RTO) is an organisation that is in charge of the entire process. They coordinate, control, and monitor the electric grid in a specific region, typically a multi-state area.
 
 ## Meter Data Pipelines
-Load forecasting is a technique used by ISO's, and energy-providing companies to predict the power/energy needed to meet the demand and supply equilibrium of the energy grid. RTDIP defines and provides example pipelines for the two primary inputs to energy services like load forecasting, namely [weather](/rtdip/core/domains/weather/overview/) and meter data.
+Load forecasting is a technique used by ISO's, and energy-providing companies to predict the power/energy needed to meet the demand and supply equilibrium of the energy grid. RTDIP defines and provides example pipelines for the two primary inputs to energy services like load forecasting, namely [weather](../../domains/weather/overview.md) and meter data.
 
 Specifically, with respect to meter data RTDIP defines and provides two exemplar ISO's:
 

diff --git a/docs/domains/weather/overview.md b/docs/domains/weather/overview.md
@@ -1,7 +1,7 @@
 
 # Weather Services 
 ## Overview
-Many organizations need weather data for day-to-day operations. RTDIP provides the ability to consume data from examplar weather sources, transform it and store the data in an appropiate open source format to enable generic functions such as:
+Many organizations need weather data for day-to-day operations. RTDIP provides the ability to consume data from examplar weather sources, transform it and store the data in an appropriate open source format to enable generic functions such as:
 
 * Data Science, ML and AI applications to consume the data
 * BI and Analytics
@@ -13,7 +13,7 @@ A primary aim for RTDIP in 2023 is to demonstrate how the platform can be utilis
 * Energy Generation Forecasting
 * Other behind the meter services and insights
 
-Weather data is a primary driver,  together with [meter](/rtdip/core/domains/smart_meter/overview/) data, of variance in load & generation forecasting in the energy domain. 
+Weather data is a primary driver,  together with [meter](../../domains/smart_meter/overview.md) data, of variance in load & generation forecasting in the energy domain. 
 
 ## Weather Data in the Energy Domain
 

diff --git a/docs/sdk/code-reference/query/query_builder.md b/docs/sdk/code-reference/query/query_builder.md
@@ -0,0 +1,3 @@
+# Query Builder
+::: src.sdk.python.rtdip_sdk.queries.query_builder
+
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -228,6 +228,7 @@ nav:
               - Deploy:
                   - Databricks: sdk/code-reference/pipelines/deploy/databricks.md
           - Queries:
+              - Query Builder: sdk/code-reference/query/query_builder.md
               - Functions:
                   - Resample: sdk/code-reference/query/resample.md
                   - Interpolate: sdk/code-reference/query/interpolate.md

diff --git a/src/sdk/python/rtdip_sdk/queries/__init__.py b/src/sdk/python/rtdip_sdk/queries/__init__.py
@@ -12,4 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .time_series import raw, resample, interpolate, interpolation_at_time, time_weighted_average
+from .time_series import raw, resample, interpolate, interpolation_at_time, time_weighted_average, circular_standard_deviation, circular_average
+from .query_builder import *
diff --git a/src/sdk/python/rtdip_sdk/queries/query_builder.py b/src/sdk/python/rtdip_sdk/queries/query_builder.py
diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/_query_builder.py b/src/sdk/python/rtdip_sdk/queries/time_series/_query_builder.py
diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/time_weighted_average.py b/src/sdk/python/rtdip_sdk/queries/time_series/time_weighted_average.py
@@ -17,7 +17,7 @@
 
 def get(connection: object, parameters_dict: dict) -> pd.DataFrame:
     '''
-    A function that receives a dataframe of raw tag data and performs a timeweighted average, returning the results. 
+    A function that receives a dataframe of raw tag data and performs a time weighted averages, returning the results. 
 
     This function requires the input of a pandas dataframe acquired via the rtdip.functions.raw() method and the user to input a dictionary of parameters. (See Attributes table below)
 

diff --git a/tests/docs/test_mkdocs_build.py b/tests/docs/test_mkdocs_build.py
diff --git a/tests/sdk/python/rtdip_sdk/queries/test_circular_average.py b/tests/sdk/python/rtdip_sdk/queries/test_circular_average.py
@@ -27,9 +27,9 @@
 ACCESS_TOKEN = "mock_databricks_token"
 DATABRICKS_SQL_CONNECT = 'databricks.sql.connect'
 DATABRICKS_SQL_CONNECT_CURSOR = 'databricks.sql.connect.cursor'
-MOCKED_QUERY= 'WITH raw_events AS (SELECT EventTime,TagName ,Status ,Value FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE EventDate BETWEEN TO_DATE(to_timestamp("2011-01-01T00:00:00+00:00")) AND TO_DATE(to_timestamp("2011-01-02T23:59:59+00:00")) AND EventTime BETWEEN TO_TIMESTAMP("2011-01-01T00:00:00+00:00") AND TO_TIMESTAMP("2011-01-02T23:59:59+00:00") AND TagName IN (\'MOCKED-TAGNAME\') ) ,date_array AS (SELECT EXPLODE(SEQUENCE(FROM_UTC_TIMESTAMP(TO_TIMESTAMP("2011-01-01T00:00:00+00:00"), "+0000"), FROM_UTC_TIMESTAMP(TO_TIMESTAMP("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS EventTime, EXPLODE(ARRAY(\'MOCKED-TAGNAME\')) AS TagName)  ,window_events AS (SELECT COALESCE(a.TagName, b.TagName) AS TagName, COALESCE(a.EventTime, b.EventTime) AS EventTime, WINDOW(COALESCE(a.EventTime, b.EventTime), \'15 minute\').START WindowEventTime, b.Status, b.Value FROM date_array a FULL OUTER JOIN raw_events b ON CAST(a.EventTime AS LONG) = CAST(b.EventTime AS LONG) AND a.TagName = b.TagName) ,calculation_set_up AS (SELECT EventTime ,WindowEventTime ,TagName ,Value ,MOD(Value- 0, (360 - 0))*(2*pi()/(360 - 0)) as Value_in_Radians ,LAG(EventTime) OVER (PARTITION BY TagName ORDER BY EventTime) AS Previous_EventTime ,(unix_millis(EventTime) - unix_millis(Previous_EventTime)) / 86400000 AS Time_Difference ,COS(Value_in_Radians) as Cos_Value ,SIN(Value_in_Radians) as Sin_Value FROM window_events) ,circular_average_calculations AS (SELECT WindowEventTime ,TagName ,Time_Difference ,AVG(Cos_Value) OVER (PARTITION BY TagName ORDER BY EventTime ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS Average_Cos ,AVG(Sin_Value) OVER (PARTITION BY TagName ORDER BY EventTime ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS Average_Sin ,SQRT(POW(Average_Cos, 2) + POW(Average_Sin, 2)) AS Vector_Length ,Average_Cos/Vector_Length AS Rescaled_Average_Cos ,Average_Sin/Vector_Length AS Rescaled_Average_Sin ,Time_Difference * Rescaled_Average_Cos AS Diff_Average_Cos ,Time_Difference * Rescaled_Average_Sin AS Diff_Average_Sin FROM calculation_set_up)  ,project_circular_average_results AS (SELECT WindowEventTime AS EventTime ,TagName ,sum(Diff_Average_Cos)/sum(Time_Difference) AS Cos_Time_Averages ,sum(Diff_Average_Sin)/sum(Time_Difference) AS Sin_Time_Averages ,array_min(array(1, sqrt(pow(Cos_Time_Averages, 2) + pow(Sin_Time_Averages, 2)))) AS R ,mod(2*pi() + atan2(Sin_Time_Averages, Cos_Time_Averages), 2*pi()) AS Circular_Average_Value_in_Radians ,(Circular_Average_Value_in_Radians * (360 - 0)) / (2*pi())+ 0 AS Circular_Average_Value_in_Degrees FROM circular_average_calculations GROUP BY TagName, WindowEventTime) SELECT EventTime ,TagName ,Circular_Average_Value_in_Degrees AS Value FROM project_circular_average_results ORDER BY TagName, EventTime '
+MOCKED_QUERY= 'WITH raw_events AS (SELECT `EventTime`, `TagName`,  `Status`,  `Value` FROM `mocked-business-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN TO_TIMESTAMP("2011-01-01T00:00:00+00:00") AND TO_TIMESTAMP("2011-01-02T23:59:59+00:00") AND `TagName` IN (\'MOCKED-TAGNAME\') ) ,date_array AS (SELECT EXPLODE(SEQUENCE(FROM_UTC_TIMESTAMP(TO_TIMESTAMP("2011-01-01T00:00:00+00:00"), "+0000"), FROM_UTC_TIMESTAMP(TO_TIMESTAMP("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS `EventTime`, EXPLODE(ARRAY(\'MOCKED-TAGNAME\')) AS `TagName`)  ,window_events AS (SELECT COALESCE(a.`TagName`, b.`TagName`) AS `TagName`, COALESCE(a.`EventTime`, b.`EventTime`) AS `EventTime`, WINDOW(COALESCE(a.`EventTime`, b.`EventTime`), \'15 minute\').START `WindowEventTime`, b.`Status`, b.`Value` FROM date_array a FULL OUTER JOIN raw_events b ON CAST(a.`EventTime` AS LONG) = CAST(b.`EventTime` AS LONG) AND a.`TagName` = b.`TagName`) ,calculation_set_up AS (SELECT `EventTime`, `WindowEventTime`, `TagName`, `Value`, MOD(`Value` - 0, (360 - 0))*(2*pi()/(360 - 0)) as `Value_in_Radians`, LAG(`EventTime`) OVER (PARTITION BY `TagName` ORDER BY `EventTime`) AS `Previous_EventTime`, (unix_millis(`EventTime`) - unix_millis(`Previous_EventTime`)) / 86400000 AS Time_Difference, COS(`Value_in_Radians`) as Cos_Value, SIN(`Value_in_Radians`) as Sin_Value FROM window_events) ,circular_average_calculations AS (SELECT `WindowEventTime`, `TagName`, Time_Difference, AVG(Cos_Value) OVER (PARTITION BY `TagName` ORDER BY `EventTime` ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS Average_Cos, AVG(Sin_Value) OVER (PARTITION BY `TagName` ORDER BY `EventTime` ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS Average_Sin, SQRT(POW(Average_Cos, 2) + POW(Average_Sin, 2)) AS Vector_Length, Average_Cos/Vector_Length AS Rescaled_Average_Cos, Average_Sin/Vector_Length AS Rescaled_Average_Sin, Time_Difference * Rescaled_Average_Cos AS Diff_Average_Cos, Time_Difference * Rescaled_Average_Sin AS Diff_Average_Sin FROM calculation_set_up)  ,project_circular_average_results AS (SELECT `WindowEventTime` AS `EventTime`, `TagName`, sum(Diff_Average_Cos)/sum(Time_Difference) AS Cos_Time_Averages, sum(Diff_Average_Sin)/sum(Time_Difference) AS Sin_Time_Averages, array_min(array(1, sqrt(pow(Cos_Time_Averages, 2) + pow(Sin_Time_Averages, 2)))) AS R, mod(2*pi() + atan2(Sin_Time_Averages, Cos_Time_Averages), 2*pi()) AS Circular_Average_Value_in_Radians, (Circular_Average_Value_in_Radians * (360 - 0)) / (2*pi())+ 0 AS Circular_Average_Value_in_Degrees FROM circular_average_calculations GROUP BY `TagName`, `WindowEventTime`) SELECT `EventTime`, `TagName`, Circular_Average_Value_in_Degrees AS `Value` FROM project_circular_average_results ORDER BY `TagName`, `EventTime` '
 MOCKED_PARAMETER_DICT = {
-        "business_unit": "mocked-buiness-unit",
+        "business_unit": "mocked-business-unit",
         "region": "mocked-region",
         "asset": "mocked-asset",
         "data_security_level": "mocked-data-security-level",

diff --git a/tests/sdk/python/rtdip_sdk/queries/test_circular_standard_deviation.py b/tests/sdk/python/rtdip_sdk/queries/test_circular_standard_deviation.py
@@ -27,7 +27,7 @@
 ACCESS_TOKEN = "mock_databricks_token"
 DATABRICKS_SQL_CONNECT = 'databricks.sql.connect'
 DATABRICKS_SQL_CONNECT_CURSOR = 'databricks.sql.connect.cursor'
-MOCKED_QUERY= 'WITH raw_events AS (SELECT EventTime,TagName ,Status ,Value FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE EventDate BETWEEN TO_DATE(to_timestamp("2011-01-01T00:00:00+00:00")) AND TO_DATE(to_timestamp("2011-01-02T23:59:59+00:00")) AND EventTime BETWEEN TO_TIMESTAMP("2011-01-01T00:00:00+00:00") AND TO_TIMESTAMP("2011-01-02T23:59:59+00:00") AND TagName IN (\'MOCKED-TAGNAME\') ) ,date_array AS (SELECT EXPLODE(SEQUENCE(FROM_UTC_TIMESTAMP(TO_TIMESTAMP("2011-01-01T00:00:00+00:00"), "+0000"), FROM_UTC_TIMESTAMP(TO_TIMESTAMP("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS EventTime, EXPLODE(ARRAY(\'MOCKED-TAGNAME\')) AS TagName)  ,window_events AS (SELECT COALESCE(a.TagName, b.TagName) AS TagName, COALESCE(a.EventTime, b.EventTime) AS EventTime, WINDOW(COALESCE(a.EventTime, b.EventTime), \'15 minute\').START WindowEventTime, b.Status, b.Value FROM date_array a FULL OUTER JOIN raw_events b ON CAST(a.EventTime AS LONG) = CAST(b.EventTime AS LONG) AND a.TagName = b.TagName) ,calculation_set_up AS (SELECT EventTime ,WindowEventTime ,TagName ,Value ,MOD(Value- 0, (360 - 0))*(2*pi()/(360 - 0)) as Value_in_Radians ,LAG(EventTime) OVER (PARTITION BY TagName ORDER BY EventTime) AS Previous_EventTime ,(unix_millis(EventTime) - unix_millis(Previous_EventTime)) / 86400000 AS Time_Difference ,COS(Value_in_Radians) as Cos_Value ,SIN(Value_in_Radians) as Sin_Value FROM window_events) ,circular_average_calculations AS (SELECT WindowEventTime ,TagName ,Time_Difference ,AVG(Cos_Value) OVER (PARTITION BY TagName ORDER BY EventTime ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS Average_Cos ,AVG(Sin_Value) OVER (PARTITION BY TagName ORDER BY EventTime ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS Average_Sin ,SQRT(POW(Average_Cos, 2) + POW(Average_Sin, 2)) AS Vector_Length ,Average_Cos/Vector_Length AS Rescaled_Average_Cos ,Average_Sin/Vector_Length AS Rescaled_Average_Sin ,Time_Difference * Rescaled_Average_Cos AS Diff_Average_Cos ,Time_Difference * Rescaled_Average_Sin AS Diff_Average_Sin FROM calculation_set_up)  ,project_circular_average_results AS (SELECT WindowEventTime AS EventTime ,TagName ,sum(Diff_Average_Cos)/sum(Time_Difference) AS Cos_Time_Averages ,sum(Diff_Average_Sin)/sum(Time_Difference) AS Sin_Time_Averages ,array_min(array(1, sqrt(pow(Cos_Time_Averages, 2) + pow(Sin_Time_Averages, 2)))) AS R ,mod(2*pi() + atan2(Sin_Time_Averages, Cos_Time_Averages), 2*pi()) AS Circular_Average_Value_in_Radians ,SQRT(-2*LN(R)) * ( 360 - 0) / (2*PI()) AS Circular_Standard_Deviation FROM circular_average_calculations GROUP BY TagName, WindowEventTime) SELECT EventTime ,TagName , Circular_Standard_Deviation AS Value FROM project_circular_average_results ORDER BY TagName, EventTime '
+MOCKED_QUERY= 'WITH raw_events AS (SELECT `EventTime`, `TagName`,  `Status`,  `Value` FROM `mocked-buiness-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN TO_TIMESTAMP("2011-01-01T00:00:00+00:00") AND TO_TIMESTAMP("2011-01-02T23:59:59+00:00") AND `TagName` IN (\'MOCKED-TAGNAME\') ) ,date_array AS (SELECT EXPLODE(SEQUENCE(FROM_UTC_TIMESTAMP(TO_TIMESTAMP("2011-01-01T00:00:00+00:00"), "+0000"), FROM_UTC_TIMESTAMP(TO_TIMESTAMP("2011-01-02T23:59:59+00:00"), "+0000"), INTERVAL \'15 minute\')) AS `EventTime`, EXPLODE(ARRAY(\'MOCKED-TAGNAME\')) AS `TagName`)  ,window_events AS (SELECT COALESCE(a.`TagName`, b.`TagName`) AS `TagName`, COALESCE(a.`EventTime`, b.`EventTime`) AS `EventTime`, WINDOW(COALESCE(a.`EventTime`, b.`EventTime`), \'15 minute\').START `WindowEventTime`, b.`Status`, b.`Value` FROM date_array a FULL OUTER JOIN raw_events b ON CAST(a.`EventTime` AS LONG) = CAST(b.`EventTime` AS LONG) AND a.`TagName` = b.`TagName`) ,calculation_set_up AS (SELECT `EventTime`, `WindowEventTime`, `TagName`, `Value`, MOD(`Value` - 0, (360 - 0))*(2*pi()/(360 - 0)) as `Value_in_Radians`, LAG(`EventTime`) OVER (PARTITION BY `TagName` ORDER BY `EventTime`) AS `Previous_EventTime`, (unix_millis(`EventTime`) - unix_millis(`Previous_EventTime`)) / 86400000 AS Time_Difference, COS(`Value_in_Radians`) as Cos_Value, SIN(`Value_in_Radians`) as Sin_Value FROM window_events) ,circular_average_calculations AS (SELECT `WindowEventTime`, `TagName`, Time_Difference, AVG(Cos_Value) OVER (PARTITION BY `TagName` ORDER BY `EventTime` ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS Average_Cos, AVG(Sin_Value) OVER (PARTITION BY `TagName` ORDER BY `EventTime` ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS Average_Sin, SQRT(POW(Average_Cos, 2) + POW(Average_Sin, 2)) AS Vector_Length, Average_Cos/Vector_Length AS Rescaled_Average_Cos, Average_Sin/Vector_Length AS Rescaled_Average_Sin, Time_Difference * Rescaled_Average_Cos AS Diff_Average_Cos, Time_Difference * Rescaled_Average_Sin AS Diff_Average_Sin FROM calculation_set_up)  ,project_circular_average_results AS (SELECT `WindowEventTime` AS `EventTime`, `TagName`, sum(Diff_Average_Cos)/sum(Time_Difference) AS Cos_Time_Averages, sum(Diff_Average_Sin)/sum(Time_Difference) AS Sin_Time_Averages, array_min(array(1, sqrt(pow(Cos_Time_Averages, 2) + pow(Sin_Time_Averages, 2)))) AS R, mod(2*pi() + atan2(Sin_Time_Averages, Cos_Time_Averages), 2*pi()) AS Circular_Average_Value_in_Radians, SQRT(-2*LN(R)) * ( 360 - 0) / (2*PI()) AS Circular_Standard_Deviation FROM circular_average_calculations GROUP BY `TagName`, `WindowEventTime`) SELECT `EventTime`, `TagName`, Circular_Standard_Deviation AS `Value` FROM project_circular_average_results ORDER BY `TagName`, `EventTime` '
 MOCKED_PARAMETER_DICT = {
         "business_unit": "mocked-buiness-unit",
         "region": "mocked-region",
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Query Builder
		::: src.sdk.python.rtdip_sdk.queries.query_builder