rtdip · GBBBAS · Sep 19, 2023 · Sep 19, 2023 · Sep 19, 2023
diff --git a/docs/sdk/code-reference/pipelines/transformers/spark/mqtt_json_to_pcdm.md b/docs/sdk/code-reference/pipelines/transformers/spark/mqtt_json_to_pcdm.md
diff --git a/docs/sdk/code-reference/pipelines/transformers/spark/sem_json_to_pcdm.md b/docs/sdk/code-reference/pipelines/transformers/spark/sem_json_to_pcdm.md
@@ -0,0 +1,2 @@
+# Convert SEM Json to Process Control Data Model
+::: src.sdk.python.rtdip_sdk.pipelines.transformers.spark.sem_json_to_pcdm
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -184,7 +184,7 @@ nav:
                       - OPC Publisher Json To Process Control Data Model: sdk/code-reference/pipelines/transformers/spark/opc_publisher_opcua_json_to_pcdm.md
                       - Fledge Json To Process Control Data Model: sdk/code-reference/pipelines/transformers/spark/fledge_opcua_json_to_pcdm.md
                       - EdgeX JSON data To Process Control Data Model: sdk/code-reference/pipelines/transformers/spark/edgex_opcua_json_to_pcdm.md
-                      - MQTT data To Process Control Data Model: sdk/code-reference/pipelines/transformers/spark/mqtt_json_to_pcdm.md
+                      - SEM data To Process Control Data Model: sdk/code-reference/pipelines/transformers/spark/sem_json_to_pcdm.md
                       - SSIP PI Binary File data To Process Control Data Model: sdk/code-reference/pipelines/transformers/spark/ssip_pi_binary_file_to_pcdm.md
                       - SSIP PI Binary JSON data To Process Control Data Model: sdk/code-reference/pipelines/transformers/spark/ssip_pi_binary_json_to_pcdm.md
                       - Pandas to PySpark DataFrame Conversion: sdk/code-reference/pipelines/transformers/spark/pandas_to_pyspark.md

diff --git a/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/spark.py b/src/sdk/python/rtdip_sdk/pipelines/_pipeline_utils/spark.py
@@ -276,7 +276,7 @@ def get_dbutils(
     ]
 )
 
-MQTT_SCHEMA = StructType(
+SEM_SCHEMA = StructType(
     [
         StructField("apiVersion", StringType(), True),
         StructField("deviceName", StringType(), True),

diff --git a/src/sdk/python/rtdip_sdk/pipelines/transformers/__init__.py b/src/sdk/python/rtdip_sdk/pipelines/transformers/__init__.py
@@ -25,4 +25,4 @@
 from .spark.the_weather_company.raw_forecast_to_weather_data_model import *
 from .spark.pcdm_to_honeywell_apm import *
 from .spark.honeywell_apm_to_pcdm import *
-from .spark.mqtt_json_to_pcdm import *
+from .spark.sem_json_to_pcdm import *
diff --git a/...s/transformers/spark/mqtt_json_to_pcdm.py → ...es/transformers/spark/sem_json_to_pcdm.py b/...s/transformers/spark/mqtt_json_to_pcdm.py → ...es/transformers/spark/sem_json_to_pcdm.py
@@ -26,20 +26,20 @@
     split,
     expr,
 )
-from src.sdk.python.rtdip_sdk._sdk_utils.compare_versions import (
+from ...._sdk_utils.compare_versions import (
     _package_version_meets_minimum,
 )
 from ..interfaces import TransformerInterface
 from ..._pipeline_utils.models import Libraries, SystemType
-from ..._pipeline_utils.spark import MQTT_SCHEMA
+from ..._pipeline_utils.spark import SEM_SCHEMA
 from ..._pipeline_utils import obc_field_mappings
 
 
-class MQTTJsonToPCDMTransformer(TransformerInterface):
+class SEMJsonToPCDMTransformer(TransformerInterface):
     """
-    Converts a Spark Dataframe column containing a json string created by MQTT to the Process Control Data Model
+    Converts a Spark Dataframe column containing a json string created by SEM to the Process Control Data Model
     Args:
-        data (DataFrame): Dataframe containing the column with MQTT data
+        data (DataFrame): Dataframe containing the column with SEM data
         source_column_name (str): Spark Dataframe column containing the OPC Publisher Json OPC UA data
         version (int): The version for the OBC field mappings. The latest version is 10.
         status_null_value (optional str): If populated, will replace 'Good' in the Status column with the specified value.
@@ -100,7 +100,7 @@ def transform(self) -> DataFrame:
             df = (
                 self.data.withColumn(
                     self.source_column_name,
-                    from_json(self.source_column_name, MQTT_SCHEMA),
+                    from_json(self.source_column_name, SEM_SCHEMA),
                 )
                 .select(self.source_column_name + ".readings")
                 .melt(

diff --git a/...nsformers/spark/test_mqtt_json_to_pcdm.py → ...ansformers/spark/test_sem_json_to_pcdm.py b/...nsformers/spark/test_mqtt_json_to_pcdm.py → ...ansformers/spark/test_sem_json_to_pcdm.py
@@ -15,8 +15,8 @@
 import sys
 
 sys.path.insert(0, ".")
-from src.sdk.python.rtdip_sdk.pipelines.transformers.spark.mqtt_json_to_pcdm import (
-    MQTTJsonToPCDMTransformer,
+from src.sdk.python.rtdip_sdk.pipelines.transformers.spark.sem_json_to_pcdm import (
+    SEMJsonToPCDMTransformer,
 )
 from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
     Libraries,
@@ -32,9 +32,9 @@
 )
 
 
-def test_mqtt_json_to_pcdm(spark_session: SparkSession):
-    mqtt_json_data = '{"readings":[{"resourceName":"d","value":"[1685025760.46]"},{"resourceName":"dID","value":"502"},{"resourceName":"t", "value":"1695047439192"}]}'
-    mqtt_df: DataFrame = spark_session.createDataFrame([{"body": mqtt_json_data}])
+def test_sem_json_to_pcdm(spark_session: SparkSession):
+    sem_json_data = '{"readings":[{"resourceName":"d","value":"[1685025760.46]"},{"resourceName":"dID","value":"502"},{"resourceName":"t", "value":"1695047439192"}]}'
+    sem_df: DataFrame = spark_session.createDataFrame([{"body": sem_json_data}])
 
     expected_schema = StructType(
         [
@@ -64,17 +64,17 @@ def test_mqtt_json_to_pcdm(spark_session: SparkSession):
 
     try:
         if _package_version_meets_minimum("pyspark", "3.4.0"):
-            mqtt_json_to_pcdm_transformer = MQTTJsonToPCDMTransformer(
-                data=mqtt_df, source_column_name="body", version=10
+            sem_json_to_pcdm_transformer = SEMJsonToPCDMTransformer(
+                data=sem_df, source_column_name="body", version=10
             )
-            actual_df = mqtt_json_to_pcdm_transformer.transform()
+            actual_df = sem_json_to_pcdm_transformer.transform()
 
-            assert mqtt_json_to_pcdm_transformer.system_type() == SystemType.PYSPARK
-            assert isinstance(mqtt_json_to_pcdm_transformer.libraries(), Libraries)
+            assert sem_json_to_pcdm_transformer.system_type() == SystemType.PYSPARK
+            assert isinstance(sem_json_to_pcdm_transformer.libraries(), Libraries)
             assert expected_schema == actual_df.schema
             assert expected_df.collect() == actual_df.collect()
     except:
         with pytest.raises(Exception):
-            mqtt_json_to_pcdm_transformer = MQTTJsonToPCDMTransformer(
-                data=mqtt_df, source_column_name="body", version=10
+            sem_json_to_pcdm_transformer = SEMJsonToPCDMTransformer(
+                data=sem_df, source_column_name="body", version=10
             )
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# Convert SEM Json to Process Control Data Model
		::: src.sdk.python.rtdip_sdk.pipelines.transformers.spark.sem_json_to_pcdm