Merge branch 'main' into misc-rearrange-pipeline-action-dropdown-items

chandrasekharan-zipstack · chandrasekharan-zipstack · commit 6d65ed402f88 · 2025-08-21T17:35:30.000+05:30
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
@@ -126,6 +126,7 @@ def get(
                 )
             if not enable_highlight:
                 response.remove_result_metadata_keys(["highlight_data"])
+                response.remove_result_metadata_keys(["extracted_text"])
             if not include_metadata:
                 response.remove_result_metadata_keys()
             if not include_metrics:
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
@@ -218,6 +218,7 @@ def execute_workflow(
                 )
             if not enable_highlight:
                 result.remove_result_metadata_keys(["highlight_data"])
+                result.remove_result_metadata_keys(["extracted_text"])
             if not include_metadata:
                 result.remove_result_metadata_keys()
             if not include_metrics:
diff --git a/backend/sample.env b/backend/sample.env
@@ -78,9 +78,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
 
 # Structure Tool Image (Runs prompt studio exported tools)
 # https://hub.docker.com/r/unstract/tool-structure
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.84"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.85"
 STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.84"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.85"
 
 # Feature Flags
 EVALUATION_SERVER_IP=unstract-flipt
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
@@ -532,14 +532,22 @@ def get_tool_execution_result_from_metadata(
         return result
 
     def has_valid_metadata(self, metadata: Any) -> bool:
-        # Check if metadata is not None and metadata is a non-empty string
-        if not metadata:
+        # Check if metadata is not None and is either a non-empty dict or valid string
+        if metadata is None:
             return False
-        if not isinstance(metadata, str):
-            return False
-        if metadata.strip().lower() == "none":
-            return False
-        return True
+
+        # Handle dict metadata (which is valid and contains extracted_text)
+        if isinstance(metadata, dict):
+            return bool(metadata)  # Return True if dict is not empty
+
+        # Handle string metadata
+        if isinstance(metadata, str):
+            if metadata.strip().lower() == "none" or not metadata.strip():
+                return False
+            return True
+
+        # For other types, consider them valid if they're truthy
+        return bool(metadata)
 
     def get_metadata(
         self, file_history: FileHistory | None = None
@@ -555,7 +563,6 @@ def get_metadata(
             else:
                 return None
         metadata: dict[str, Any] = self.get_workflow_metadata()
-
         return metadata
 
     def delete_file_execution_directory(self) -> None:
@@ -777,6 +784,9 @@ def _push_to_queue(
             q_name = self._get_review_queue_name()
             whisper_hash = meta_data.get("whisper-hash") if meta_data else None
 
+            # Get extracted text from metadata (added by structure tool)
+            extracted_text = meta_data.get("extracted_text") if meta_data else None
+
             queue_result = QueueResult(
                 file=file_name,
                 status=QueueResultStatus.SUCCESS,
@@ -785,6 +795,7 @@ def _push_to_queue(
                 file_content=file_content_base64,
                 whisper_hash=whisper_hash,
                 file_execution_id=file_execution_id,
+                extracted_text=extracted_text,
             ).to_dict()
 
             queue_result_json = json.dumps(queue_result)
@@ -811,6 +822,9 @@ def _push_to_queue(
             else:
                 whisper_hash = None
 
+            # Get extracted text from metadata (added by structure tool)
+            extracted_text = meta_data.get("extracted_text") if meta_data else None
+
             # Create QueueResult with TTL metadata
             queue_result_obj = QueueResult(
                 file=file_name,
@@ -820,6 +834,7 @@ def _push_to_queue(
                 file_content=file_content_base64,
                 whisper_hash=whisper_hash,
                 file_execution_id=file_execution_id,
+                extracted_text=extracted_text,
             )
 
             # Add TTL metadata based on HITLSettings
diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.py
@@ -97,6 +97,7 @@ class QueueResult:
     file_execution_id: str | None = None
     enqueued_at: float | None = None
     ttl_seconds: int | None = None
+    extracted_text: str | None = None
 
     def __post_init__(self):
         """Initialize enqueued_at timestamp if not provided and validate required fields"""
@@ -122,5 +123,6 @@ def to_dict(self) -> Any:
             "file_execution_id": self.file_execution_id,
             "enqueued_at": self.enqueued_at,
             "ttl_seconds": self.ttl_seconds,
+            "extracted_text": self.extracted_text,
         }
         return result_dict
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
   "schemaVersion": "0.0.1",
   "displayName": "Structure Tool",
   "functionName": "structure_tool",
-  "toolVersion": "0.0.84",
+  "toolVersion": "0.0.85",
   "description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
   "input": {
     "description": "File that needs to be indexed and parsed for answers"
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
@@ -325,11 +325,25 @@ def run(
             )
 
         # HACK: Replacing actual file's name instead of INFILE
-        if SettingsKeys.METADATA in structured_output:
-            structured_output[SettingsKeys.METADATA][SettingsKeys.FILE_NAME] = (
-                self.source_file_name
-            )
+        # Ensure metadata section exists
+        if SettingsKeys.METADATA not in structured_output:
+            structured_output[SettingsKeys.METADATA] = {}
+            self.stream_log("Created metadata section in structured_output")
+
+        structured_output[SettingsKeys.METADATA][SettingsKeys.FILE_NAME] = (
+            self.source_file_name
+        )
 
+        # Add extracted text for HITL raw view
+        if extracted_text:
+            structured_output[SettingsKeys.METADATA]["extracted_text"] = extracted_text
+            self.stream_log(
+                f"Added text extracted from the document to metadata (length: {len(extracted_text)} characters)"
+            )
+        else:
+            self.stream_log(
+                "No text is extracted from the document to add to the metadata"
+            )
         if merged_metrics := self._merge_metrics(
             structured_output.get(SettingsKeys.METRICS, {}), index_metrics
         ):

Original file line number	Diff line number	Diff line change
`@@ -126,6 +126,7 @@ def get(`
`126`	`126`	`)`
`127`	`127`	`if not enable_highlight:`
`128`	`128`	`response.remove_result_metadata_keys(["highlight_data"])`
	`129`	`+ response.remove_result_metadata_keys(["extracted_text"])`
`129`	`130`	`if not include_metadata:`
`130`	`131`	`response.remove_result_metadata_keys()`
`131`	`132`	`if not include_metrics:`
Original file line number	Diff line number	Diff line change
`@@ -218,6 +218,7 @@ def execute_workflow(`
`218`	`218`	`)`
`219`	`219`	`if not enable_highlight:`
`220`	`220`	`result.remove_result_metadata_keys(["highlight_data"])`
	`221`	`+ result.remove_result_metadata_keys(["extracted_text"])`
`221`	`222`	`if not include_metadata:`
`222`	`223`	`result.remove_result_metadata_keys()`
`223`	`224`	`if not include_metrics:`