From 20ebeb753d8332629533d11e87be055387329d1e Mon Sep 17 00:00:00 2001 From: Joel Bryan Juliano Date: Sun, 7 Dec 2025 18:13:47 +0100 Subject: [PATCH] add self-correcting JSON in case of invalid structure; update README --- README.md | 29 +- docs/index.md | 29 +- pkg/resolver/chat_decoder_test.go | 10 +- pkg/resolver/resource_chat.go | 291 +++++++++++++++---- pkg/resolver/resource_chat_tool_processor.go | 30 +- pkg/resolver/tool_processor_test.go | 14 +- 6 files changed, 318 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 6212da3..15162fd 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,35 @@ [![tests](https://img.shields.io/endpoint?style=flat-square&url=https://gist.githubusercontent.com/jjuliano/ce695f832cd51d014ae6d37353311c59/raw/kdeps-go-tests.json)](https://github.com/kdeps/kdeps/actions/workflows/build-test.yml) [![coverage](https://img.shields.io/endpoint?style=flat-square&url=https://gist.githubusercontent.com/jjuliano/ce695f832cd51d014ae6d37353311c59/raw/kdeps-go-coverage.json)](https://github.com/kdeps/kdeps/actions/workflows/build-test.yml) -KDeps is an all-in-one, offline-ready AI framework for building Dockerized full-stack applications with declarative PKL configuration, -featuring integrated open-source LLMs for AI-powered APIs and workflows. Runs fully local with no external AI APIs required. +KDeps is a framework that packages everything needed for RAG and AI agents in a single Dockerized image, eliminating the complexity of building self-hosted APIs with open-source LLMs. Instead of juggling multiple tools and dependencies, you can use KDeps to run Python scripts in isolated environments, execute custom shell commands, integrate with external APIs, and leverage endless opinionated LLM combinations and configurations—all while maintaining control over your infrastructure. + +The framework uses atomic configurations and a graph-based dependency workflow for orchestrating resources, with built-in support for multimodal LLMs, making it particularly appealing for teams looking to avoid vendor lock-in or subscription costs. > 📋 **New**: Read our comprehensive [**KDeps Whitepaper**](./docs/KDeps_Whitepaper.md) for detailed technical insights, architecture overview, and competitive analysis. +## Key Highlights + +### Atomic Configurations +Build AI agents using small, self-contained configuration blocks that can be combined and reused. Each resource is an atomic unit with its own dependencies, validations, and logic—making it easy to compose complex workflows from simple, maintainable pieces. + +### Endless LLM Combinations +Mix and match different open-source LLMs within a single workflow. Use vision models for image analysis, small models for fast responses, and larger models for complex reasoning—all configured declaratively. Create opinionated LLM pipelines tailored to your specific use case without being locked into a single provider or model. + +### Docker-First Development +Package everything your RAG AI agent needs into a single Docker image—LLMs, dependencies, scripts, and workflows. Run locally during development, then deploy the same container to any environment without modification. No need to manage multiple systems or complex setups. + +### Graph-Based Workflow Engine +Build complex AI agent logic using a dependency-driven workflow system. Chain together different components like LLM calls, scripts, and API requests while the framework automatically handles the execution order and data flow between them. + +### Mix-and-Match Components +Run Python scripts in isolated Anaconda environments, execute shell commands, make HTTP requests, and interact with LLMs—all orchestrated through a unified workflow. Resources can be shared and remixed between different AI agents, promoting code reuse. + +### Production-Ready Features +Built-in support for structured JSON outputs, file uploads, and multimodal LLM interactions. The framework includes preflight validations, skip conditions, and custom error handling to help you build reliable AI agents. API routes can be defined with granular control over HTTP methods and request handling. + ## About the name -> “KDeps, short for ‘knowledge dependencies,’ is inspired by the principle that knowledge—whether from AI, machines, or humans—can be represented, organized, orchestrated, and interacted with through graph-based systems. The name grew out of my work on Kartographer, a lightweight graph library for organizing and interacting with information. KDeps builds on Kartographer’s foundation and serves as a RAG-first (Retrieval-Augmented Generation) AI agent framework.” — Joel Bryan Juliano, KDeps creator +> "KDeps, short for 'knowledge dependencies,' is inspired by the principle that knowledge—whether from AI, machines, or humans—can be represented, organized, orchestrated, and interacted with through graph-based systems. The name grew out of my work on Kartographer, a lightweight graph library for organizing and interacting with information. KDeps builds on Kartographer's foundation and serves as a RAG-first (Retrieval-Augmented Generation) AI agent framework." — Joel Bryan Juliano, KDeps creator ## Why Offline-First? @@ -27,7 +48,7 @@ featuring integrated open-source LLMs for AI-powered APIs and workflows. Runs fu - **Control and independence**: Avoid vendor lock-in and ensure reproducible, auditable deployments. - **Data residency**: Run on-premises or at the edge to meet jurisdictional requirements. - **Security**: Reduce external attack surface by eliminating third-party AI API dependencies. -- **Edge readiness**: Process data close to where it’s generated for real-time use cases. +- **Edge readiness**: Process data close to where it's generated for real-time use cases. - **Developer productivity**: Fully local dev loop; everything runs in self-contained Docker images. KDeps enables offline-first by integrating open-source LLMs via Ollama and packaging complete applications (FE/BE, models, and runtimes) into Docker images—no external AI APIs required. diff --git a/docs/index.md b/docs/index.md index e7bd796..6fbeb42 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,14 +2,35 @@

-KDeps is an all-in-one, offline-ready AI framework for building Dockerized full-stack applications with declarative PKL configuration, -featuring integrated open-source LLMs for AI-powered APIs and workflows. Runs fully local with no external AI APIs required. +KDeps is a framework that packages everything needed for RAG and AI agents in a single Dockerized image, eliminating the complexity of building self-hosted APIs with open-source LLMs. Instead of juggling multiple tools and dependencies, you can use KDeps to run Python scripts in isolated environments, execute custom shell commands, integrate with external APIs, and leverage endless opinionated LLM combinations and configurations—all while maintaining control over your infrastructure. + +The framework uses atomic configurations and a graph-based dependency workflow for orchestrating resources, with built-in support for multimodal LLMs, making it particularly appealing for teams looking to avoid vendor lock-in or subscription costs. > 📋 **New**: Read our comprehensive [**KDeps Whitepaper**](./KDeps_Whitepaper.md) for detailed technical insights, architecture overview, and competitive analysis. +## Key Highlights + +### Atomic Configurations +Build AI agents using small, self-contained configuration blocks that can be combined and reused. Each resource is an atomic unit with its own dependencies, validations, and logic—making it easy to compose complex workflows from simple, maintainable pieces. + +### Endless LLM Combinations +Mix and match different open-source LLMs within a single workflow. Use vision models for image analysis, small models for fast responses, and larger models for complex reasoning—all configured declaratively. Create opinionated LLM pipelines tailored to your specific use case without being locked into a single provider or model. + +### Docker-First Development +Package everything your RAG AI agent needs into a single Docker image—LLMs, dependencies, scripts, and workflows. Run locally during development, then deploy the same container to any environment without modification. No need to manage multiple systems or complex setups. + +### Graph-Based Workflow Engine +Build complex AI agent logic using a dependency-driven workflow system. Chain together different components like LLM calls, scripts, and API requests while the framework automatically handles the execution order and data flow between them. + +### Mix-and-Match Components +Run Python scripts in isolated Anaconda environments, execute shell commands, make HTTP requests, and interact with LLMs—all orchestrated through a unified workflow. Resources can be shared and remixed between different AI agents, promoting code reuse. + +### Production-Ready Features +Built-in support for structured JSON outputs, file uploads, and multimodal LLM interactions. The framework includes preflight validations, skip conditions, and custom error handling to help you build reliable AI agents. API routes can be defined with granular control over HTTP methods and request handling. + ## About the name -> “KDeps, short for ‘knowledge dependencies,’ is inspired by the principle that knowledge—whether from AI, machines, or humans—can be represented, organized, orchestrated, and interacted with through graph-based systems. The name grew out of my work on Kartographer, a lightweight graph library for organizing and interacting with information. KDeps builds on Kartographer’s foundation and serves as a RAG-first (Retrieval-Augmented Generation) AI agent framework.” — Joel Bryan Juliano, KDeps creator +> "KDeps, short for 'knowledge dependencies,' is inspired by the principle that knowledge—whether from AI, machines, or humans—can be represented, organized, orchestrated, and interacted with through graph-based systems. The name grew out of my work on Kartographer, a lightweight graph library for organizing and interacting with information. KDeps builds on Kartographer's foundation and serves as a RAG-first (Retrieval-Augmented Generation) AI agent framework." — Joel Bryan Juliano, KDeps creator ## Why Offline-First? @@ -20,7 +41,7 @@ featuring integrated open-source LLMs for AI-powered APIs and workflows. Runs fu - **Control and independence**: Avoids vendor lock-in; reproducible on-prem. - **Data residency**: Meets jurisdictional requirements. - **Security**: Minimizes external attack surface, no third-party AI APIs. -- **Edge readiness**: Process data where it’s generated. +- **Edge readiness**: Process data where it's generated. - **Productivity**: Self-contained Docker images for local dev and deployment. KDeps achieves offline-first by integrating open-source LLMs via Ollama and packaging full applications—including models and runtimes—into Docker images. No external AI APIs are required. diff --git a/pkg/resolver/chat_decoder_test.go b/pkg/resolver/chat_decoder_test.go index 2105f79..c0a5a12 100644 --- a/pkg/resolver/chat_decoder_test.go +++ b/pkg/resolver/chat_decoder_test.go @@ -550,13 +550,19 @@ func TestConstructToolCallsFromJSON(t *testing.T) { logger := logging.GetLogger() // Array form jsonStr := `[{"name": "echo", "arguments": {"msg": "hi"}}]` - calls := constructToolCallsFromJSON(jsonStr, logger) + calls, err := constructToolCallsFromJSON(jsonStr, logger) + if err != nil { + t.Errorf("unexpected error: %v", err) + } if len(calls) != 1 || calls[0].FunctionCall.Name != "echo" { t.Errorf("unexpected calls parsed: %v", calls) } // Single object form single := `{"name":"sum","arguments": {"a":1}}` - calls2 := constructToolCallsFromJSON(single, logger) + calls2, err2 := constructToolCallsFromJSON(single, logger) + if err2 != nil { + t.Errorf("unexpected error: %v", err2) + } if len(calls2) != 1 || calls2[0].FunctionCall.Name != "sum" { t.Errorf("single object parse failed: %v", calls2) } diff --git a/pkg/resolver/resource_chat.go b/pkg/resolver/resource_chat.go index f71126d..0d42944 100644 --- a/pkg/resolver/resource_chat.go +++ b/pkg/resolver/resource_chat.go @@ -39,6 +39,7 @@ const ( RoleAction = "action" RoleTool = "tool" maxLogContentLength = 100 + maxJSONRetries = 3 // Maximum number of retries for JSON validation failures ) // HandleLLMChat initiates asynchronous processing of an LLM chat interaction. @@ -116,6 +117,68 @@ func (dr *DependencyResolver) ensureModelAvailable(model string) error { return nil } +// jsonErrorAttempt tracks a single failed JSON validation attempt +type jsonErrorAttempt struct { + attempt int + response string + error string +} + +// buildRetryPrompt constructs a prompt that includes previous JSON errors for self-correction +func buildRetryPrompt(errorHistory []jsonErrorAttempt) string { + if len(errorHistory) == 0 { + return "" + } + + var promptBuilder strings.Builder + promptBuilder.WriteString(fmt.Sprintf("\n\nPREVIOUS ATTEMPTS FAILED (%d attempts):\n", len(errorHistory))) + + for _, hist := range errorHistory { + promptBuilder.WriteString(fmt.Sprintf("\n--- Attempt %d ---\n", hist.attempt)) + promptBuilder.WriteString(fmt.Sprintf("Your Response: %s\n", utils.TruncateString(hist.response, 200))) + promptBuilder.WriteString(fmt.Sprintf("Error That Occurred: %s\n", hist.error)) + } + + promptBuilder.WriteString("\nINSTRUCTIONS FOR RETRY:\n") + promptBuilder.WriteString("Please analyze ALL the errors above and fix them in your next response.\n") + promptBuilder.WriteString("Avoid repeating the same mistakes.\n") + promptBuilder.WriteString("Common patterns to avoid:\n") + promptBuilder.WriteString("- Including explanatory text before/after JSON\n") + promptBuilder.WriteString("- Using markdown code blocks (```)\n") + promptBuilder.WriteString("- Missing required fields\n") + promptBuilder.WriteString("- Incorrect data types\n") + promptBuilder.WriteString("- Malformed JSON syntax (missing brackets, commas, quotes)\n") + promptBuilder.WriteString("Return ONLY valid JSON with no additional commentary.\n") + + return promptBuilder.String() +} + +// validateJSONResponse validates that a response is valid JSON +func validateJSONResponse(response string, logger *logging.Logger) error { + if response == "" { + return fmt.Errorf("response is empty") + } + + trimmed := strings.TrimSpace(response) + if trimmed == "" { + return fmt.Errorf("response contains only whitespace") + } + + // Check for markdown code blocks + if strings.HasPrefix(trimmed, "```") { + return fmt.Errorf("response contains markdown code blocks - must return raw JSON only") + } + + // Try to unmarshal to validate JSON structure + var jsonData interface{} + if err := json.Unmarshal([]byte(trimmed), &jsonData); err != nil { + return fmt.Errorf("invalid JSON syntax: %w", err) + } + + logger.Info("JSON validation successful") + return nil +} + // generateChatResponse generates a response from the LLM based on the chat block, executing tools via toolreader. func generateChatResponse(ctx context.Context, fs afero.Fs, llm *ollama.LLM, chatBlock *pklLLM.ResourceChat, toolreader *tool.PklResourceReader, logger *logging.Logger) (string, error) { logger.Info("Processing chatBlock", @@ -202,76 +265,179 @@ func generateChatResponse(ctx context.Context, fs afero.Fs, llm *ollama.LLM, cha "json_mode", utils.SafeDerefBool(chatBlock.JSONResponse), "tool_count", len(availableTools)) - // First GenerateContent call - response, err := llm.GenerateContent(ctx, messageHistory, opts...) - if err != nil { - errMsg := strings.ToLower(err.Error()) - - // Check for various Ollama error conditions that indicate we should try to pull the model - shouldTryPull := strings.Contains(errMsg, "not found") || - strings.Contains(errMsg, "model") && strings.Contains(errMsg, "not found") || - strings.Contains(errMsg, "no such file or directory") || - strings.Contains(errMsg, "connection refused") || - strings.Contains(errMsg, "eof") || - strings.Contains(errMsg, "try pulling it first") + // Initialize error history for self-correcting retries + var jsonErrorHistory []jsonErrorAttempt + var response *llms.ContentResponse + var respChoice *llms.ContentChoice + var toolCalls []llms.ToolCall + var err error - if shouldTryPull { - logger.Info("model error during content generation, attempting to pull", "model", chatBlock.Model, "error", err.Error()) + // Self-correcting retry loop for JSON validation + for attempt := 1; attempt <= maxJSONRetries; attempt++ { + logger.Info("LLM generation attempt", "attempt", attempt, "max_retries", maxJSONRetries) - // Try to pull the model - this will also ensure Ollama server is running - if pullErr := pullOllamaModel(ctx, chatBlock.Model, logger); pullErr != nil { - logger.Error("failed to pull model during content generation", "model", chatBlock.Model, "error", pullErr) - return "", fmt.Errorf("failed to pull model %s during content generation: %w", chatBlock.Model, pullErr) - } + // First GenerateContent call + response, err = llm.GenerateContent(ctx, messageHistory, opts...) + if err != nil { + errMsg := strings.ToLower(err.Error()) + + // Check for various Ollama error conditions that indicate we should try to pull the model + shouldTryPull := strings.Contains(errMsg, "not found") || + strings.Contains(errMsg, "model") && strings.Contains(errMsg, "not found") || + strings.Contains(errMsg, "no such file or directory") || + strings.Contains(errMsg, "connection refused") || + strings.Contains(errMsg, "eof") || + strings.Contains(errMsg, "try pulling it first") + + if shouldTryPull { + logger.Info("model error during content generation, attempting to pull", "model", chatBlock.Model, "error", err.Error()) + + // Try to pull the model - this will also ensure Ollama server is running + if pullErr := pullOllamaModel(ctx, chatBlock.Model, logger); pullErr != nil { + logger.Error("failed to pull model during content generation", "model", chatBlock.Model, "error", pullErr) + return "", fmt.Errorf("failed to pull model %s during content generation: %w", chatBlock.Model, pullErr) + } - // Retry GenerateContent after pulling - response, err = llm.GenerateContent(ctx, messageHistory, opts...) - if err != nil { - // Try once more after a brief delay to allow server to fully start - time.Sleep(1 * time.Second) + // Retry GenerateContent after pulling response, err = llm.GenerateContent(ctx, messageHistory, opts...) if err != nil { - logger.Error("Failed to generate content after model pull retry", "error", err) - return "", fmt.Errorf("failed to generate content after model pull: %w", err) + // Try once more after a brief delay to allow server to fully start + time.Sleep(1 * time.Second) + response, err = llm.GenerateContent(ctx, messageHistory, opts...) + if err != nil { + logger.Error("Failed to generate content after model pull retry", "error", err) + return "", fmt.Errorf("failed to generate content after model pull: %w", err) + } } + } else { + logger.Error("Failed to generate content in first call", "error", err) + return "", fmt.Errorf("failed to generate content in first call: %w", err) } - } else { - logger.Error("Failed to generate content in first call", "error", err) - return "", fmt.Errorf("failed to generate content in first call: %w", err) } - } - if len(response.Choices) == 0 { - logger.Error("No choices in LLM response") - return "", errors.New("no choices in LLM response") - } + if len(response.Choices) == 0 { + logger.Error("No choices in LLM response") + return "", errors.New("no choices in LLM response") + } - // Select choice with tool calls, if any - var respChoice *llms.ContentChoice - if len(availableTools) > 0 { - for _, choice := range response.Choices { - if len(choice.ToolCalls) > 0 { - respChoice = choice - break + // Select choice with tool calls, if any + respChoice = nil + if len(availableTools) > 0 { + for _, choice := range response.Choices { + if len(choice.ToolCalls) > 0 { + respChoice = choice + break + } } } - } - if respChoice == nil && len(response.Choices) > 0 { - respChoice = response.Choices[0] - } + if respChoice == nil && len(response.Choices) > 0 { + respChoice = response.Choices[0] + } + + logger.Info("LLM response received", + "attempt", attempt, + "content", utils.TruncateString(respChoice.Content, maxLogContentLength), + "tool_calls", len(respChoice.ToolCalls), + "stop_reason", respChoice.StopReason, + "tool_names", extractToolNames(respChoice.ToolCalls)) - logger.Info("First LLM response", - "content", utils.TruncateString(respChoice.Content, maxLogContentLength), - "tool_calls", len(respChoice.ToolCalls), - "stop_reason", respChoice.StopReason, - "tool_names", extractToolNames(respChoice.ToolCalls)) + // Validate JSON if JSON mode is enabled or tools are available + shouldValidateJSON := (chatBlock.JSONResponse != nil && *chatBlock.JSONResponse) || len(availableTools) > 0 + var jsonValidationErr error + + if shouldValidateJSON && len(availableTools) == 0 { + // Pure JSON response mode (not tool calls) + jsonValidationErr = validateJSONResponse(respChoice.Content, logger) + if jsonValidationErr != nil { + logger.Warn("JSON validation failed", + "attempt", attempt, + "error", jsonValidationErr.Error(), + "content", utils.TruncateString(respChoice.Content, 150)) + + // Record the error for self-correction + jsonErrorHistory = append(jsonErrorHistory, jsonErrorAttempt{ + attempt: attempt, + response: respChoice.Content, + error: jsonValidationErr.Error(), + }) + + // If we haven't exhausted retries, add error context and retry + if attempt < maxJSONRetries { + retryPrompt := buildRetryPrompt(jsonErrorHistory) + // Update the last message (user prompt) to include error history + if len(messageHistory) > 0 { + lastMsgIdx := len(messageHistory) - 1 + for i := lastMsgIdx; i >= 0; i-- { + if messageHistory[i].Role == llms.ChatMessageTypeHuman { + if len(messageHistory[i].Parts) > 0 { + if textPart, ok := messageHistory[i].Parts[0].(llms.TextContent); ok { + messageHistory[i].Parts[0] = llms.TextContent{Text: textPart.Text + retryPrompt} + logger.Info("Added error context to prompt for retry", "attempt", attempt+1) + break + } + } + } + } + } + continue + } else { + // All retries exhausted + logger.Error("JSON validation failed after all retries", + "attempts", maxJSONRetries, + "final_error", jsonValidationErr.Error()) + return "", fmt.Errorf("JSON validation failed after %d attempts: %w", maxJSONRetries, jsonValidationErr) + } + } + } + + // Process tool calls + toolCalls = respChoice.ToolCalls + if len(toolCalls) == 0 && len(availableTools) > 0 { + logger.Info("No direct ToolCalls, attempting to construct from JSON") + var toolCallErr error + toolCalls, toolCallErr = constructToolCallsFromJSON(respChoice.Content, logger) + if toolCallErr != nil { + logger.Warn("Tool call JSON parsing failed", + "attempt", attempt, + "error", toolCallErr.Error(), + "content", utils.TruncateString(respChoice.Content, 150)) + + // Record the error for self-correction + jsonErrorHistory = append(jsonErrorHistory, jsonErrorAttempt{ + attempt: attempt, + response: respChoice.Content, + error: fmt.Sprintf("Tool call parsing failed: %v", toolCallErr), + }) + + // If we haven't exhausted retries, add error context and retry + if attempt < maxJSONRetries { + retryPrompt := buildRetryPrompt(jsonErrorHistory) + // Update the system prompt to include error history for tool calls + systemPrompt := buildSystemPrompt(chatBlock.JSONResponse, chatBlock.JSONResponseKeys, availableTools) + systemPrompt += retryPrompt + messageHistory[0] = llms.MessageContent{ + Role: llms.ChatMessageTypeSystem, + Parts: []llms.ContentPart{llms.TextContent{Text: systemPrompt}}, + } + logger.Info("Added tool call error context to system prompt for retry", "attempt", attempt+1) + continue + } else { + // All retries exhausted + logger.Error("Tool call JSON parsing failed after all retries", + "attempts", maxJSONRetries, + "final_error", toolCallErr.Error()) + return "", fmt.Errorf("tool call JSON parsing failed after %d attempts: %w", maxJSONRetries, toolCallErr) + } + } + } - // Process first response - toolCalls := respChoice.ToolCalls - if len(toolCalls) == 0 && len(availableTools) > 0 { - logger.Info("No direct ToolCalls, attempting to construct from JSON") - constructedToolCalls := constructToolCallsFromJSON(respChoice.Content, logger) - toolCalls = constructedToolCalls + // Success! Break out of retry loop + if jsonValidationErr == nil { + if attempt > 1 { + logger.Info("JSON validation succeeded after retry", "successful_attempt", attempt) + } + break + } } // Deduplicate tool calls @@ -381,8 +547,17 @@ func generateChatResponse(ctx context.Context, fs afero.Fs, llm *ollama.LLM, cha toolCalls = respChoice.ToolCalls if len(toolCalls) == 0 && len(availableTools) > 0 { logger.Info("No direct ToolCalls, attempting to construct from JSON", "iteration", iteration+1) - constructedToolCalls := constructToolCallsFromJSON(respChoice.Content, logger) - toolCalls = constructedToolCalls + constructedToolCalls, toolCallErr := constructToolCallsFromJSON(respChoice.Content, logger) + if toolCallErr != nil { + logger.Warn("Failed to construct tool calls from JSON in iteration", + "iteration", iteration+1, + "error", toolCallErr.Error(), + "content", utils.TruncateString(respChoice.Content, 150)) + // Continue with empty tool calls, will exit iteration + toolCalls = nil + } else { + toolCalls = constructedToolCalls + } } // Deduplicate tool calls diff --git a/pkg/resolver/resource_chat_tool_processor.go b/pkg/resolver/resource_chat_tool_processor.go index 3b9c92a..75e9327 100644 --- a/pkg/resolver/resource_chat_tool_processor.go +++ b/pkg/resolver/resource_chat_tool_processor.go @@ -100,10 +100,11 @@ func generateAvailableTools(chatBlock *pklLLM.ResourceChat, logger *logging.Logg } // constructToolCallsFromJSON parses a JSON string into a slice of llms.ToolCall. -func constructToolCallsFromJSON(jsonContent string, logger *logging.Logger) []llms.ToolCall { +// Returns the tool calls and a detailed error if JSON parsing fails. +func constructToolCallsFromJSON(jsonContent string, logger *logging.Logger) ([]llms.ToolCall, error) { if jsonContent == "" { logger.Info("JSON content is empty, returning empty ToolCalls") - return nil + return nil, nil } type jsonToolCall struct { @@ -117,32 +118,36 @@ func constructToolCallsFromJSON(jsonContent string, logger *logging.Logger) []ll err := json.Unmarshal([]byte(jsonContent), &toolCalls) if err != nil { if err := json.Unmarshal([]byte(jsonContent), &singleCall); err != nil { - logger.Warn("Failed to unmarshal JSON content as array or single object", "content", utils.TruncateString(jsonContent, 100), "error", err) - return nil + detailedError := fmt.Errorf("failed to unmarshal JSON as array or single object: %w. Content preview: %s", + err, utils.TruncateString(jsonContent, 150)) + logger.Warn("Failed to unmarshal JSON content", "content", utils.TruncateString(jsonContent, 100), "error", err) + return nil, detailedError } toolCalls = []jsonToolCall{singleCall} } if len(toolCalls) == 0 { logger.Info("No tool calls found in JSON content") - return nil + return nil, nil } result := make([]llms.ToolCall, 0, len(toolCalls)) seen := make(map[string]struct{}) - var errors []string + var validationErrors []string for i, tc := range toolCalls { if tc.Name == "" || tc.Arguments == nil { + errMsg := fmt.Sprintf("tool call at index %d has empty name or nil arguments", i) logger.Warn("Skipping invalid tool call", "index", i, "name", tc.Name) - errors = append(errors, "tool call at index "+strconv.Itoa(i)+" has empty name or nil arguments") + validationErrors = append(validationErrors, errMsg) continue } argsJSON, err := json.Marshal(tc.Arguments) if err != nil { + errMsg := fmt.Sprintf("failed to marshal arguments for %s at index %d: %v", tc.Name, i, err) logger.Warn("Failed to marshal arguments", "index", i, "name", tc.Name, "error", err) - errors = append(errors, "failed to marshal arguments for "+tc.Name+" at index "+strconv.Itoa(i)+": "+err.Error()) + validationErrors = append(validationErrors, errMsg) continue } @@ -170,13 +175,14 @@ func constructToolCallsFromJSON(jsonContent string, logger *logging.Logger) []ll "arguments", utils.TruncateString(string(argsJSON), 100)) } - if len(result) == 0 && len(errors) > 0 { - logger.Warn("No valid tool calls constructed", "errors", errors) - return nil + if len(result) == 0 && len(validationErrors) > 0 { + combinedError := fmt.Errorf("no valid tool calls constructed. Errors: %s", strings.Join(validationErrors, "; ")) + logger.Warn("No valid tool calls constructed", "errors", validationErrors) + return nil, combinedError } logger.Info("Constructed tool calls", "count", len(result)) - return result + return result, nil } // extractToolParams extracts and validates tool call parameters. diff --git a/pkg/resolver/tool_processor_test.go b/pkg/resolver/tool_processor_test.go index 5656634..37e9a0a 100644 --- a/pkg/resolver/tool_processor_test.go +++ b/pkg/resolver/tool_processor_test.go @@ -158,16 +158,19 @@ func TestConstructToolCallsFromJSONAndDeduplication(t *testing.T) { logger := logging.NewTestLogger() // case 1: empty string returns nil - result := constructToolCallsFromJSON("", logger) + result, err := constructToolCallsFromJSON("", logger) assert.Nil(t, result) + assert.Nil(t, err) - // case 2: invalid json returns nil - result = constructToolCallsFromJSON("{bad json}", logger) + // case 2: invalid json returns error + result, err = constructToolCallsFromJSON("{bad json}", logger) assert.Nil(t, result) + assert.NotNil(t, err) // case 3: single valid object single := `{"name":"echo","arguments":{"msg":"hi"}}` - result = constructToolCallsFromJSON(single, logger) + result, err = constructToolCallsFromJSON(single, logger) + assert.Nil(t, err) assert.Len(t, result, 1) assert.Equal(t, "echo", result[0].FunctionCall.Name) @@ -177,7 +180,8 @@ func TestConstructToolCallsFromJSONAndDeduplication(t *testing.T) { {"name":"echo","arguments":{"msg":"hi"}}, {"name":"sum","arguments":{"a":1,"b":2}} ]` - result = constructToolCallsFromJSON(arr, logger) + result, err = constructToolCallsFromJSON(arr, logger) + assert.Nil(t, err) // before dedup, duplicates exist; after dedup should be 2 unique dedup := deduplicateToolCalls(result, logger) assert.Len(t, dedup, 2)