From 32b193973f241219ca9f22574ceffe7af8de2368 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 13 Sep 2026 00:57:50 +0000 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=A4=96=20fix(tools):=20summarize=20in?= =?UTF-8?q?valid=20tool=20input=20errors=20for=20the=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the AI SDK rejects a tool call before execution, StreamManager relayed the raw AI_InvalidToolInputError string as the tool result. That string echoes the entire submitted input and buries the single zod issue at the end without stating the received length, so the model retried blind. Add a shared formatToolInputIssues helper that renders zod issues as ": (received N characters)" without echoing the value, use it for the kernel bridge's existing "Invalid arguments" path, and have StreamManager derive the same concise message from the invalid tool-call part's cause chain before the matching tool-error arrives. --- .../utils/tools/formatToolInputIssues.test.ts | 75 ++++++++++++++++ .../utils/tools/formatToolInputIssues.ts | 59 +++++++++++++ src/node/services/ptc/toolBridge.test.ts | 34 ++++++++ src/node/services/ptc/toolBridge.ts | 8 +- src/node/services/streamManager.test.ts | 85 +++++++++++++++++++ src/node/services/streamManager.ts | 37 +++++++- 6 files changed, 292 insertions(+), 6 deletions(-) create mode 100644 src/common/utils/tools/formatToolInputIssues.test.ts create mode 100644 src/common/utils/tools/formatToolInputIssues.ts diff --git a/src/common/utils/tools/formatToolInputIssues.test.ts b/src/common/utils/tools/formatToolInputIssues.test.ts new file mode 100644 index 00000000000..b40cca5888f --- /dev/null +++ b/src/common/utils/tools/formatToolInputIssues.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test"; +import { z } from "zod"; +import { formatToolInputIssues, isToolInputIssueArray } from "./formatToolInputIssues"; + +function issuesFor(schema: z.ZodType, value: unknown) { + const result = schema.safeParse(value); + if (result.success) { + throw new Error("expected schema to reject value"); + } + return result.error.issues; +} + +describe("formatToolInputIssues", () => { + test("appends the received length for a string max violation without echoing the value", () => { + const input = { question: "x".repeat(2100) }; + const schema = z.object({ question: z.string().max(2000) }); + + const message = formatToolInputIssues(issuesFor(schema, input), input); + + expect(message).toBe( + "question: Too big: expected string to have <=2000 characters (received 2100 characters)" + ); + expect(message).not.toContain("xxx"); + }); + + test("appends the received length for a string min violation", () => { + const input = { question: "ab" }; + const schema = z.object({ question: z.string().min(5) }); + + expect(formatToolInputIssues(issuesFor(schema, input), input)).toEndWith( + "(received 2 characters)" + ); + }); + + test("joins multiple issues and only annotates string-origin size issues", () => { + const input = { question: "x".repeat(2100), tags: ["a", "b"], nested: { count: 1 } }; + const schema = z.object({ + question: z.string().max(2000), + tags: z.array(z.string()).max(1), + nested: z.object({ count: z.number().min(2) }), + }); + + const message = formatToolInputIssues(issuesFor(schema, input), input); + const parts = message.split("; "); + + expect(parts).toHaveLength(3); + expect(parts[0]).toContain("received 2100 characters"); + expect(parts[1]).toStartWith("tags: "); + expect(parts[1]).not.toContain("received"); + expect(parts[2]).toStartWith("nested.count: "); + expect(parts[2]).not.toContain("received"); + }); + + test("labels a root-level issue as the whole input", () => { + const message = formatToolInputIssues(issuesFor(z.object({ a: z.string() }), "oops"), "oops"); + + expect(message).toStartWith("input: "); + }); + + test("skips the received suffix when the value at the path is not a string", () => { + const issues = [{ path: ["question"], message: "Too big", code: "too_big", origin: "string" }]; + + expect(formatToolInputIssues(issues, { question: 42 })).toBe("question: Too big"); + expect(formatToolInputIssues(issues, null)).toBe("question: Too big"); + }); +}); + +describe("isToolInputIssueArray", () => { + test("accepts zod issues and rejects other shapes", () => { + expect(isToolInputIssueArray(issuesFor(z.string(), 1))).toBe(true); + expect(isToolInputIssueArray([{ path: "question", message: "x" }])).toBe(false); + expect(isToolInputIssueArray([{ path: [], message: 1 }])).toBe(false); + expect(isToolInputIssueArray("issues")).toBe(false); + }); +}); diff --git a/src/common/utils/tools/formatToolInputIssues.ts b/src/common/utils/tools/formatToolInputIssues.ts new file mode 100644 index 00000000000..2101e7b741f --- /dev/null +++ b/src/common/utils/tools/formatToolInputIssues.ts @@ -0,0 +1,59 @@ +/** + * Zod v4 issue fields read when rendering tool input validation failures. + * Duck-typed so callers can pass issues recovered from an AI SDK error cause + * chain (InvalidToolInputError -> TypeValidationError -> ZodError) without + * importing zod. + */ +export interface ToolInputIssue { + path: readonly PropertyKey[]; + message: string; + code?: string; + origin?: string; +} + +export function isToolInputIssueArray(value: unknown): value is ToolInputIssue[] { + return ( + Array.isArray(value) && + value.every( + (issue: unknown) => + typeof issue === "object" && + issue !== null && + Array.isArray((issue as { path?: unknown }).path) && + typeof (issue as { message?: unknown }).message === "string" + ) + ); +} + +/** + * Render zod issues as one concise line: ": [ (received N characters)]". + * Never echoes the input value itself; the AI SDK's own message does, which buries the + * actionable issue under kilobytes of the model's rejected input. + */ +export function formatToolInputIssues(issues: readonly ToolInputIssue[], input: unknown): string { + return issues.map((issue) => formatToolInputIssue(issue, input)).join("; "); +} + +function formatToolInputIssue(issue: ToolInputIssue, input: unknown): string { + const label = issue.path.length > 0 ? issue.path.map(String).join(".") : "input"; + let text = `${label}: ${issue.message}`; + // Zod v4 size messages state the limit but not the actual length, which is + // the number the model needs to shorten its retry. + if ((issue.code === "too_big" || issue.code === "too_small") && issue.origin === "string") { + const value = resolvePath(input, issue.path); + if (typeof value === "string") { + text += ` (received ${value.length} characters)`; + } + } + return text; +} + +function resolvePath(input: unknown, path: readonly PropertyKey[]): unknown { + let current: unknown = input; + for (const key of path) { + if (current === null || typeof current !== "object") { + return undefined; + } + current = (current as Record)[key]; + } + return current; +} diff --git a/src/node/services/ptc/toolBridge.test.ts b/src/node/services/ptc/toolBridge.test.ts index 748fed60b4f..5e4bac46b91 100644 --- a/src/node/services/ptc/toolBridge.test.ts +++ b/src/node/services/ptc/toolBridge.test.ts @@ -225,6 +225,40 @@ describe("ToolBridge", () => { expect(mockExecute).toHaveBeenCalledTimes(1); }); + it("reports the received length for a string size violation", async () => { + const tools: Record = { + file_read: createMockTool( + "file_read", + z.object({ path: z.string().max(10) }), + mock(() => ({ result: "ok" })) + ), + }; + const bridge = new ToolBridge(tools); + let registeredMux: Record Promise> = {}; + bridge.register( + createMockRuntime({ + registerObject: mock( + (name: string, obj: Record Promise>) => { + if (name === "mux") registeredMux = obj; + return undefined; + } + ), + }) + ); + + const fileRead = registeredMux.file_read as (...args: unknown[]) => Promise; + const path = "y".repeat(25); + try { + await fileRead({ path }); + expect.unreachable("Should have thrown"); + } catch (e) { + const message = String(e); + expect(message).toContain("Invalid arguments for file_read: path:"); + expect(message).toContain("(received 25 characters)"); + expect(message).not.toContain(path); + } + }); + it("passes args through for JSON-Schema (MCP-style) tools", async () => { // MCP tools carry the AI SDK's jsonSchema() wrapper instead of a Zod // schema. Regression: validateArgs called schema.safeParse on it and diff --git a/src/node/services/ptc/toolBridge.ts b/src/node/services/ptc/toolBridge.ts index 31946721536..1fa75fecab3 100644 --- a/src/node/services/ptc/toolBridge.ts +++ b/src/node/services/ptc/toolBridge.ts @@ -21,6 +21,7 @@ import { isBridgeToolGranted, type CapabilityGrants, } from "@/common/types/capabilityGrants"; +import { formatToolInputIssues } from "@/common/utils/tools/formatToolInputIssues"; import { isToolContentResult } from "@/common/utils/tools/toolContentResult"; import { TOOL_DEFINITIONS } from "@/common/utils/tools/toolDefinitions"; import { isSupportedAttachmentMediaType } from "@/common/utils/attachments/supportedAttachmentMediaTypes"; @@ -592,10 +593,9 @@ export class ToolBridge { if (typeof schema === "object" && "_def" in schema) { const result = (schema as z.ZodType).safeParse(args); if (!result.success) { - const issues = result.error.issues - .map((i) => `${i.path.join(".")}: ${i.message}`) - .join("; "); - throw new Error(`Invalid arguments for ${toolName}: ${issues}`); + throw new Error( + `Invalid arguments for ${toolName}: ${formatToolInputIssues(result.error.issues, args)}` + ); } return result.data; } diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 7711d0d0f4c..28f640220c7 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -23,6 +23,7 @@ import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import type { WorkflowRunRecord } from "@/common/types/workflow"; import { Ok, Err } from "@/common/types/result"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; +import { AdvisorToolInputSchema } from "@/common/utils/tools/toolDefinitions"; import type { ToolSearchStreamState } from "@/common/utils/tools/toolCatalog"; import { StreamManager, @@ -40,7 +41,9 @@ import { stripEncryptedContent } from "@/node/utils/messages/stripEncryptedConte import * as aiSdk from "ai"; import { APICallError, + InvalidToolInputError, RetryError, + TypeValidationError, tool, type LanguageModel, type ModelMessage, @@ -4118,6 +4121,88 @@ describe("StreamManager - exact step indices", () => { ); }); +describe("StreamManager - invalid tool input", () => { + test("relays a concise zod summary instead of the SDK's input-echoing error", async () => { + const streamManager = new StreamManager(historyService); + const workspaceId = "invalid-tool-input-workspace"; + const messageId = "invalid-tool-input-message"; + await appendPartialAssistantForTests(workspaceId, messageId, 1); + Reflect.set(streamManager, "tokenTracker", { + setModel: () => Promise.resolve(undefined), + countTokens: () => Promise.resolve(0), + }); + + // Mirror the SDK's pre-execution rejection: an invalid dynamic tool-call + // carrying the Error, then a tool-error carrying only its message string. + const input = { question: "x".repeat(2100) }; + const validation = AdvisorToolInputSchema.safeParse(input); + if (validation.success) throw new Error("Expected the advisor schema to reject the input"); + const error = new InvalidToolInputError({ + toolName: "advisor", + toolInput: JSON.stringify(input), + cause: TypeValidationError.wrap({ value: input, cause: validation.error }), + }); + const toolCallId = "advisor-invalid-call"; + const streamInfo = createStreamInfoForTests({ + messageId, + streamResult: createStreamResultForTests( + (async function* () { + await Promise.resolve(); + yield { type: "start-step" }; + yield { + type: "tool-call", + toolCallId, + toolName: "advisor", + input, + dynamic: true, + invalid: true, + error, + }; + yield { + type: "tool-error", + toolCallId, + toolName: "advisor", + input, + error: error.message, + dynamic: true, + }; + yield { + type: "finish-step", + usage: { inputTokens: 10, outputTokens: 1, totalTokens: 11 }, + }; + yield { type: "finish", finishReason: "stop" }; + })() + ), + }); + + await getProcessStreamWithCleanupForTests(streamManager).call( + streamManager, + workspaceId, + streamInfo, + 1 + ); + + const history = await historyService.getHistoryFromLatestBoundary(workspaceId); + expect(history.success).toBe(true); + if (!history.success) throw new Error(history.error); + const toolPart = history.data + .find((row) => row.id === messageId) + ?.parts.find((part) => part.type === "dynamic-tool"); + if (toolPart?.type !== "dynamic-tool" || toolPart.state !== "output-available") { + throw new Error("Expected a completed dynamic-tool part"); + } + const output = toolPart.output as { success: boolean; error: string }; + + expect(output.success).toBe(false); + expect(output.error).toContain("advisor"); + expect(output.error).toContain("question"); + expect(output.error).toContain("2000"); + expect(output.error).toContain("2100"); + expect(output.error.length).toBeLessThan(500); + expect(output.error).not.toContain("xxxx"); + }); +}); + describe("StreamManager - empty stream completions", () => { const runtime = createRuntime({ type: "local", srcBaseDir: "/tmp" }); diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index 196cb0228ef..ddae91ccd30 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -26,6 +26,7 @@ import { type ToolSet, LoadAPIKeyError, APICallError, + InvalidToolInputError, RetryError, } from "ai"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; @@ -86,6 +87,10 @@ import { import { linkAbortSignal } from "@/node/utils/abort"; import { AsyncMutex } from "@/node/utils/concurrency/asyncMutex"; import { stripInternalToolResultFields } from "@/common/utils/tools/internalToolResultFields"; +import { + formatToolInputIssues, + isToolInputIssueArray, +} from "@/common/utils/tools/formatToolInputIssues"; import { buildRequiredToolPatterns, type ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { computeActiveToolNames, @@ -190,10 +195,32 @@ interface ToolCallState { toolName: string; input: unknown; output?: unknown; + /** Concise message for a call the SDK rejected before execution; replaces the raw tool-error string. */ + invalidInputError?: string; } type ToolCallMap = Map; +/** + * The SDK's InvalidToolInputError message echoes the entire submitted input + * and buries the zod issue at the end, so the model retries blind. Render only + * the issues (with received string lengths) when the cause chain + * (InvalidToolInputError -> TypeValidationError -> ZodError) exposes them. + */ +function describeInvalidToolInput(toolName: string, input: unknown, error: unknown): string { + if (InvalidToolInputError.isInstance(error)) { + let cause: unknown = error.cause; + for (let depth = 0; depth < 3 && typeof cause === "object" && cause !== null; depth += 1) { + const issues: unknown = (cause as { issues?: unknown }).issues; + if (isToolInputIssueArray(issues)) { + return `Invalid input for tool ${toolName}: ${formatToolInputIssues(issues, input)}`; + } + cause = (cause as { cause?: unknown }).cause; + } + } + return clampErrorMessage(getErrorMessage(error)); +} + type WorkspaceId = string & { __brand: "WorkspaceId" }; type StreamToken = string & { __brand: "StreamToken" }; @@ -4080,10 +4107,15 @@ export class StreamManager { case "tool-call": { // Tool call started - store in map for later lookup + const invalidInputError = + part.dynamic === true && part.invalid === true && part.error != null + ? describeInvalidToolInput(part.toolName, part.input, part.error) + : undefined; toolCalls.set(part.toolCallId, { toolCallId: part.toolCallId, toolName: part.toolName, input: part.input, + ...(invalidInputError != null ? { invalidInputError } : {}), }); // Note: Tool availability is handled by the SDK, which emits tool-error events @@ -4186,11 +4218,12 @@ export class StreamManager { const errorOutput = { success: false, error: - typeof toolErrorPart.error === "string" + toolCalls.get(toolErrorPart.toolCallId)?.invalidInputError ?? + (typeof toolErrorPart.error === "string" ? toolErrorPart.error : toolErrorPart.error instanceof Error ? toolErrorPart.error.message - : getErrorMessage(toolErrorPart.error), + : getErrorMessage(toolErrorPart.error)), }; // Use shared completion logic (await to ensure partial is flushed before event) From b5a71760992758f13d286437f9145617e2dbc418 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 13 Sep 2026 00:57:50 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=A4=96=20fix(advisor):=20state=20the?= =?UTF-8?q?=20question=20cap=20and=20reject=20empty=20advice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Share the 2000-character question limit between the schema and the tool description so the model knows the cap up front. Return an error result when the advisor stream finishes without any text instead of a success-shaped empty advice string, so the caller and the tool card see that the consultation produced nothing. --- src/common/constants/advisor.ts | 5 ++++- src/common/utils/tools/toolDefinitions.ts | 4 ++-- src/node/services/tools/advisor.test.ts | 20 ++++++++++++++++++++ src/node/services/tools/advisor.ts | 10 ++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/common/constants/advisor.ts b/src/common/constants/advisor.ts index 2530c6741f7..f551ab53090 100644 --- a/src/common/constants/advisor.ts +++ b/src/common/constants/advisor.ts @@ -3,6 +3,9 @@ import { normalizeAgentId } from "@/common/utils/agentIds"; /** Default per-turn usage cap for the experimental advisor tool. */ export const ADVISOR_DEFAULT_MAX_USES_PER_TURN = 3; +/** Upper bound on the advisor tool's `question` input (schema and description share it). */ +export const ADVISOR_QUESTION_MAX_CHARS = 2000; + const ADVISOR_ENABLED_BY_DEFAULT_AGENT_IDS = new Set(["exec", "plan"]); export function isAdvisorEnabledByDefaultForAgent(agentId: string): boolean { @@ -45,7 +48,7 @@ export const ADVISOR_USAGE_GUIDANCE = export const ADVISOR_TOOL_DESCRIPTION = "Ask a stronger model for strategic advice based on the live conversation transcript. " + ADVISOR_USAGE_GUIDANCE + - " Pass a brief `question` summarizing the decision or ambiguity."; + ` Pass a brief \`question\` (at most ${ADVISOR_QUESTION_MAX_CHARS} characters) summarizing the decision or ambiguity.`; /** * System prompt for the nested advisor model call. diff --git a/src/common/utils/tools/toolDefinitions.ts b/src/common/utils/tools/toolDefinitions.ts index 3a254d679c9..87e63be23ee 100644 --- a/src/common/utils/tools/toolDefinitions.ts +++ b/src/common/utils/tools/toolDefinitions.ts @@ -60,7 +60,7 @@ import { BASH_MAX_TOTAL_BYTES, WEB_FETCH_MAX_OUTPUT_BYTES, } from "@/common/constants/toolLimits"; -import { ADVISOR_TOOL_DESCRIPTION } from "@/common/constants/advisor"; +import { ADVISOR_QUESTION_MAX_CHARS, ADVISOR_TOOL_DESCRIPTION } from "@/common/constants/advisor"; import { MEMORY_INTUITION_MAX_CUE_CHARS, MEMORY_INTUITION_MAX_EXCERPT_CHARS, @@ -246,7 +246,7 @@ export const HeartbeatToolArgsSchema = z export const AdvisorToolInputSchema = z .object({ // Advisor prompts often need tradeoff context; keep bounded while allowing a compact brief. - question: z.string().min(1).max(2000).nullish(), + question: z.string().min(1).max(ADVISOR_QUESTION_MAX_CHARS).nullish(), }) .strict(); diff --git a/src/node/services/tools/advisor.test.ts b/src/node/services/tools/advisor.test.ts index dca7ea07479..1819d6a76a3 100644 --- a/src/node/services/tools/advisor.test.ts +++ b/src/node/services/tools/advisor.test.ts @@ -683,6 +683,26 @@ describe("advisor tool", () => { expect(reportModelUsage).not.toHaveBeenCalled(); }); + it("returns an error instead of empty advice when the stream produced no text", async () => { + using tempDir = new TestTempDir("advisor-tool-empty-advice"); + const reportModelUsage = mock((_event: ToolModelUsageEvent) => undefined); + const { config } = createToolConfig(tempDir.path, { reportModelUsage }); + mockStreamTextSuccess({ + text: "", + finishReason: "stop", + usage: { inputTokens: 12, outputTokens: 0, totalTokens: 12 }, + }); + + const tool = createAdvisorTool(config); + const rawResult: unknown = await Promise.resolve(tool.execute!({}, mockToolCallOptions)); + + const result = rawResult as { type?: unknown; isError?: unknown; message?: unknown }; + expect(result.type).toBe("error"); + expect(result.isError).toBe(true); + expect(result.message).toEqual(expect.stringContaining("stop")); + expect(reportModelUsage).toHaveBeenCalledTimes(1); + }); + it("sanitizes binary-like advisor provider failures", async () => { using tempDir = new TestTempDir("advisor-tool-sanitized-error"); const { config } = createToolConfig(tempDir.path); diff --git a/src/node/services/tools/advisor.ts b/src/node/services/tools/advisor.ts index 5ab419641c1..2178eeb5001 100644 --- a/src/node/services/tools/advisor.ts +++ b/src/node/services/tools/advisor.ts @@ -366,6 +366,16 @@ export function createAdvisorTool(config: ToolConfiguration): Tool { } } + // An empty stream (for example a gateway hiccup) used to surface as a + // success-shaped result, so the caller saw "advice" that said nothing. + if (advice.trim().length === 0) { + return { + type: "error" as const, + isError: true, + message: `Advisor returned no advice (finish reason: ${finishReason}). Retry once or continue without it.`, + }; + } + return { type: "advice" as const, advice, From 44aafafbe2613288ef4f79ec7f6aa65bb0a86536 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 13 Sep 2026 03:04:20 +0000 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=A4=96=20fix(stream):=20summarize=20i?= =?UTF-8?q?nvalid=20tool=20input=20for=20the=20model's=20same-turn=20retry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit streamText builds the next step's tool-result from the stream after experimental_transform runs, and prepareStep leaves that content alone, so the fullStream consumer's rewrite only reached the tool card and later turns. Within the same turn the model still received the SDK's input-echoing AI_InvalidToolInputError text on its retry (found in UAT round 2). Move the summary into a streamText transform that records each rejected tool-call's concise message and rewrites the matching tool-error, so the model's same-turn retry, the persisted history, and the card all carry the same text. Drop the consumer-side plumbing that this replaces, and cover the same-turn path with an end-to-end StreamManager test that asserts the second provider request's tool-result. --- .../streamManager.invalidToolInput.test.ts | 127 ++++++++++++++++++ src/node/services/streamManager.test.ts | 85 ------------ src/node/services/streamManager.ts | 39 +----- .../summarizeInvalidToolInputErrors.test.ts | 97 +++++++++++++ .../summarizeInvalidToolInputErrors.ts | 70 ++++++++++ 5 files changed, 298 insertions(+), 120 deletions(-) create mode 100644 src/node/services/streamManager.invalidToolInput.test.ts create mode 100644 src/node/utils/messages/summarizeInvalidToolInputErrors.test.ts create mode 100644 src/node/utils/messages/summarizeInvalidToolInputErrors.ts diff --git a/src/node/services/streamManager.invalidToolInput.test.ts b/src/node/services/streamManager.invalidToolInput.test.ts new file mode 100644 index 00000000000..51c1f2a1f1d --- /dev/null +++ b/src/node/services/streamManager.invalidToolInput.test.ts @@ -0,0 +1,127 @@ +import { tmpdir } from "node:os"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { describe, expect, test } from "bun:test"; +import { MockLanguageModelV3, simulateReadableStream } from "ai/test"; +import { tool } from "ai"; +import type { LanguageModelV3Prompt, LanguageModelV3StreamPart } from "@ai-sdk/provider"; +import { createMuxMessage } from "@/common/types/message"; +import { AdvisorToolInputSchema } from "@/common/utils/tools/toolDefinitions"; +import { LocalRuntime } from "@/node/runtime/LocalRuntime"; +import { StreamManager } from "./streamManager"; +import { createTestHistoryService } from "./testHistoryService"; + +describe("StreamManager - invalid tool input", () => { + test("the model's same-turn retry and the persisted history both get the concise summary", async () => { + const h = await createTestHistoryService(); + const workspaceId = "invalid-tool-input"; + const messageId = "invalid-tool-input-assistant"; + const toolCallId = "advisor-invalid-call"; + const oversized = "x".repeat(2100); + let providerCalls = 0; + let retryPrompt: LanguageModelV3Prompt | undefined; + const model = new MockLanguageModelV3({ + doStream: (request) => { + providerCalls++; + if (providerCalls === 2) retryPrompt = request.prompt; + const usage = { + inputTokens: { total: 100, noCache: 100, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 10, text: 10, reasoning: 0 }, + }; + const chunks: LanguageModelV3StreamPart[] = + providerCalls === 1 + ? [ + { type: "stream-start", warnings: [] }, + { + type: "tool-call", + toolCallId, + toolName: "advisor", + input: JSON.stringify({ question: oversized }), + }, + { + type: "finish", + finishReason: { unified: "tool-calls", raw: "tool_calls" }, + usage, + }, + ] + : [ + { type: "stream-start", warnings: [] }, + { type: "text-start", id: "answer" }, + { type: "text-delta", id: "answer", delta: "Done" }, + { type: "text-end", id: "answer" }, + { type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage }, + ]; + return Promise.resolve({ stream: simulateReadableStream({ chunks }) }); + }, + }); + const manager = new StreamManager(h.historyService); + const runtimeDir = await fs.mkdtemp(path.join(tmpdir(), "invalid-tool-input-stream-")); + try { + expect( + ( + await h.historyService.appendManyToHistory(workspaceId, [ + createMuxMessage("user", "user", "Ask the advisor"), + createMuxMessage(messageId, "assistant", ""), + ]) + ).success + ).toBe(true); + const started = await manager.startStream({ + workspaceId, + messageId, + historySequence: 1, + model, + modelString: "openai:gpt-4o", + messages: [{ role: "user", content: "Ask the advisor" }], + system: "Use tools", + runtime: new LocalRuntime(h.tempDir), + providedRuntimeTempDir: runtimeDir, + tools: { + advisor: tool({ + description: "test advisor", + inputSchema: AdvisorToolInputSchema, + execute: (): Promise<{ advice: string }> => + Promise.reject(new Error("invalid input must not execute")), + }), + }, + }); + expect(started.success).toBe(true); + if (!started.success) throw new Error("Expected stream construction"); + const completion = await started.data.completion; + expect(completion.status).toBe("completed"); + expect(providerCalls).toBe(2); + + // Same turn: streamText feeds the rejected call's tool-result straight into the + // next step, so the retry prompt is where the SDK's input-echoing text would land. + const retryToolResult = retryPrompt + ?.flatMap((message) => (message.role === "tool" ? message.content : [])) + .find((part) => part.type === "tool-result" && part.toolCallId === toolCallId); + if (retryToolResult?.type !== "tool-result") throw new Error("Expected a tool-result"); + expect(retryToolResult.output.type).toBe("error-text"); + if (retryToolResult.output.type !== "error-text") throw new Error("Expected error-text"); + const summary = retryToolResult.output.value; + expect(summary).toContain("advisor"); + expect(summary).toContain("question"); + expect(summary).toContain("2000"); + expect(summary).toContain("received 2100 characters"); + expect(summary).not.toContain("xxxx"); + expect(summary.length).toBeLessThan(300); + + // Later turns: the persisted tool part carries the same summary. + expect((await h.historyService.commitPartial(workspaceId)).success).toBe(true); + const history = await h.historyService.getHistoryFromLatestBoundary(workspaceId); + if (!history.success) throw new Error(history.error); + const toolPart = history.data + .find((row) => row.id === messageId) + ?.parts.find((part) => part.type === "dynamic-tool"); + expect(toolPart).toMatchObject({ + toolCallId, + state: "output-available", + output: { success: false, error: summary }, + }); + } finally { + await manager.stopStream(workspaceId); + await fs.rm(runtimeDir, { recursive: true, force: true }); + await h.cleanup(); + } + }); +}); diff --git a/src/node/services/streamManager.test.ts b/src/node/services/streamManager.test.ts index 28f640220c7..7711d0d0f4c 100644 --- a/src/node/services/streamManager.test.ts +++ b/src/node/services/streamManager.test.ts @@ -23,7 +23,6 @@ import { createMuxMessage, type MuxMessage } from "@/common/types/message"; import type { WorkflowRunRecord } from "@/common/types/workflow"; import { Ok, Err } from "@/common/types/result"; import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; -import { AdvisorToolInputSchema } from "@/common/utils/tools/toolDefinitions"; import type { ToolSearchStreamState } from "@/common/utils/tools/toolCatalog"; import { StreamManager, @@ -41,9 +40,7 @@ import { stripEncryptedContent } from "@/node/utils/messages/stripEncryptedConte import * as aiSdk from "ai"; import { APICallError, - InvalidToolInputError, RetryError, - TypeValidationError, tool, type LanguageModel, type ModelMessage, @@ -4121,88 +4118,6 @@ describe("StreamManager - exact step indices", () => { ); }); -describe("StreamManager - invalid tool input", () => { - test("relays a concise zod summary instead of the SDK's input-echoing error", async () => { - const streamManager = new StreamManager(historyService); - const workspaceId = "invalid-tool-input-workspace"; - const messageId = "invalid-tool-input-message"; - await appendPartialAssistantForTests(workspaceId, messageId, 1); - Reflect.set(streamManager, "tokenTracker", { - setModel: () => Promise.resolve(undefined), - countTokens: () => Promise.resolve(0), - }); - - // Mirror the SDK's pre-execution rejection: an invalid dynamic tool-call - // carrying the Error, then a tool-error carrying only its message string. - const input = { question: "x".repeat(2100) }; - const validation = AdvisorToolInputSchema.safeParse(input); - if (validation.success) throw new Error("Expected the advisor schema to reject the input"); - const error = new InvalidToolInputError({ - toolName: "advisor", - toolInput: JSON.stringify(input), - cause: TypeValidationError.wrap({ value: input, cause: validation.error }), - }); - const toolCallId = "advisor-invalid-call"; - const streamInfo = createStreamInfoForTests({ - messageId, - streamResult: createStreamResultForTests( - (async function* () { - await Promise.resolve(); - yield { type: "start-step" }; - yield { - type: "tool-call", - toolCallId, - toolName: "advisor", - input, - dynamic: true, - invalid: true, - error, - }; - yield { - type: "tool-error", - toolCallId, - toolName: "advisor", - input, - error: error.message, - dynamic: true, - }; - yield { - type: "finish-step", - usage: { inputTokens: 10, outputTokens: 1, totalTokens: 11 }, - }; - yield { type: "finish", finishReason: "stop" }; - })() - ), - }); - - await getProcessStreamWithCleanupForTests(streamManager).call( - streamManager, - workspaceId, - streamInfo, - 1 - ); - - const history = await historyService.getHistoryFromLatestBoundary(workspaceId); - expect(history.success).toBe(true); - if (!history.success) throw new Error(history.error); - const toolPart = history.data - .find((row) => row.id === messageId) - ?.parts.find((part) => part.type === "dynamic-tool"); - if (toolPart?.type !== "dynamic-tool" || toolPart.state !== "output-available") { - throw new Error("Expected a completed dynamic-tool part"); - } - const output = toolPart.output as { success: boolean; error: string }; - - expect(output.success).toBe(false); - expect(output.error).toContain("advisor"); - expect(output.error).toContain("question"); - expect(output.error).toContain("2000"); - expect(output.error).toContain("2100"); - expect(output.error.length).toBeLessThan(500); - expect(output.error).not.toContain("xxxx"); - }); -}); - describe("StreamManager - empty stream completions", () => { const runtime = createRuntime({ type: "local", srcBaseDir: "/tmp" }); diff --git a/src/node/services/streamManager.ts b/src/node/services/streamManager.ts index ddae91ccd30..e8b52af5d86 100644 --- a/src/node/services/streamManager.ts +++ b/src/node/services/streamManager.ts @@ -26,7 +26,6 @@ import { type ToolSet, LoadAPIKeyError, APICallError, - InvalidToolInputError, RetryError, } from "ai"; import type { LanguageModelV2Usage } from "@ai-sdk/provider"; @@ -87,10 +86,7 @@ import { import { linkAbortSignal } from "@/node/utils/abort"; import { AsyncMutex } from "@/node/utils/concurrency/asyncMutex"; import { stripInternalToolResultFields } from "@/common/utils/tools/internalToolResultFields"; -import { - formatToolInputIssues, - isToolInputIssueArray, -} from "@/common/utils/tools/formatToolInputIssues"; +import { summarizeInvalidToolInputErrors } from "@/node/utils/messages/summarizeInvalidToolInputErrors"; import { buildRequiredToolPatterns, type ToolPolicy } from "@/common/utils/tools/toolPolicy"; import { computeActiveToolNames, @@ -195,32 +191,10 @@ interface ToolCallState { toolName: string; input: unknown; output?: unknown; - /** Concise message for a call the SDK rejected before execution; replaces the raw tool-error string. */ - invalidInputError?: string; } type ToolCallMap = Map; -/** - * The SDK's InvalidToolInputError message echoes the entire submitted input - * and buries the zod issue at the end, so the model retries blind. Render only - * the issues (with received string lengths) when the cause chain - * (InvalidToolInputError -> TypeValidationError -> ZodError) exposes them. - */ -function describeInvalidToolInput(toolName: string, input: unknown, error: unknown): string { - if (InvalidToolInputError.isInstance(error)) { - let cause: unknown = error.cause; - for (let depth = 0; depth < 3 && typeof cause === "object" && cause !== null; depth += 1) { - const issues: unknown = (cause as { issues?: unknown }).issues; - if (isToolInputIssueArray(issues)) { - return `Invalid input for tool ${toolName}: ${formatToolInputIssues(issues, input)}`; - } - cause = (cause as { cause?: unknown }).cause; - } - } - return clampErrorMessage(getErrorMessage(error)); -} - type WorkspaceId = string & { __brand: "WorkspaceId" }; type StreamToken = string & { __brand: "StreamToken" }; @@ -2828,6 +2802,7 @@ export class StreamManager { }, onChunk: request.onChunk, tools: request.tools, + experimental_transform: summarizeInvalidToolInputErrors(), stopWhen: this.createStopWhenCondition(request), // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment providerOptions: request.providerOptions as any, // Pass provider-specific options (thinking/reasoning config) @@ -4107,15 +4082,10 @@ export class StreamManager { case "tool-call": { // Tool call started - store in map for later lookup - const invalidInputError = - part.dynamic === true && part.invalid === true && part.error != null - ? describeInvalidToolInput(part.toolName, part.input, part.error) - : undefined; toolCalls.set(part.toolCallId, { toolCallId: part.toolCallId, toolName: part.toolName, input: part.input, - ...(invalidInputError != null ? { invalidInputError } : {}), }); // Note: Tool availability is handled by the SDK, which emits tool-error events @@ -4218,12 +4188,11 @@ export class StreamManager { const errorOutput = { success: false, error: - toolCalls.get(toolErrorPart.toolCallId)?.invalidInputError ?? - (typeof toolErrorPart.error === "string" + typeof toolErrorPart.error === "string" ? toolErrorPart.error : toolErrorPart.error instanceof Error ? toolErrorPart.error.message - : getErrorMessage(toolErrorPart.error)), + : getErrorMessage(toolErrorPart.error), }; // Use shared completion logic (await to ensure partial is flushed before event) diff --git a/src/node/utils/messages/summarizeInvalidToolInputErrors.test.ts b/src/node/utils/messages/summarizeInvalidToolInputErrors.test.ts new file mode 100644 index 00000000000..7bd08d0c5da --- /dev/null +++ b/src/node/utils/messages/summarizeInvalidToolInputErrors.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test"; +import { InvalidToolInputError, TypeValidationError, type TextStreamPart, type ToolSet } from "ai"; +import { z } from "zod"; +import { summarizeInvalidToolInputErrors } from "./summarizeInvalidToolInputErrors"; + +async function runTransform( + parts: Array> +): Promise>> { + const transform = summarizeInvalidToolInputErrors()({ + tools: {}, + stopStream: () => undefined, + }); + const output: Array> = []; + await new ReadableStream>({ + start(controller) { + for (const part of parts) controller.enqueue(part); + controller.close(); + }, + }) + .pipeThrough(transform) + .pipeTo( + new WritableStream({ + write(part) { + output.push(part); + }, + }) + ); + return output; +} + +function rejectedCall(toolCallId: string, input: unknown, cause: unknown) { + const error = new InvalidToolInputError({ + toolName: "note", + toolInput: JSON.stringify(input), + cause, + }); + return { + call: { + type: "tool-call" as const, + toolCallId, + toolName: "note", + input, + dynamic: true as const, + invalid: true, + error, + }, + toolError: { + type: "tool-error" as const, + toolCallId, + toolName: "note", + input, + error: error.message, + dynamic: true as const, + }, + }; +} + +describe("summarizeInvalidToolInputErrors", () => { + test("rewrites only the rejected call's tool-error and leaves other errors alone", async () => { + const schema = z.object({ text: z.string().max(5) }); + const input = { text: "toolong" }; + const validation = schema.safeParse(input); + if (validation.success) throw new Error("Expected the schema to reject the input"); + const rejected = rejectedCall( + "rejected", + input, + TypeValidationError.wrap({ value: input, cause: validation.error }) + ); + const executionFailure = { + type: "tool-error" as const, + toolCallId: "executed", + toolName: "note", + input: { text: "ok" }, + error: new Error("disk full"), + }; + + const output = await runTransform([rejected.call, rejected.toolError, executionFailure]); + + expect(output).toHaveLength(3); + expect(output[0]).toBe(rejected.call); + expect(output[1]).toMatchObject({ + type: "tool-error", + toolCallId: "rejected", + error: + "Invalid input for tool note: text: Too big: expected string to have <=5 characters (received 7 characters)", + }); + expect(output[2]).toBe(executionFailure); + }); + + test("falls back to the error message when the cause chain has no issues", async () => { + const rejected = rejectedCall("unparsable", "{not json", new Error("JSON parsing failed")); + + const output = await runTransform([rejected.call, rejected.toolError]); + + expect(output[1]).toMatchObject({ type: "tool-error", error: rejected.toolError.error }); + }); +}); diff --git a/src/node/utils/messages/summarizeInvalidToolInputErrors.ts b/src/node/utils/messages/summarizeInvalidToolInputErrors.ts new file mode 100644 index 00000000000..3b6824db3f4 --- /dev/null +++ b/src/node/utils/messages/summarizeInvalidToolInputErrors.ts @@ -0,0 +1,70 @@ +import { + InvalidToolInputError, + type StreamTextTransform, + type TextStreamPart, + type ToolSet, +} from "ai"; +import { clampErrorMessage, getErrorMessage } from "@/common/utils/errors"; +import { + formatToolInputIssues, + isToolInputIssueArray, +} from "@/common/utils/tools/formatToolInputIssues"; + +/** + * The SDK's InvalidToolInputError message echoes the entire submitted input + * and buries the zod issue at the end, so the model retries blind. Render only + * the issues (with received string lengths) when the cause chain + * (InvalidToolInputError -> TypeValidationError -> ZodError) exposes them. + */ +export function describeInvalidToolInput(toolName: string, input: unknown, error: unknown): string { + if (InvalidToolInputError.isInstance(error)) { + let cause: unknown = error.cause; + for (let depth = 0; depth < 3 && typeof cause === "object" && cause !== null; depth += 1) { + const issues: unknown = (cause as { issues?: unknown }).issues; + if (isToolInputIssueArray(issues)) { + return `Invalid input for tool ${toolName}: ${formatToolInputIssues(issues, input)}`; + } + cause = (cause as { cause?: unknown }).cause; + } + } + return clampErrorMessage(getErrorMessage(error)); +} + +/** + * streamText transform that rewrites the tool-error of a call the SDK rejected + * before execution (invalid input) to describeInvalidToolInput's summary. + * + * This has to be a stream transform rather than fullStream consumer logic: + * streamText builds the next step's tool-result message from the transformed + * stream, so only a transform changes what the model sees on its same-turn + * retry. The consumer then persists the same summary for later turns. + */ +export function summarizeInvalidToolInputErrors< + TOOLS extends ToolSet, +>(): StreamTextTransform { + return () => { + const summaries = new Map(); + return new TransformStream, TextStreamPart>({ + transform(part, controller) { + if ( + part.type === "tool-call" && + part.dynamic === true && + part.invalid === true && + part.error != null + ) { + summaries.set( + part.toolCallId, + describeInvalidToolInput(part.toolName, part.input, part.error) + ); + } else if (part.type === "tool-error") { + const summary = summaries.get(part.toolCallId); + if (summary != null) { + controller.enqueue({ ...part, error: summary }); + return; + } + } + controller.enqueue(part); + }, + }); + }; +} From d65a6739c0926089a4e6b1f777a2347f20bafee7 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 13 Sep 2026 03:13:52 +0000 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=A4=96=20fix(tools):=20cap=20the=20re?= =?UTF-8?q?ndered=20issues=20in=20tool=20input=20summaries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A malformed array input yields one zod issue per entry per field, so a single rejected call could expand into an unbounded summary in the next model request and in history. Render at most eight issues and count the rest. Follow-up to #4227 (Codex P1 on formatToolInputIssues.ts). --- .../utils/tools/formatToolInputIssues.test.ts | 15 +++++++++++++++ src/common/utils/tools/formatToolInputIssues.ts | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/common/utils/tools/formatToolInputIssues.test.ts b/src/common/utils/tools/formatToolInputIssues.test.ts index b40cca5888f..c812507864b 100644 --- a/src/common/utils/tools/formatToolInputIssues.test.ts +++ b/src/common/utils/tools/formatToolInputIssues.test.ts @@ -32,6 +32,21 @@ describe("formatToolInputIssues", () => { ); }); + test("caps the rendered issues and counts the rest", () => { + const input = { questions: Array.from({ length: 20 }, () => ({})) }; + const schema = z.object({ + questions: z.array(z.object({ question: z.string(), header: z.string() })), + }); + const issues = issuesFor(schema, input); + expect(issues.length).toBe(40); + + const message = formatToolInputIssues(issues, input); + + expect(message.split("; ")).toHaveLength(9); + expect(message).toStartWith("questions.0.question: "); + expect(message).toEndWith("; and 32 more issues"); + }); + test("joins multiple issues and only annotates string-origin size issues", () => { const input = { question: "x".repeat(2100), tags: ["a", "b"], nested: { count: 1 } }; const schema = z.object({ diff --git a/src/common/utils/tools/formatToolInputIssues.ts b/src/common/utils/tools/formatToolInputIssues.ts index 2101e7b741f..4a8d5e08fa8 100644 --- a/src/common/utils/tools/formatToolInputIssues.ts +++ b/src/common/utils/tools/formatToolInputIssues.ts @@ -24,13 +24,26 @@ export function isToolInputIssueArray(value: unknown): value is ToolInputIssue[] ); } +/** + * A malformed array input yields one issue per entry per field, so a single bad call + * could otherwise expand into an unbounded summary in the next model request. + */ +const MAX_RENDERED_ISSUES = 8; + /** * Render zod issues as one concise line: ": [ (received N characters)]". * Never echoes the input value itself; the AI SDK's own message does, which buries the * actionable issue under kilobytes of the model's rejected input. */ export function formatToolInputIssues(issues: readonly ToolInputIssue[], input: unknown): string { - return issues.map((issue) => formatToolInputIssue(issue, input)).join("; "); + const rendered = issues + .slice(0, MAX_RENDERED_ISSUES) + .map((issue) => formatToolInputIssue(issue, input)) + .join("; "); + const omitted = issues.length - Math.min(issues.length, MAX_RENDERED_ISSUES); + return omitted > 0 + ? `${rendered}; and ${omitted} more issue${omitted === 1 ? "" : "s"}` + : rendered; } function formatToolInputIssue(issue: ToolInputIssue, input: unknown): string {