From f17368bcdaa6343f081939fc4b81cdc8f92538cf Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:52:08 +0000 Subject: [PATCH 1/6] fix(coderd): retry transient HTTP/2 hook aborts --- coderd/util/xnet/xnet.go | 36 ++++++++++++- coderd/util/xnet/xnet_test.go | 54 +++++++++++++++++++ .../dispatch/dispatcher_internal_test.go | 45 ++++++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) diff --git a/coderd/util/xnet/xnet.go b/coderd/util/xnet/xnet.go index 2957b9dd393..82aa30017cd 100644 --- a/coderd/util/xnet/xnet.go +++ b/coderd/util/xnet/xnet.go @@ -7,6 +7,8 @@ import ( "io" "net" "syscall" + + "golang.org/x/net/http2" ) // IsTimeoutError reports whether err indicates the peer did not respond in @@ -31,5 +33,37 @@ func IsConnectionError(err error) bool { return true } var opErr *net.OpError - return errors.As(err, &opErr) + if errors.As(err, &opErr) { + return true + } + + // Only codes that can result from transient peer or transport conditions + // are safe to retry. Protocol failures must remain terminal. + var streamErr http2.StreamError + if errors.As(err, &streamErr) && isTransientHTTP2Error(streamErr.Code) { + return true + } + var streamErrPtr *http2.StreamError + if errors.As(err, &streamErrPtr) && isTransientHTTP2Error(streamErrPtr.Code) { + return true + } + var goAwayErr http2.GoAwayError + if errors.As(err, &goAwayErr) && isTransientHTTP2Error(goAwayErr.ErrCode) { + return true + } + var goAwayErrPtr *http2.GoAwayError + return errors.As(err, &goAwayErrPtr) && isTransientHTTP2Error(goAwayErrPtr.ErrCode) +} + +func isTransientHTTP2Error(code http2.ErrCode) bool { + switch code { + case http2.ErrCodeNo, + http2.ErrCodeInternal, + http2.ErrCodeRefusedStream, + http2.ErrCodeCancel, + http2.ErrCodeEnhanceYourCalm: + return true + default: + return false + } } diff --git a/coderd/util/xnet/xnet_test.go b/coderd/util/xnet/xnet_test.go index cc731ace455..83e91fff453 100644 --- a/coderd/util/xnet/xnet_test.go +++ b/coderd/util/xnet/xnet_test.go @@ -4,10 +4,14 @@ import ( "context" "io" "net" + "net/http" + "net/http/httptest" "syscall" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/net/http2" "golang.org/x/xerrors" "github.com/coder/coder/v2/coderd/util/xnet" @@ -32,4 +36,54 @@ func TestIsConnectionError(t *testing.T) { require.True(t, xnet.IsConnectionError(io.ErrUnexpectedEOF)) require.True(t, xnet.IsConnectionError(xerrors.Errorf("write: %w", syscall.EPIPE))) require.True(t, xnet.IsConnectionError(&net.OpError{Op: "read", Err: syscall.ECONNRESET})) + + for _, err := range []error{ + http2.StreamError{Code: http2.ErrCodeNo}, + &http2.StreamError{Code: http2.ErrCodeInternal}, + http2.StreamError{Code: http2.ErrCodeRefusedStream}, + &http2.StreamError{Code: http2.ErrCodeCancel}, + http2.GoAwayError{ErrCode: http2.ErrCodeEnhanceYourCalm}, + &http2.GoAwayError{ErrCode: http2.ErrCodeInternal}, + } { + require.True(t, xnet.IsConnectionError(err), err) + } + require.False(t, xnet.IsConnectionError(http2.StreamError{Code: http2.ErrCodeProtocol})) + require.False(t, xnet.IsConnectionError(&http2.GoAwayError{ErrCode: http2.ErrCodeProtocol})) +} + +func TestIsConnectionErrorHTTPResponseAbort(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + enableHTTP2 bool + proto string + }{ + {name: "http1", proto: "HTTP/1.1"}, + {name: "http2", enableHTTP2: true, proto: "HTTP/2.0"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write([]byte(`{"partial":`)) + assert.NoError(t, err) + assert.NoError(t, http.NewResponseController(w).Flush()) + panic(http.ErrAbortHandler) + })) + server.EnableHTTP2 = tc.enableHTTP2 + server.StartTLS() + t.Cleanup(server.Close) + + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil) + require.NoError(t, err) + response, err := server.Client().Do(request) + require.NoError(t, err) + require.Equal(t, tc.proto, response.Proto) + _, err = io.ReadAll(response.Body) + require.Error(t, err) + require.NoError(t, response.Body.Close()) + require.True(t, xnet.IsConnectionError(err), err) + }) + } } diff --git a/coderd/x/agenthooks/dispatch/dispatcher_internal_test.go b/coderd/x/agenthooks/dispatch/dispatcher_internal_test.go index 78527720a75..320fd33b14c 100644 --- a/coderd/x/agenthooks/dispatch/dispatcher_internal_test.go +++ b/coderd/x/agenthooks/dispatch/dispatcher_internal_test.go @@ -252,6 +252,51 @@ func TestDispatcherRetriesConnectionErrorWithSameJTI(t *testing.T) { require.Equal(t, event.ChatID, chatID) } +func TestDispatcherRetriesHTTP2AbortWithSameDispatchID(t *testing.T) { + t.Parallel() + + event := newTestEvent(t, agenthooks.EventPostCompact, agenthooks.PostCompactData{}) + type attemptIdentity struct { + jti uuid.UUID + dispatchID uuid.UUID + } + identities := make(chan attemptIdentity, 2) + var attempts atomic.Int32 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, 2, r.ProtoMajor) + body, err := io.ReadAll(r.Body) + assert.NoError(t, err) + claims, err := agenthooks.Verify(r.Header.Get("Authorization"), []byte(testSecret)) + assert.NoError(t, err) + var request agenthooks.Request + assert.NoError(t, json.Unmarshal(body, &request)) + identities <- attemptIdentity{jti: claims.JTI, dispatchID: request.Meta.DispatchID} + + if attempts.Add(1) == 1 { + _, err = w.Write([]byte(`{"partial":`)) + assert.NoError(t, err) + assert.NoError(t, http.NewResponseController(w).Flush()) + panic(http.ErrAbortHandler) + } + _, err = w.Write([]byte(`{}`)) + assert.NoError(t, err) + })) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + + _, dispatchID, err := newTestDispatcher(t, server.Client(), server.URL, time.Second).Dispatch( + testutil.Context(t, testutil.WaitLong), event, + ) + require.NoError(t, err) + require.Equal(t, int32(2), attempts.Load()) + first := <-identities + second := <-identities + require.Equal(t, first, second) + require.Equal(t, dispatchID, first.jti) + require.Equal(t, dispatchID, first.dispatchID) +} + func TestDispatcherRetriesMidBodyConnectionError(t *testing.T) { t.Parallel() From f6a64ff5bf9f9b316e198099528b1c71d1a6f12b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:53:33 +0000 Subject: [PATCH 2/6] fix(coderd/x/agenthooks): flatten dispatch log errors --- coderd/x/agenthooks/dispatch/dispatcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/x/agenthooks/dispatch/dispatcher.go b/coderd/x/agenthooks/dispatch/dispatcher.go index f8ec2018d56..db7e1566e1c 100644 --- a/coderd/x/agenthooks/dispatch/dispatcher.go +++ b/coderd/x/agenthooks/dispatch/dispatcher.go @@ -231,7 +231,7 @@ func (d *Dispatcher) finish( slog.F("dispatch_id", dispatchID), slog.F("event", event.Type), slog.F("result", outcome.result), - slog.Error(outcome.err), + slog.F("error", outcome.err.Error()), ) } else { d.logger.Debug(context.WithoutCancel(ctx), "chat hook dispatched", From 051c0b349358eb413e6b986301908dd5c9b86d93 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:56:47 +0000 Subject: [PATCH 3/6] fix(site/src/pages/AgentsPage): label denied execute calls as failed --- .../components/ChatElements/tools/ExecuteTool.tsx | 11 ++++++++++- .../components/ChatElements/tools/Tool.stories.tsx | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/site/src/pages/AgentsPage/components/ChatElements/tools/ExecuteTool.tsx b/site/src/pages/AgentsPage/components/ChatElements/tools/ExecuteTool.tsx index 7389610ce61..128a18f613e 100644 --- a/site/src/pages/AgentsPage/components/ChatElements/tools/ExecuteTool.tsx +++ b/site/src/pages/AgentsPage/components/ChatElements/tools/ExecuteTool.tsx @@ -74,6 +74,8 @@ export const ExecuteTool: React.FC = ({ modelIntent, parsedCommands, durationLabel, + isRunning, + isError, }); const defaultView = resolveAgentDisplayState( shellToolDisplayMode, @@ -159,6 +161,8 @@ type ShellCommandLineInput = { modelIntent?: string; parsedCommands?: readonly string[][]; durationLabel: string; + isRunning: boolean; + isError: boolean; }; const getShellCommandLine = ({ @@ -166,6 +170,8 @@ const getShellCommandLine = ({ modelIntent, parsedCommands, durationLabel, + isRunning, + isError, }: ShellCommandLineInput): { commandLabel: string; durationSuffix: string } => { const intentLabel = sanitizeExecuteModelIntent(modelIntent, command); const summary = @@ -173,9 +179,12 @@ const getShellCommandLine = ({ ? summarizeParsedCommands(parsedCommands) : ""; const commandDisplay = summary || command; - const commandLabel = intentLabel + let commandLabel = intentLabel ? `${intentLabel} using ${commandDisplay}` : `Ran ${commandDisplay}`; + if (!isRunning && isError) { + commandLabel = `Failed to run ${commandDisplay}`; + } return { commandLabel, diff --git a/site/src/pages/AgentsPage/components/ChatElements/tools/Tool.stories.tsx b/site/src/pages/AgentsPage/components/ChatElements/tools/Tool.stories.tsx index 7413f39a636..a0f06ac9d9a 100644 --- a/site/src/pages/AgentsPage/components/ChatElements/tools/Tool.stories.tsx +++ b/site/src/pages/AgentsPage/components/ChatElements/tools/Tool.stories.tsx @@ -414,6 +414,10 @@ export const ExecuteDeniedByHook: Story = { }, play: async ({ canvasElement }) => { const canvas = within(canvasElement); + expect(canvas.getByText(/Failed to run cat \/etc\/secrets/)).toBeVisible(); + expect( + canvas.queryByText(/Ran cat \/etc\/secrets/), + ).not.toBeInTheDocument(); await expect( canvas.getByRole("img", { name: /blocked by an external policy/, From 68c7eb7532578b2369babed01bac90c492685521 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:59:28 +0000 Subject: [PATCH 4/6] fix(site/src/pages/AgentsPage): place hook notices after messages --- .../ConversationTimeline.stories.tsx | 6 +++++- .../ChatConversation/ConversationTimeline.tsx | 16 ++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/site/src/pages/AgentsPage/components/ChatConversation/ConversationTimeline.stories.tsx b/site/src/pages/AgentsPage/components/ChatConversation/ConversationTimeline.stories.tsx index 529dc062b8e..2e120d34a57 100644 --- a/site/src/pages/AgentsPage/components/ChatConversation/ConversationTimeline.stories.tsx +++ b/site/src/pages/AgentsPage/components/ChatConversation/ConversationTimeline.stories.tsx @@ -463,7 +463,11 @@ export const LifecycleHookNoticeOnUserMessage: Story = { const notice = canvas.getByRole("note"); expect(notice).toBeVisible(); expect(within(notice).getByText("Lifecycle hook")).toBeVisible(); - expect(canvas.getByText("original prompt")).toBeVisible(); + const prompt = canvas.getByText("original prompt"); + expect(prompt).toBeVisible(); + expect( + prompt.compareDocumentPosition(notice) & Node.DOCUMENT_POSITION_FOLLOWING, + ).toBeTruthy(); const link = within(notice).getByRole("link", { name: "policy" }); expect(link).toHaveAttribute("href", "https://proxy.example.com/policy"); }, diff --git a/site/src/pages/AgentsPage/components/ChatConversation/ConversationTimeline.tsx b/site/src/pages/AgentsPage/components/ChatConversation/ConversationTimeline.tsx index 61912f4c4af..27c621d4345 100644 --- a/site/src/pages/AgentsPage/components/ChatConversation/ConversationTimeline.tsx +++ b/site/src/pages/AgentsPage/components/ChatConversation/ConversationTimeline.tsx @@ -652,14 +652,6 @@ const ChatMessageItem = memo<{ )} inert={isAfterEditingMessage ? true : undefined} > - {parsed.hookNotices.map((notice, index) => ( - - {notice} - - ))} {isUser ? ( )} + {parsed.hookNotices.map((notice, index) => ( + + {notice} + + ))} {!hideActions && (displayState.hasCopyableContent || (isUser && onEditUserMessage)) && ( From 294ebfaa87ed7b942128014622be9a47b324b4ae Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:01:01 +0000 Subject: [PATCH 5/6] docs(docs/admin/setup): clarify hook input convergence latency --- docs/admin/setup/chat-lifecycle-hooks.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/admin/setup/chat-lifecycle-hooks.md b/docs/admin/setup/chat-lifecycle-hooks.md index efe90a66ded..abe81bd72ca 100644 --- a/docs/admin/setup/chat-lifecycle-hooks.md +++ b/docs/admin/setup/chat-lifecycle-hooks.md @@ -141,7 +141,9 @@ Coder dispatches `pre_tool_use` once per tool call, after the model finishes pro Two consequences follow: -- Clients stream the model's proposed tool input while the dispatch is in flight, then converge on the stored input once the message is committed. A rewritten call briefly displays the original input. +- Clients stream the model's proposed tool input while the dispatch is in flight, then converge on the stored input once the message is committed. + A rewritten call can display the original input for up to `CODER_CHAT_HOOK_TIMEOUT`; the row spinner remains after the live **Thinking** indicator clears. + Because Coder dispatches tool calls sequentially, step latency increases with each tool call. - A tool call that is already in chat history was admitted before it was stored, so Coder executes it with the stored input instead of dispatching a second decision. If a consumer's policy changes between those two points, the change applies to later calls, not to calls already admitted. A call stored before hooks were configured is likewise not admitted retroactively, the same way an earlier prompt isn't. The per-chat debug endpoint records what the model proposed, including tool input that a consumer replaced. It reports provider behavior and is not part of the chat transcript. From 3eb2aca38b0c1b587743081b4cca8ceff45d3a03 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:21:11 +0000 Subject: [PATCH 6/6] fix(coderd/util/xnet): match only the reachable HTTP/2 stream error The pointer and GOAWAY branches were unreachable. net/http bundles its own HTTP/2 types and h2_error.go bridges only the struct form of a stream error, so errors.As never matched the other three targets. Their test coverage asserted x/net values that production never produces. Also correct the stale-input window in the hooks docs: a batch dispatches sequentially before the assistant row commits, so the bound scales with the number of tool calls rather than one timeout. --- coderd/util/xnet/xnet.go | 23 ++++++++--------------- coderd/util/xnet/xnet_test.go | 10 +++++----- docs/admin/setup/chat-lifecycle-hooks.md | 4 ++-- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/coderd/util/xnet/xnet.go b/coderd/util/xnet/xnet.go index 82aa30017cd..d0595e6c257 100644 --- a/coderd/util/xnet/xnet.go +++ b/coderd/util/xnet/xnet.go @@ -37,24 +37,17 @@ func IsConnectionError(err error) bool { return true } - // Only codes that can result from transient peer or transport conditions - // are safe to retry. Protocol failures must remain terminal. + // net/http bundles its own HTTP/2 implementation with unexported error + // types, and net/http/h2_error.go bridges only the struct form of a + // stream error to this package's type. A pointer target never matches, + // and GOAWAY errors have no such bridge. var streamErr http2.StreamError - if errors.As(err, &streamErr) && isTransientHTTP2Error(streamErr.Code) { - return true - } - var streamErrPtr *http2.StreamError - if errors.As(err, &streamErrPtr) && isTransientHTTP2Error(streamErrPtr.Code) { - return true - } - var goAwayErr http2.GoAwayError - if errors.As(err, &goAwayErr) && isTransientHTTP2Error(goAwayErr.ErrCode) { - return true - } - var goAwayErrPtr *http2.GoAwayError - return errors.As(err, &goAwayErrPtr) && isTransientHTTP2Error(goAwayErrPtr.ErrCode) + return errors.As(err, &streamErr) && isTransientHTTP2Error(streamErr.Code) } +// isTransientHTTP2Error reports whether an HTTP/2 error code can result from a +// transient peer or transport condition. Deterministic protocol failures stay +// terminal so a malformed consumer response is not retried. func isTransientHTTP2Error(code http2.ErrCode) bool { switch code { case http2.ErrCodeNo, diff --git a/coderd/util/xnet/xnet_test.go b/coderd/util/xnet/xnet_test.go index 83e91fff453..43beb0da835 100644 --- a/coderd/util/xnet/xnet_test.go +++ b/coderd/util/xnet/xnet_test.go @@ -39,16 +39,16 @@ func TestIsConnectionError(t *testing.T) { for _, err := range []error{ http2.StreamError{Code: http2.ErrCodeNo}, - &http2.StreamError{Code: http2.ErrCodeInternal}, + http2.StreamError{Code: http2.ErrCodeInternal}, http2.StreamError{Code: http2.ErrCodeRefusedStream}, - &http2.StreamError{Code: http2.ErrCodeCancel}, - http2.GoAwayError{ErrCode: http2.ErrCodeEnhanceYourCalm}, - &http2.GoAwayError{ErrCode: http2.ErrCodeInternal}, + http2.StreamError{Code: http2.ErrCodeCancel}, + http2.StreamError{Code: http2.ErrCodeEnhanceYourCalm}, + xerrors.Errorf("read response: %w", http2.StreamError{Code: http2.ErrCodeInternal}), } { require.True(t, xnet.IsConnectionError(err), err) } require.False(t, xnet.IsConnectionError(http2.StreamError{Code: http2.ErrCodeProtocol})) - require.False(t, xnet.IsConnectionError(&http2.GoAwayError{ErrCode: http2.ErrCodeProtocol})) + require.False(t, xnet.IsConnectionError(http2.StreamError{Code: http2.ErrCodeFlowControl})) } func TestIsConnectionErrorHTTPResponseAbort(t *testing.T) { diff --git a/docs/admin/setup/chat-lifecycle-hooks.md b/docs/admin/setup/chat-lifecycle-hooks.md index abe81bd72ca..1487cf7f56f 100644 --- a/docs/admin/setup/chat-lifecycle-hooks.md +++ b/docs/admin/setup/chat-lifecycle-hooks.md @@ -142,8 +142,8 @@ Coder dispatches `pre_tool_use` once per tool call, after the model finishes pro Two consequences follow: - Clients stream the model's proposed tool input while the dispatch is in flight, then converge on the stored input once the message is committed. - A rewritten call can display the original input for up to `CODER_CHAT_HOOK_TIMEOUT`; the row spinner remains after the live **Thinking** indicator clears. - Because Coder dispatches tool calls sequentially, step latency increases with each tool call. + A rewritten call keeps displaying the original input until the whole batch is admitted, and the row spinner remains after the live **Thinking** indicator clears. + Coder dispatches a batch sequentially, so that window is bounded by `CODER_CHAT_HOOK_TIMEOUT` multiplied by the number of tool calls in the step, not by a single timeout. - A tool call that is already in chat history was admitted before it was stored, so Coder executes it with the stored input instead of dispatching a second decision. If a consumer's policy changes between those two points, the change applies to later calls, not to calls already admitted. A call stored before hooks were configured is likewise not admitted retroactively, the same way an earlier prompt isn't. The per-chat debug endpoint records what the model proposed, including tool input that a consumer replaced. It reports provider behavior and is not part of the chat transcript.