From 7a6a6c6baa4a120570900320994cd047a772cb7a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sat, 4 Jul 2026 13:16:26 +0000 Subject: [PATCH 1/4] feat(coderd/x/chatd): improve title generation repeatability and quality Pin temperature 0 on title and turn-status-label generation so repeated runs over the same input produce stable output. Extend the title prompt with a same-language rule and four worked examples, which measurably improved titles for vague, non-English, and error-paste inputs across gpt-4o-mini and claude-haiku-4-5. --- coderd/x/chatd/quickgen.go | 23 ++++++++++++++++++++++- coderd/x/chatd/quickgen_internal_test.go | 5 +++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/coderd/x/chatd/quickgen.go b/coderd/x/chatd/quickgen.go index 3000d4551d4..14692cb1103 100644 --- a/coderd/x/chatd/quickgen.go +++ b/coderd/x/chatd/quickgen.go @@ -21,6 +21,7 @@ import ( "cdr.dev/slog/v3" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/util/ptr" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" @@ -34,7 +35,24 @@ const titleGenerationPrompt = "Write a short title for the user's message. " + "Do not answer the user or describe the title-writing task. " + "Preserve specific identifiers such as PR numbers, repo names, file paths, function names, and error messages. " + "If the message is short or vague, stay close to the user's wording instead of inventing context. " + - "Sentence case. No quotes, emoji, markdown, or trailing punctuation." + "Write the title in the same language as the user's message. " + + "Sentence case. No quotes, emoji, markdown, or trailing punctuation.\n\n" + + "Examples:\n" + + "Message: how do I set up SSO with Okta?\n" + + "Title: Set up SSO with Okta\n\n" + + "Message: getting `pq: duplicate key value violates unique constraint` when running migrations in coderd\n" + + "Title: Fix duplicate key error in coderd migrations\n\n" + + "Message: review PR #123 in acme/webapp and flag risky changes\n" + + "Title: Review risky changes in PR #123\n\n" + + "Message: help\n" + + "Title: Help request" + +// quickgenTemperature pins title and status-label generation to +// temperature 0 so repeated runs over the same input produce stable +// output. Fantasy providers drop the setting with a call warning for +// models that reject it (OpenAI reasoning models, Anthropic thinking +// models). +const quickgenTemperature = 0.0 const ( // maxConversationContextRunes caps the conversation sample in manual @@ -504,6 +522,7 @@ func generateStructuredTitleWithUsage( SchemaName: "propose_title", SchemaDescription: "Propose a short chat title.", MaxOutputTokens: &maxOutputTokens, + Temperature: ptr.Ref(quickgenTemperature), }) return genErr }, nil) @@ -774,6 +793,7 @@ func renderManualTitlePrompt( write("- Do not answer the user or describe the title-writing task.\n") write("- Preserve specific identifiers (PR numbers, repo names, file paths, function names, error messages).\n") write("- If the conversation is short or vague, stay close to the user's wording.\n") + write("- Write the title in the same language as the user's messages.\n") write("- Sentence case. No quotes, emoji, markdown, or trailing punctuation.\n") return prompt.String() } @@ -936,6 +956,7 @@ func generateStructuredTurnStatusLabel( SchemaName: "propose_turn_status_label", SchemaDescription: "Propose a compact chat status label.", MaxOutputTokens: &maxOutputTokens, + Temperature: ptr.Ref(quickgenTemperature), }) return genErr }, nil) diff --git a/coderd/x/chatd/quickgen_internal_test.go b/coderd/x/chatd/quickgen_internal_test.go index c809ca45b74..a810944c7cb 100644 --- a/coderd/x/chatd/quickgen_internal_test.go +++ b/coderd/x/chatd/quickgen_internal_test.go @@ -342,6 +342,7 @@ func Test_renderManualTitlePrompt(t *testing.T) { require.Contains(t, prompt, "- Return only the title text in 2-8 words.") require.Contains(t, prompt, "Do not answer the user or describe the title-writing task") require.Contains(t, prompt, "stay close to the user's wording") + require.Contains(t, prompt, "same language as the user's messages") if tt.wantConversationSample { require.Contains(t, prompt, "Conversation sample:") @@ -457,6 +458,8 @@ func Test_titleGenerationPrompt_UsesSlimRules(t *testing.T) { require.Contains(t, titleGenerationPrompt, "Return only the title text in 2-8 words") require.Contains(t, titleGenerationPrompt, "Do not answer the user or describe the title-writing task") require.Contains(t, titleGenerationPrompt, "stay close to the user's wording") + require.Contains(t, titleGenerationPrompt, "same language as the user's message") + require.Contains(t, titleGenerationPrompt, "Examples:") require.NotContains(t, titleGenerationPrompt, "I am a title generator") } @@ -674,6 +677,8 @@ func TestGenerateStructuredTitleWithUsage_OpenAICompatibleRequiredToolChoice(t * body := testutil.TryReceive(t.Context(), t, requests) require.Equal(t, "required", body["tool_choice"]) + require.Equal(t, quickgenTemperature, body["temperature"], + "title generation should pin temperature for repeatable output") } func newOpenAICompatStructuredOutputServer( From 259022e5b8d7a6e0a33cbf628cf77a42c33cd93b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:04:56 +0000 Subject: [PATCH 2/4] test(coderd/x/chatd): assert pinned temperature in status-label request Also tighten the quickgenTemperature comment per review feedback. --- coderd/x/chatd/quickgen.go | 9 ++++----- coderd/x/chatd/quickgen_internal_test.go | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/coderd/x/chatd/quickgen.go b/coderd/x/chatd/quickgen.go index 14692cb1103..e0c8a3f23b0 100644 --- a/coderd/x/chatd/quickgen.go +++ b/coderd/x/chatd/quickgen.go @@ -47,11 +47,10 @@ const titleGenerationPrompt = "Write a short title for the user's message. " + "Message: help\n" + "Title: Help request" -// quickgenTemperature pins title and status-label generation to -// temperature 0 so repeated runs over the same input produce stable -// output. Fantasy providers drop the setting with a call warning for -// models that reject it (OpenAI reasoning models, Anthropic thinking -// models). +// quickgenTemperature keeps title and status-label output stable +// across repeated runs over the same input. Fantasy providers drop +// this with a call warning for models that reject it (OpenAI +// reasoning models, Anthropic thinking models). const quickgenTemperature = 0.0 const ( diff --git a/coderd/x/chatd/quickgen_internal_test.go b/coderd/x/chatd/quickgen_internal_test.go index a810944c7cb..729bf506f3c 100644 --- a/coderd/x/chatd/quickgen_internal_test.go +++ b/coderd/x/chatd/quickgen_internal_test.go @@ -789,6 +789,8 @@ func TestGenerateStructuredTurnStatusLabel(t *testing.T) { body := testutil.TryReceive(t.Context(), t, requests) require.Equal(t, "required", body["tool_choice"]) + require.Equal(t, quickgenTemperature, body["temperature"], + "status-label generation should pin temperature for repeatable output") }) t.Run("rejects narrative label", func(t *testing.T) { From 7b6ee189a80ff6163e4ccadaf88399a5424e810c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:31:55 +0000 Subject: [PATCH 3/4] feat(coderd/x/chatd): add non-English few-shot example to title prompt The four English examples created a behavioral prior toward English output that could undermine the same-language rule for non-English messages. Add a Spanish example demonstrating same-language preservation. --- coderd/x/chatd/quickgen.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/coderd/x/chatd/quickgen.go b/coderd/x/chatd/quickgen.go index e0c8a3f23b0..74ec7660a38 100644 --- a/coderd/x/chatd/quickgen.go +++ b/coderd/x/chatd/quickgen.go @@ -44,6 +44,8 @@ const titleGenerationPrompt = "Write a short title for the user's message. " + "Title: Fix duplicate key error in coderd migrations\n\n" + "Message: review PR #123 in acme/webapp and flag risky changes\n" + "Title: Review risky changes in PR #123\n\n" + + "Message: corrige el error de compilación en main.go\n" + + "Title: Corregir error de compilación en main.go\n\n" + "Message: help\n" + "Title: Help request" From cd533eefd4279974434a52ee10fb063be5db1d91 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:40:25 +0000 Subject: [PATCH 4/4] fix(coderd/x/chatd): preserve identifiers in few-shot title examples The example titles dropped the pq error name and the acme/webapp repo, teaching the model to generalize identifiers despite the preservation rule above. --- coderd/x/chatd/quickgen.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/x/chatd/quickgen.go b/coderd/x/chatd/quickgen.go index 74ec7660a38..91f37a87463 100644 --- a/coderd/x/chatd/quickgen.go +++ b/coderd/x/chatd/quickgen.go @@ -41,9 +41,9 @@ const titleGenerationPrompt = "Write a short title for the user's message. " + "Message: how do I set up SSO with Okta?\n" + "Title: Set up SSO with Okta\n\n" + "Message: getting `pq: duplicate key value violates unique constraint` when running migrations in coderd\n" + - "Title: Fix duplicate key error in coderd migrations\n\n" + + "Title: Fix pq duplicate key violation in coderd migrations\n\n" + "Message: review PR #123 in acme/webapp and flag risky changes\n" + - "Title: Review risky changes in PR #123\n\n" + + "Title: Review risky changes in acme/webapp PR #123\n\n" + "Message: corrige el error de compilación en main.go\n" + "Title: Corregir error de compilación en main.go\n\n" + "Message: help\n" +