Thanks to visit codestin.com
Credit goes to github.com

Skip to content

feat(coderd/x/chatd): improve title generation repeatability and quality #26982

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ibetitsmike merged 4 commits into from
Jul 6, 2026
Merged

feat(coderd/x/chatd): improve title generation repeatability and quality #26982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7a6a6c6
feat(coderd/x/chatd): improve title generation repeatability and quality
ibetitsmike Jul 4, 2026
259022e
test(coderd/x/chatd): assert pinned temperature in status-label request
ibetitsmike Jul 4, 2026
7b6ee18
feat(coderd/x/chatd): add non-English few-shot example to title prompt
ibetitsmike Jul 4, 2026
cd533ee
fix(coderd/x/chatd): preserve identifiers in few-shot title examples
ibetitsmike Jul 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion coderd/x/chatd/quickgen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/util/ptr"
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
Expand All @@ -34,7 +35,25 @@ const titleGenerationPrompt = "Write a short title for the user's message. " +
"Do not answer the user or describe the title-writing task. " +
"Preserve specific identifiers such as PR numbers, repo names, file paths, function names, and error messages. " +
"If the message is short or vague, stay close to the user's wording instead of inventing context. " +
"Sentence case. No quotes, emoji, markdown, or trailing punctuation."
"Write the title in the same language as the user's message. " +
"Sentence case. No quotes, emoji, markdown, or trailing punctuation.\n\n" +
"Examples:\n" +
Comment thread
ibetitsmike marked this conversation as resolved.
"Message: how do I set up SSO with Okta?\n" +
"Title: Set up SSO with Okta\n\n" +
"Message: getting `pq: duplicate key value violates unique constraint` when running migrations in coderd\n" +
Comment thread
ibetitsmike marked this conversation as resolved.
"Title: Fix pq duplicate key violation in coderd migrations\n\n" +
"Message: review PR #123 in acme/webapp and flag risky changes\n" +
"Title: Review risky changes in acme/webapp PR #123\n\n" +
"Message: corrige el error de compilación en main.go\n" +
"Title: Corregir error de compilación en main.go\n\n" +
"Message: help\n" +
"Title: Help request"

// quickgenTemperature keeps title and status-label output stable
// across repeated runs over the same input. Fantasy providers drop
// this with a call warning for models that reject it (OpenAI
// reasoning models, Anthropic thinking models).
const quickgenTemperature = 0.0

const (
// maxConversationContextRunes caps the conversation sample in manual
Expand Down Expand Up @@ -504,6 +523,7 @@ func generateStructuredTitleWithUsage(
SchemaName: "propose_title",
SchemaDescription: "Propose a short chat title.",
MaxOutputTokens: &maxOutputTokens,
Temperature: ptr.Ref(quickgenTemperature),
Comment thread
ibetitsmike marked this conversation as resolved.
})
return genErr
}, nil)
Expand Down Expand Up @@ -774,6 +794,7 @@ func renderManualTitlePrompt(
write("- Do not answer the user or describe the title-writing task.\n")
write("- Preserve specific identifiers (PR numbers, repo names, file paths, function names, error messages).\n")
write("- If the conversation is short or vague, stay close to the user's wording.\n")
write("- Write the title in the same language as the user's messages.\n")
write("- Sentence case. No quotes, emoji, markdown, or trailing punctuation.\n")
return prompt.String()
}
Expand Down Expand Up @@ -936,6 +957,7 @@ func generateStructuredTurnStatusLabel(
SchemaName: "propose_turn_status_label",
SchemaDescription: "Propose a compact chat status label.",
MaxOutputTokens: &maxOutputTokens,
Temperature: ptr.Ref(quickgenTemperature),
})
return genErr
}, nil)
Expand Down
7 changes: 7 additions & 0 deletions coderd/x/chatd/quickgen_internal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,7 @@ func Test_renderManualTitlePrompt(t *testing.T) {
require.Contains(t, prompt, "- Return only the title text in 2-8 words.")
require.Contains(t, prompt, "Do not answer the user or describe the title-writing task")
require.Contains(t, prompt, "stay close to the user's wording")
require.Contains(t, prompt, "same language as the user's messages")

if tt.wantConversationSample {
require.Contains(t, prompt, "Conversation sample:")
Expand Down Expand Up @@ -457,6 +458,8 @@ func Test_titleGenerationPrompt_UsesSlimRules(t *testing.T) {
require.Contains(t, titleGenerationPrompt, "Return only the title text in 2-8 words")
require.Contains(t, titleGenerationPrompt, "Do not answer the user or describe the title-writing task")
require.Contains(t, titleGenerationPrompt, "stay close to the user's wording")
require.Contains(t, titleGenerationPrompt, "same language as the user's message")
require.Contains(t, titleGenerationPrompt, "Examples:")
require.NotContains(t, titleGenerationPrompt, "I am a title generator")
}

Expand Down Expand Up @@ -674,6 +677,8 @@ func TestGenerateStructuredTitleWithUsage_OpenAICompatibleRequiredToolChoice(t *

body := testutil.TryReceive(t.Context(), t, requests)
require.Equal(t, "required", body["tool_choice"])
require.Equal(t, quickgenTemperature, body["temperature"],
"title generation should pin temperature for repeatable output")
}

func newOpenAICompatStructuredOutputServer(
Expand Down Expand Up @@ -784,6 +789,8 @@ func TestGenerateStructuredTurnStatusLabel(t *testing.T) {

body := testutil.TryReceive(t.Context(), t, requests)
require.Equal(t, "required", body["tool_choice"])
require.Equal(t, quickgenTemperature, body["temperature"],
"status-label generation should pin temperature for repeatable output")
})

t.Run("rejects narrative label", func(t *testing.T) {
Expand Down
Loading