Classifier: feature verb wins over debug keyword (part of getagentseal#196) (getagentseal#289)

iamtoruk · web-flow · commit 7a878f4d19e3 · 2026-05-09T22:48:11.000-07:00
Messages like "add error handling", "create an issue tracker", or "implement the 404 page" were landing in the Debugging bucket because the classifier checked DEBUG_KEYWORDS (which matches `error`, `issue`, `404`) before FEATURE_KEYWORDS in both `refineByKeywords` (tool-bearing turns) and `classifyConversation` (chat-only turns). The position of the matched word in the sentence is a much stronger intent signal than the order of the checks in code, so we now pick whichever pattern matches earliest. The new helper `firstMatchingCategory` runs each candidate regex once with `RegExp.exec` and keeps the match with the lowest `index`. Ties (rare in practice — same start position) break by the order the candidates were listed, which is `refactoring > feature > debugging` for coding turns. That ordering preserves existing behavior for plain bug reports (e.g. "login is broken, traceback below") while flipping mislabeled feature work to its correct category. 8 regression tests in `tests/classifier.test.ts` cover the mislabel cases from getagentseal#196 plus tie-break / chat-only cases. Full suite: 45 files / 609 tests, all green. Closes the activity-misattribution half of getagentseal#196. The Cursor provider attribution half (single 'cursor' project for all sessions) is addressed in a separate PR.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -40,6 +40,17 @@
   period-level `activities[]` rollup so a consumer can sum across days and
   reconcile. Closes #279.
 
+### Fixed (CLI)
+- **Activity classifier no longer mislabels feature work as debugging.**
+  Messages like "add error handling", "create an issue tracker", or
+  "implement the 404 page" used to land in the Debugging bucket because
+  the classifier checked the debug-keyword regex (which matches `error`,
+  `issue`, `404`) before the feature regex. Now the keyword that appears
+  earliest in the user message wins, so "add" beats "error", "create"
+  beats "issue", etc. A real bug report ("login is broken, traceback
+  below") still classifies as debugging because the debug word leads.
+  Fixes the activity-misattribution half of #196.
+
 ### Changed (CLI)
 - **`optimize` suggestions now declare their destination.** Every paste-style
   fix carries an explicit destination — `claude-md` (permanent project rule),
diff --git a/src/classifier.ts b/src/classifier.ts
@@ -93,12 +93,38 @@ function classifyByToolPattern(turn: ParsedTurn): TaskCategory | null {
   return null
 }
 
+/// Picks the category whose keyword pattern matches earliest in the message.
+/// On a tie (same start index) the candidate listed first in `candidates` wins,
+/// so callers control tie-break priority by ordering. Returns null when no
+/// pattern matches. The first-match heuristic fixes the long-standing problem
+/// where "add error handling" was tagged Debugging because the DEBUG regex was
+/// checked before FEATURE; now FEATURE wins because "add" appears before
+/// "error". Issue #196.
+function firstMatchingCategory(
+  text: string,
+  candidates: ReadonlyArray<{ regex: RegExp; category: TaskCategory }>,
+): TaskCategory | null {
+  let best: { index: number; order: number; category: TaskCategory } | null = null
+  for (let i = 0; i < candidates.length; i++) {
+    const c = candidates[i]!
+    const m = c.regex.exec(text)
+    if (!m) continue
+    if (!best || m.index < best.index || (m.index === best.index && i < best.order)) {
+      best = { index: m.index, order: i, category: c.category }
+    }
+  }
+  return best?.category ?? null
+}
+
 function refineByKeywords(category: TaskCategory, userMessage: string): TaskCategory {
   if (category === 'coding') {
-    if (DEBUG_KEYWORDS.test(userMessage)) return 'debugging'
-    if (REFACTOR_KEYWORDS.test(userMessage)) return 'refactoring'
-    if (FEATURE_KEYWORDS.test(userMessage)) return 'feature'
-    return 'coding'
+    // Tie-break order (when two keywords match at the same index): refactoring
+    // first because its words are the most specific, then feature, then debug.
+    return firstMatchingCategory(userMessage, [
+      { regex: REFACTOR_KEYWORDS, category: 'refactoring' },
+      { regex: FEATURE_KEYWORDS, category: 'feature' },
+      { regex: DEBUG_KEYWORDS, category: 'debugging' },
+    ]) ?? 'coding'
   }
 
   if (category === 'exploration') {
@@ -113,8 +139,14 @@ function refineByKeywords(category: TaskCategory, userMessage: string): TaskCate
 function classifyConversation(userMessage: string): TaskCategory {
   if (BRAINSTORM_KEYWORDS.test(userMessage)) return 'brainstorming'
   if (RESEARCH_KEYWORDS.test(userMessage)) return 'exploration'
-  if (DEBUG_KEYWORDS.test(userMessage)) return 'debugging'
-  if (FEATURE_KEYWORDS.test(userMessage)) return 'feature'
+  // Same first-match-wins logic as refineByKeywords so a chat-only message
+  // starting with a feature verb does not flip to debugging because of an
+  // incidental "error" or "fix" word later in the same sentence.
+  const debugOrFeature = firstMatchingCategory(userMessage, [
+    { regex: FEATURE_KEYWORDS, category: 'feature' },
+    { regex: DEBUG_KEYWORDS, category: 'debugging' },
+  ])
+  if (debugOrFeature) return debugOrFeature
   if (FILE_PATTERNS.test(userMessage)) return 'coding'
   if (SCRIPT_PATTERNS.test(userMessage)) return 'coding'
   if (URL_PATTERN.test(userMessage)) return 'exploration'
diff --git a/tests/classifier.test.ts b/tests/classifier.test.ts
@@ -101,3 +101,53 @@ describe('classifyTurn — Skill subCategory', () => {
     expect(c.subCategory).toBeUndefined()
   })
 })
+
+// Regression coverage for issue #196: feature verbs that lead a message
+// were previously hijacked into 'debugging' just because the message contained
+// an incidental "error" / "fix" / "issue" word later in the same sentence.
+// Now whichever keyword pattern matches earliest wins.
+describe('classifyTurn — feature vs debugging precedence (#196)', () => {
+  function codingTurn(userMessage: string): ParsedTurn {
+    return makeTurn([makeCall({ tools: ['Edit'] })], userMessage)
+  }
+
+  it('classifies "add error handling" as feature, not debugging', () => {
+    const c = classifyTurn(codingTurn('add error handling to the auth module'))
+    expect(c.category).toBe('feature')
+  })
+
+  it('classifies "create an issue tracker" as feature, not debugging', () => {
+    const c = classifyTurn(codingTurn('create an issue tracker page in the dashboard'))
+    expect(c.category).toBe('feature')
+  })
+
+  it('classifies "implement the 404 page" as feature, not debugging', () => {
+    const c = classifyTurn(codingTurn('implement the 404 page with a friendly redirect'))
+    expect(c.category).toBe('feature')
+  })
+
+  it('still classifies "fix the layout for the new feature" as debugging', () => {
+    const c = classifyTurn(codingTurn('fix the layout for the new feature'))
+    expect(c.category).toBe('debugging')
+  })
+
+  it('still classifies a plain bug report as debugging', () => {
+    const c = classifyTurn(codingTurn('login is broken, traceback below'))
+    expect(c.category).toBe('debugging')
+  })
+
+  it('classifies "refactor the error handling" as refactoring', () => {
+    const c = classifyTurn(codingTurn('refactor the error handling so it is cleaner'))
+    expect(c.category).toBe('refactoring')
+  })
+
+  it('chat-only message starting with "add" stays feature even with "fix" later', () => {
+    const c = classifyTurn(makeTurn([], 'add a setting page; we will fix the styles after'))
+    expect(c.category).toBe('feature')
+  })
+
+  it('chat-only message starting with "fix" stays debugging even with "add" later', () => {
+    const c = classifyTurn(makeTurn([], 'fix the bug introduced when we added the new flag'))
+    expect(c.category).toBe('debugging')
+  })
+})
