fix(ingestion-service): skip tokenization for obs level ERROR (#9133)

hassiebp · web-flow · commit 9a09fa2ad08b · 2025-09-15T13:11:05.000Z
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
@@ -1,6 +1,12 @@
 import { Cluster, Redis } from "ioredis";
 import { v4 } from "uuid";
-import { Model, Price, PrismaClient, Prompt } from "@langfuse/shared";
+import {
+  Model,
+  ObservationLevel,
+  Price,
+  PrismaClient,
+  Prompt,
+} from "@langfuse/shared";
 import {
   ClickhouseClientType,
   convertDateToClickhouseDateTime,
@@ -803,9 +809,10 @@ export class IngestionService {
     );
 
     if (
-      // Manual tokenisation when no user provided usage
+      // Manual tokenisation when no user provided usage and generation has not status ERROR
       model &&
-      Object.keys(providedUsageDetails).length === 0
+      Object.keys(providedUsageDetails).length === 0 &&
+      observationRecord.level !== ObservationLevel.ERROR
     ) {
       let newInputCount: number | undefined;
       let newOutputCount: number | undefined;
diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
@@ -1201,4 +1201,61 @@ describe("Token Cost Calculation", () => {
     expect(generation.usage_details.output).toBe(generationUsage1.usage.output);
     expect(generation.usage_details.total).toBe(generationUsage1.usage.total);
   });
+
+  it("should skip tokenization and cost calculation if generation status is ERROR", async () => {
+    const generationUsage1 = {
+      model: modelName,
+      input: "hello world",
+      output: "hey whassup",
+      usage: null,
+      level: "ERROR",
+    };
+
+    const events = [
+      {
+        id: uuidv4(),
+        type: "generation-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: generationId,
+          startTime: new Date().toISOString(),
+          ...generationUsage1,
+        },
+      },
+    ];
+
+    await (mockIngestionService as any).processObservationEventList({
+      projectId,
+      entityId: generationId,
+      createdAtTimestamp: new Date(),
+      observationEventList: events,
+    });
+
+    expect(mockAddToClickhouseWriter).toHaveBeenCalled();
+    const args = mockAddToClickhouseWriter.mock.calls[0];
+    const tableName = args[0];
+    const generation = args[1];
+
+    expect(tableName).toBe("observations");
+    expect(generation).toBeDefined();
+    expect(generation.type).toBe("GENERATION");
+    expect(generation.level).toBe("ERROR");
+
+    // Model name should be matched
+    expect(generation.internal_model_id).toBe(tokenModelData.id);
+
+    // No user provided cost
+    expect(generation.provided_cost_details.input).toBeUndefined();
+    expect(generation.provided_cost_details.output).toBeUndefined();
+    expect(generation.provided_cost_details.total).toBeUndefined();
+
+    // No calculated cost
+    expect(generation.cost_details.input).toBeUndefined();
+    expect(generation.cost_details.output).toBeUndefined();
+    expect(generation.cost_details.total).toBeUndefined();
+
+    expect(generation.usage_details.input).toBeUndefined();
+    expect(generation.usage_details.output).toBeUndefined();
+    expect(generation.usage_details.total).toBeUndefined();
+  });
 });
