diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts index 7dbb21a4963a..fb9e7a69dbdb 100644 --- a/worker/src/services/IngestionService/index.ts +++ b/worker/src/services/IngestionService/index.ts @@ -1,6 +1,12 @@ import { Cluster, Redis } from "ioredis"; import { v4 } from "uuid"; -import { Model, Price, PrismaClient, Prompt } from "@langfuse/shared"; +import { + Model, + ObservationLevel, + Price, + PrismaClient, + Prompt, +} from "@langfuse/shared"; import { ClickhouseClientType, convertDateToClickhouseDateTime, @@ -841,9 +847,10 @@ export class IngestionService { ); if ( - // Manual tokenisation when no user provided usage + // Manual tokenisation when no user provided usage and generation has not status ERROR model && - Object.keys(providedUsageDetails).length === 0 + Object.keys(providedUsageDetails).length === 0 && + observationRecord.level !== ObservationLevel.ERROR ) { let newInputCount: number | undefined; let newOutputCount: number | undefined; diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts index a64a0374761f..f907429537af 100644 --- a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts +++ b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts @@ -1201,4 +1201,61 @@ describe("Token Cost Calculation", () => { expect(generation.usage_details.output).toBe(generationUsage1.usage.output); expect(generation.usage_details.total).toBe(generationUsage1.usage.total); }); + + it("should skip tokenization and cost calculation if generation status is ERROR", async () => { + const generationUsage1 = { + model: modelName, + input: "hello world", + output: "hey whassup", + usage: null, + level: "ERROR", + }; + + const events = [ + { + id: uuidv4(), + type: "generation-create", + timestamp: new Date().toISOString(), + body: { + id: generationId, + startTime: new Date().toISOString(), + ...generationUsage1, + }, + }, + ]; + + await (mockIngestionService as any).processObservationEventList({ + projectId, + entityId: generationId, + createdAtTimestamp: new Date(), + observationEventList: events, + }); + + expect(mockAddToClickhouseWriter).toHaveBeenCalled(); + const args = mockAddToClickhouseWriter.mock.calls[0]; + const tableName = args[0]; + const generation = args[1]; + + expect(tableName).toBe("observations"); + expect(generation).toBeDefined(); + expect(generation.type).toBe("GENERATION"); + expect(generation.level).toBe("ERROR"); + + // Model name should be matched + expect(generation.internal_model_id).toBe(tokenModelData.id); + + // No user provided cost + expect(generation.provided_cost_details.input).toBeUndefined(); + expect(generation.provided_cost_details.output).toBeUndefined(); + expect(generation.provided_cost_details.total).toBeUndefined(); + + // No calculated cost + expect(generation.cost_details.input).toBeUndefined(); + expect(generation.cost_details.output).toBeUndefined(); + expect(generation.cost_details.total).toBeUndefined(); + + expect(generation.usage_details.input).toBeUndefined(); + expect(generation.usage_details.output).toBeUndefined(); + expect(generation.usage_details.total).toBeUndefined(); + }); });