diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 2734da37dda2..93a9a99d8798 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -469,41 +469,6 @@ jobs: - name: Run e2e tests run: pnpm --filter=web run test:e2e:server - validate-pr-title: - runs-on: ubuntu-latest - needs: - - pre-job - if: needs.pre-job.outputs.should_skip != 'true' && github.event_name == 'pull_request' - permissions: - statuses: write - pull-requests: read - steps: - - name: Validate PR title follows conventional commits - uses: amannn/action-semantic-pull-request@v6 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - # Configure the types of commits allowed - types: | - feat - fix - docs - style - refactor - perf - test - build - ci - chore - revert - # Allow scopes (optional) - requireScope: false - # Allow any scope (not just the ones listed above) - validateSingleCommit: false - ignoreLabels: | - bot - ignore-semantic-pull-request - all-ci-passed: # This allows us to have a branch protection rule for tests and deploys with matrix runs-on: ubuntu-latest @@ -517,7 +482,6 @@ jobs: test-docker-build, e2e-server-tests, tests-web-async, - validate-pr-title, ] if: always() steps: diff --git a/.github/workflows/validate-pr-title.yml b/.github/workflows/validate-pr-title.yml new file mode 100644 index 000000000000..3a25d9d06835 --- /dev/null +++ b/.github/workflows/validate-pr-title.yml @@ -0,0 +1,42 @@ +--- +name: "Validate PR Title" + +on: + workflow_dispatch: + merge_group: + pull_request: + branches: + - "**" + +jobs: + validate-pr-title: + runs-on: ubuntu-latest + permissions: + statuses: write + pull-requests: read + steps: + - name: Validate PR title follows conventional commits + uses: amannn/action-semantic-pull-request@v6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Configure the types of commits allowed + types: | + feat + fix + docs + style + refactor + perf + test + build + ci + chore + revert + # Allow scopes (optional) + requireScope: false + # Allow any scope (not just the ones listed above) + validateSingleCommit: false + ignoreLabels: | + bot + ignore-semantic-pull-request diff --git a/REVIEW.md b/REVIEW.md index c704fd2fd9bd..8f6f4b702ae1 100644 --- a/REVIEW.md +++ b/REVIEW.md @@ -13,3 +13,7 @@ ### Postgres - Most `schema.prisma` changes should produce a change in `packages/shared/prisma/migrations`. + +### Environment Variables + +- Environment variables should be imported from the `env.mjs/ts` file of the respective package and not from `process.env.*` to ensure validation and typing. diff --git a/package.json b/package.json index 59427bc22607..2c2a244a6a3d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "langfuse", - "version": "3.106.4", + "version": "3.107.0", "author": "engineering@langfuse.com", "license": "MIT", "private": true, diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.ts index 8a6e9a25ed3d..d673e400c007 100644 --- a/packages/shared/src/server/repositories/scores.ts +++ b/packages/shared/src/server/repositories/scores.ts @@ -34,6 +34,7 @@ import { recordDistribution } from "../instrumentation"; import { prisma } from "../../db"; import { measureAndReturn } from "../clickhouse/measureAndReturn"; import { getTimeframesTracesAMT } from "./traces"; +import { scoresColumnsTableUiColumnDefinitions } from "../tableMappings/mapScoresColumnsTable"; export const searchExistingAnnotationScore = async ( projectId: string, @@ -526,86 +527,45 @@ export const getScoresForObservations = async < })); }; -export const getRunScoresGroupedByNameSourceType = async ( - projectId: string, - datasetRunIds: string[], - timestampFilter?: FilterState, -) => { - if (datasetRunIds.length === 0) { - return []; - } - - const chFilter = timestampFilter - ? createFilterFromFilterState(timestampFilter, [ - { - uiTableName: "Timestamp", - uiTableId: "timestamp", - clickhouseTableName: "scores", - clickhouseSelect: "timestamp", - }, - ]) - : undefined; +export const getScoresGroupedByNameSourceType = async ({ + projectId, + filter, + fromTimestamp, + toTimestamp, +}: { + projectId: string; + filter: FilterCondition[]; + fromTimestamp?: Date; + toTimestamp?: Date; +}) => { + const scoresFilter = new FilterList(); + scoresFilter.push( + ...createFilterFromFilterState( + filter, + scoresColumnsTableUiColumnDefinitions, + ), + ); + const scoresFilterRes = scoresFilter.apply(); - const timestampFilterRes = chFilter - ? new FilterList(chFilter).apply() - : undefined; + // Only join dataset run items and traces if there is a dataset run items filter + const performDatasetRunItemsAndTracesJoin = scoresFilter.some( + (f) => f.clickhouseTable === "dataset_run_items_rmt", + ); // We mainly use queries like this to retrieve filter options. // Therefore, we can skip final as some inaccuracy in count is acceptable. - const query = ` - select - name, - source, - data_type - from scores s - WHERE s.project_id = {projectId: String} - ${timestampFilterRes?.query ? `AND ${timestampFilterRes.query}` : ""} - AND s.dataset_run_id IN ({datasetRunIds: Array(String)}) - GROUP BY name, source, data_type - ORDER BY count() desc - LIMIT 1000; - `; - - const rows = await queryClickhouse<{ - name: string; - source: string; - data_type: string; - }>({ - query: query, - params: { - projectId: projectId, - ...(timestampFilterRes ? timestampFilterRes.params : {}), - datasetRunIds: datasetRunIds, - }, - tags: { - feature: "tracing", - type: "score", - kind: "list", - projectId, - }, - }); - return rows.map((row) => ({ - name: row.name, - source: row.source as ScoreSourceType, - dataType: row.data_type as ScoreDataType, - })); -}; - -export const getScoresGroupedByNameSourceType = async ( - projectId: string, - timestamp: Date | undefined, -) => { - // We mainly use queries like this to retrieve filter options. - // Therefore, we can skip final as some inaccuracy in count is acceptable. const query = ` select - name, - source, - data_type - from scores s + s.name as name, + s.source as source, + s.data_type as data_type + FROM scores s + ${performDatasetRunItemsAndTracesJoin ? `JOIN dataset_run_items_rmt dri ON s.trace_id = dri.trace_id AND s.project_id = dri.project_id` : ""} WHERE s.project_id = {projectId: String} - ${timestamp ? `AND s.timestamp >= {timestamp: DateTime64(3)}` : ""} + ${scoresFilterRes?.query ? `AND ${scoresFilterRes.query}` : ""} + ${fromTimestamp ? `AND s.timestamp >= {fromTimestamp: DateTime64(3)}` : ""} + ${toTimestamp ? `AND s.timestamp <= {toTimestamp: DateTime64(3)}` : ""} GROUP BY name, source, data_type ORDER BY count() desc LIMIT 1000; @@ -619,9 +579,13 @@ export const getScoresGroupedByNameSourceType = async ( query: query, params: { projectId: projectId, - ...(timestamp - ? { timestamp: convertDateToClickhouseDateTime(timestamp) } + ...(fromTimestamp + ? { fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp) } + : {}), + ...(toTimestamp + ? { toTimestamp: convertDateToClickhouseDateTime(toTimestamp) } : {}), + ...(scoresFilterRes ? scoresFilterRes.params : {}), }, tags: { feature: "tracing", diff --git a/packages/shared/src/server/tableMappings/mapScoresColumnsTable.ts b/packages/shared/src/server/tableMappings/mapScoresColumnsTable.ts new file mode 100644 index 000000000000..00e38582e6e8 --- /dev/null +++ b/packages/shared/src/server/tableMappings/mapScoresColumnsTable.ts @@ -0,0 +1,54 @@ +import { UiColumnMappings } from "../../tableDefinitions"; + +export const scoresColumnsTableUiColumnDefinitions: UiColumnMappings = [ + // scores native columns + { + uiTableName: "Timestamp", + uiTableId: "timestamp", + clickhouseTableName: "scores", + clickhouseSelect: "timestamp", + }, + { + uiTableName: "Session ID", + uiTableId: "sessionId", + clickhouseTableName: "scores", + clickhouseSelect: 's."session_id"', + }, + { + uiTableName: "Dataset Run IDs", + uiTableId: "datasetRunIds", + clickhouseTableName: "scores", + clickhouseSelect: 's."dataset_run_id"', + }, + { + uiTableName: "Observation ID", + uiTableId: "observationId", + clickhouseTableName: "scores", + clickhouseSelect: 's."observation_id"', + }, + { + uiTableName: "Trace ID", + uiTableId: "traceId", + clickhouseTableName: "scores", + clickhouseSelect: 's."trace_id"', + }, + // require join of scores with dataset_run_items_rmt via trace_id and project_id + { + uiTableName: "Dataset Run Item Run IDs", + uiTableId: "datasetRunItemRunIds", + clickhouseTableName: "dataset_run_items_rmt", + clickhouseSelect: 'dri."dataset_run_id"', + }, + { + uiTableName: "Dataset ID", + uiTableId: "datasetId", + clickhouseTableName: "dataset_run_items_rmt", + clickhouseSelect: 'dri."dataset_id"', + }, + { + uiTableName: "Dataset Item IDs", + uiTableId: "datasetItemIds", + clickhouseTableName: "dataset_run_items_rmt", + clickhouseSelect: 'dri."dataset_item_id"', + }, +]; diff --git a/web/package.json b/web/package.json index 24632dc5408b..f89291643899 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "web", - "version": "3.106.4", + "version": "3.107.0", "private": true, "license": "MIT", "engines": { diff --git a/web/src/__tests__/async/ingestion-api.servertest.ts b/web/src/__tests__/async/ingestion-api.servertest.ts index 78dda060d958..0bb602f6ff6d 100644 --- a/web/src/__tests__/async/ingestion-api.servertest.ts +++ b/web/src/__tests__/async/ingestion-api.servertest.ts @@ -397,8 +397,9 @@ describe("/api/public/ingestion API Endpoint", () => { expect(observation!.environment).toEqual( entity.body?.environment ?? "default", ); - }); + }, 15_000); }, + 20_000, ); it.each([ diff --git a/web/src/__tests__/async/repositories/score-repository.servertest.ts b/web/src/__tests__/async/repositories/score-repository.servertest.ts index ac4b0fbf6dbc..2efaa2160785 100644 --- a/web/src/__tests__/async/repositories/score-repository.servertest.ts +++ b/web/src/__tests__/async/repositories/score-repository.servertest.ts @@ -1,8 +1,23 @@ -import { createScoresCh, getScoreById } from "@langfuse/shared/src/server"; +import { prisma } from "@langfuse/shared/src/db"; +import { + createScoresCh, + getScoreById, + getScoresGroupedByNameSourceType, + createTracesCh, + createObservationsCh, + createTrace, + createObservation, + createTraceScore, + createDatasetRunItem, + createDatasetRunItemsCh, + createDatasetRunScore, + createSessionScore, + createOrgProjectAndApiKey, +} from "@langfuse/shared/src/server"; import { v4 } from "uuid"; -const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"; describe("Clickhouse Scores Repository Test", () => { + const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"; it("should return null if no scores are found", async () => { const result = await getScoreById({ projectId, @@ -14,8 +29,7 @@ describe("Clickhouse Scores Repository Test", () => { it("should return a score if it exists", async () => { const scoreId = v4(); - // Assuming createTraceScore is a helper function to insert a score into the database - const score = { + const score = createTraceScore({ id: scoreId, project_id: projectId, trace_id: v4(), @@ -28,7 +42,7 @@ describe("Clickhouse Scores Repository Test", () => { event_ts: Date.now(), is_deleted: 0, environment: "default", - }; + }); await createScoresCh([score]); @@ -48,4 +62,338 @@ describe("Clickhouse Scores Repository Test", () => { expect(result.createdAt).toBeInstanceOf(Date); expect(result.updatedAt).toBeInstanceOf(Date); }); + + describe("getScoresGroupedByNameSourceType", () => { + it("should return empty array when no scores exist", async () => { + const emptyProjectId = v4(); + const result = await getScoresGroupedByNameSourceType({ + projectId: emptyProjectId, + filter: [], + }); + + expect(result).toEqual([]); + }); + + it("should return grouped dataset run item scores by dataset run ids", async () => { + const traceId = v4(); + + // Create dataset + const dataset = await prisma.dataset.create({ + data: { + projectId, + name: v4(), + description: v4(), + }, + }); + + // Create dataset run + const datasetRun = await prisma.datasetRuns.create({ + data: { + projectId, + name: v4(), + datasetId: dataset.id, + }, + }); + + // Create dataset run item + const datasetRunItem = createDatasetRunItem({ + project_id: projectId, + dataset_run_id: datasetRun.id, + dataset_id: dataset.id, + dataset_run_name: datasetRun.name, + dataset_item_id: v4(), + trace_id: traceId, + }); + await createDatasetRunItemsCh([datasetRunItem]); + + // Create trace + const trace = createTrace({ id: traceId, project_id: projectId }); + await createTracesCh([trace]); + + const scoreForRunItem = createTraceScore({ + project_id: projectId, + trace_id: traceId, + name: "project_score", + source: "API", + data_type: "NUMERIC", + }); + + const scoreWithoutRun = createTraceScore({ + project_id: projectId, + trace_id: v4(), + name: "other_project_score", + source: "API", + data_type: "NUMERIC", + }); + + await createScoresCh([scoreForRunItem, scoreWithoutRun]); + + const result = await getScoresGroupedByNameSourceType({ + projectId, + filter: [ + { + column: "datasetRunItemRunIds", + operator: "any of", + value: [datasetRun.id], + type: "stringOptions", + }, + ], + }); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + name: "project_score", + source: "API", + dataType: "NUMERIC", + }); + }); + + it("should return grouped dataset run scores by dataset id", async () => { + // Create dataset + const dataset = await prisma.dataset.create({ + data: { + projectId, + name: v4(), + description: v4(), + }, + }); + + // Create dataset run + const datasetRun = await prisma.datasetRuns.create({ + data: { + projectId, + name: v4(), + datasetId: dataset.id, + }, + }); + + // Create dataset run score + const datasetRunScore = createDatasetRunScore({ + project_id: projectId, + dataset_run_id: datasetRun.id, + name: "dataset_run_score", + source: "API", + data_type: "NUMERIC", + }); + + const traceScore = createTraceScore({ + project_id: projectId, + trace_id: v4(), + name: "trace_score", + source: "API", + data_type: "NUMERIC", + }); + await createScoresCh([datasetRunScore, traceScore]); + + const result = await getScoresGroupedByNameSourceType({ + projectId, + filter: [ + { + column: "datasetRunIds", + operator: "any of", + value: [datasetRun.id], + type: "stringOptions", + }, + ], + }); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + name: "dataset_run_score", + source: "API", + dataType: "NUMERIC", + }); + }); + + it("should return grouped trace scores by trace filters", async () => { + const { projectId: isolatedProjectId } = + await createOrgProjectAndApiKey(); + const traceId1 = v4(); + const traceId2 = v4(); + const sessionId = v4(); + + // Create traces + const trace1 = createTrace({ + id: traceId1, + project_id: isolatedProjectId, + }); + const trace2 = createTrace({ + id: traceId2, + project_id: isolatedProjectId, + }); + await createTracesCh([trace1, trace2]); + + // Create trace scores and other types + const traceScore = createTraceScore({ + project_id: isolatedProjectId, + trace_id: traceId1, + observation_id: null, // Trace-level score + name: "trace_accuracy", + source: "API", + data_type: "NUMERIC", + }); + + const sessionScore = createTraceScore({ + project_id: isolatedProjectId, + trace_id: null, + session_id: sessionId, + name: "session_quality", + source: "ANNOTATION", + data_type: "CATEGORICAL", + }); + + await createScoresCh([traceScore, sessionScore]); + + // Filter for trace-level scores only + const result = await getScoresGroupedByNameSourceType({ + projectId: isolatedProjectId, + filter: [ + { + column: "traceId", + operator: "is not null", + value: "", + type: "null", + }, + { + column: "observationId", + operator: "is null", + value: "", + type: "null", + }, + ], + }); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + name: "trace_accuracy", + source: "API", + dataType: "NUMERIC", + }); + }); + + it("should return grouped session scores by session filters", async () => { + const { projectId: isolatedProjectId } = + await createOrgProjectAndApiKey(); + const traceId = v4(); + const sessionId = v4(); + + // Create trace + const trace = createTrace({ id: traceId, project_id: isolatedProjectId }); + await createTracesCh([trace]); + + // Create session scores and trace scores + const sessionScore = createSessionScore({ + project_id: isolatedProjectId, + session_id: sessionId, + name: "session_rating", + source: "ANNOTATION", + data_type: "NUMERIC", + }); + + const traceScore = createTraceScore({ + project_id: isolatedProjectId, + trace_id: traceId, + observation_id: null, + name: "trace_score", + source: "API", + data_type: "NUMERIC", + }); + + await createScoresCh([sessionScore, traceScore]); + + // Filter for session-level scores only + const result = await getScoresGroupedByNameSourceType({ + projectId: isolatedProjectId, + filter: [ + { + column: "traceId", + operator: "is null", + value: "", + type: "null", + }, + { + column: "sessionId", + operator: "is not null", + value: "", + type: "null", + }, + ], + fromTimestamp: new Date(Date.now() - 5 * 60 * 1000), // 5 minutes ago + toTimestamp: new Date(), // now + }); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + name: "session_rating", + source: "ANNOTATION", + dataType: "NUMERIC", + }); + }); + + it("should return grouped observation scores by observation filters", async () => { + const { projectId: isolatedProjectId } = + await createOrgProjectAndApiKey(); + const traceId = v4(); + const observationId1 = v4(); + const observationId2 = v4(); + + // Create trace + const trace = createTrace({ id: traceId, project_id: isolatedProjectId }); + await createTracesCh([trace]); + + // Create observations + const obs1 = createObservation({ + id: observationId1, + trace_id: traceId, + project_id: isolatedProjectId, + }); + const obs2 = createObservation({ + id: observationId2, + trace_id: traceId, + project_id: isolatedProjectId, + }); + await createObservationsCh([obs1, obs2]); + + // Create observation scores and trace scores + const observationScore = createTraceScore({ + project_id: isolatedProjectId, + trace_id: traceId, + observation_id: observationId1, + name: "observation_quality", + source: "EVAL", + data_type: "CATEGORICAL", + }); + + const traceScore = createTraceScore({ + project_id: isolatedProjectId, + trace_id: traceId, + observation_id: null, + name: "trace_accuracy", + source: "API", + data_type: "NUMERIC", + }); + + await createScoresCh([observationScore, traceScore]); + + // Filter for observation-level scores only + const result = await getScoresGroupedByNameSourceType({ + projectId: isolatedProjectId, + filter: [ + { + column: "observationId", + operator: "is not null", + value: "", + type: "null", + }, + ], + }); + + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + name: "observation_quality", + source: "EVAL", + dataType: "CATEGORICAL", + }); + }); + }); }); diff --git a/web/src/components/table/use-cases/observations.tsx b/web/src/components/table/use-cases/observations.tsx index 86a023ed560a..102a869331d3 100644 --- a/web/src/components/table/use-cases/observations.tsx +++ b/web/src/components/table/use-cases/observations.tsx @@ -25,14 +25,9 @@ import { observationsTableColsWithOptions } from "@langfuse/shared"; import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState"; import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch"; import { MemoizedIOTableCell } from "../../ui/IOTableCell"; -import { - getScoreGroupColumnProps, - verifyAndPrefixScoreDataAgainstKeys, -} from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { useTableDateRange } from "@/src/hooks/useTableDateRange"; import { useDebounce } from "@/src/hooks/useDebounce"; import { type ScoreAggregate } from "@langfuse/shared"; -import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns"; import TagList from "@/src/features/tag/components/TagList"; import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder"; import { BatchExportTableButton } from "@/src/components/BatchExportTableButton"; @@ -62,6 +57,8 @@ import { useSelectAll } from "@/src/features/table/hooks/useSelectAll"; import { showSuccessToast } from "@/src/features/notifications/showSuccessToast"; import { TableActionMenu } from "@/src/features/table/components/TableActionMenu"; import { type TableAction } from "@/src/features/table/types"; +import { useScoreColumns } from "@/src/features/scores/hooks/useScoreColumns"; +import { scoreFilters } from "@/src/features/scores/lib/scoreColumns"; export type ObservationsTableRow = { // Shown by default @@ -322,11 +319,12 @@ export default function ObservationsTable({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [generations.isSuccess, generations.data]); - const { scoreColumns, scoreKeysAndProps, isColumnLoading } = - useIndividualScoreColumns({ - projectId, + const { scoreColumns, isLoading: isColumnLoading } = + useScoreColumns({ scoreColumnKey: "scores", - selectedFilterOption: selectedOption, + projectId, + filter: scoreFilters.forObservations(), + fromTimestamp: dateRange?.from, }); const transformFilterOptions = ( @@ -716,7 +714,17 @@ export default function ObservationsTable({ }, enableHiding: true, }, - { ...getScoreGroupColumnProps(isColumnLoading), columns: scoreColumns }, + { + accessorKey: "scores", + header: "Scores", + id: "scores", + enableHiding: true, + defaultHidden: true, + cell: () => { + return isColumnLoading ? : null; + }, + columns: scoreColumns, + }, { accessorKey: "endTime", id: "endTime", @@ -993,10 +1001,7 @@ export default function ObservationsTable({ startTime: generation.startTime, endTime: generation.endTime ?? undefined, timeToFirstToken: generation.timeToFirstToken ?? undefined, - scores: verifyAndPrefixScoreDataAgainstKeys( - scoreKeysAndProps, - generation.scores, - ), + scores: generation.scores, latency: generation.latency ?? undefined, totalCost: generation.totalCost ?? undefined, cost: { @@ -1025,7 +1030,7 @@ export default function ObservationsTable({ }; }) : []; - }, [generations, scoreKeysAndProps]); + }, [generations]); return ( <> diff --git a/web/src/components/table/use-cases/sessions.tsx b/web/src/components/table/use-cases/sessions.tsx index 0a4196659fe4..7852b4b0b550 100644 --- a/web/src/components/table/use-cases/sessions.tsx +++ b/web/src/components/table/use-cases/sessions.tsx @@ -39,17 +39,14 @@ import { import { useTableViewManager } from "@/src/components/table/table-view-presets/hooks/useTableViewManager"; import { Badge } from "@/src/components/ui/badge"; import { type ScoreAggregate } from "@langfuse/shared"; -import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns"; -import { - getScoreGroupColumnProps, - verifyAndPrefixScoreDataAgainstKeys, -} from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { useSelectAll } from "@/src/features/table/hooks/useSelectAll"; import { type TableAction } from "@/src/features/table/types"; import { TableActionMenu } from "@/src/features/table/components/TableActionMenu"; import { type RowSelectionState } from "@tanstack/react-table"; import { showSuccessToast } from "@/src/features/notifications/showSuccessToast"; import { TableSelectionManager } from "@/src/features/table/components/TableSelectionManager"; +import { useScoreColumns } from "@/src/features/scores/hooks/useScoreColumns"; +import { scoreFilters } from "@/src/features/scores/lib/scoreColumns"; export type SessionTableRow = { id: string; @@ -191,12 +188,12 @@ export default function SessionsTable({ }, }); - const { scoreColumns, scoreKeysAndProps, isColumnLoading } = - useIndividualScoreColumns({ + const { scoreColumns, isLoading: isColumnLoading } = + useScoreColumns({ projectId, scoreColumnKey: "scores", - selectedFilterOption: selectedOption, - cellsLoading: !sessions.data, + fromTimestamp: dateRange?.from, + filter: scoreFilters.forSessions(), }); const sessionMetrics = api.sessions.metrics.useQuery( @@ -388,7 +385,14 @@ export default function SessionsTable({ }, }, { - ...getScoreGroupColumnProps(isColumnLoading || !sessions.data), + accessorKey: "scores", + header: "Scores", + id: "scores", + enableHiding: true, + defaultHidden: true, + cell: () => { + return isColumnLoading ? : null; + }, columns: scoreColumns, }, { @@ -709,12 +713,7 @@ export default function SessionsTable({ totalTokens: session.totalTokens, traceTags: session.traceTags, environment: session.environment, - scores: session.scores - ? verifyAndPrefixScoreDataAgainstKeys( - scoreKeysAndProps, - session.scores, - ) - : undefined, + scores: session.scores, }; }), } diff --git a/web/src/components/table/use-cases/traces.tsx b/web/src/components/table/use-cases/traces.tsx index 4b1023c3a15d..2e737dfb5414 100644 --- a/web/src/components/table/use-cases/traces.tsx +++ b/web/src/components/table/use-cases/traces.tsx @@ -39,14 +39,9 @@ import { } from "@langfuse/shared"; import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch"; import { MemoizedIOTableCell } from "../../ui/IOTableCell"; -import { - getScoreGroupColumnProps, - verifyAndPrefixScoreDataAgainstKeys, -} from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { useTableDateRange } from "@/src/hooks/useTableDateRange"; import { useDebounce } from "@/src/hooks/useDebounce"; import { type ScoreAggregate } from "@langfuse/shared"; -import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns"; import { joinTableCoreAndMetrics } from "@/src/components/table/utils/joinTableCoreAndMetrics"; import { Skeleton } from "@/src/components/ui/skeleton"; import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder"; @@ -83,6 +78,8 @@ import { useTracePeekState } from "@/src/components/table/peek/hooks/useTracePee import { useTableViewManager } from "@/src/components/table/table-view-presets/hooks/useTableViewManager"; import { useFullTextSearch } from "@/src/components/table/use-cases/useFullTextSearch"; import { type TableDateRange } from "@/src/utils/date-range-utils"; +import { useScoreColumns } from "@/src/features/scores/hooks/useScoreColumns"; +import { scoreFilters } from "@/src/features/scores/lib/scoreColumns"; export type TracesTableRow = { // Shown by default @@ -327,12 +324,12 @@ export default function TracesTable({ ); const rowHeight = hideControls ? "s" : storedRowHeight; - const { scoreColumns, scoreKeysAndProps, isColumnLoading } = - useIndividualScoreColumns({ - projectId, + const { scoreColumns, isLoading: isColumnLoading } = + useScoreColumns({ scoreColumnKey: "scores", - selectedFilterOption: selectedOption, - cellsLoading: !traceMetrics.data, + projectId, + filter: scoreFilters.forTraces(), + fromTimestamp: dateRange?.from, }); const hasTraceDeletionEntitlement = useHasEntitlement("trace-deletion"); @@ -718,7 +715,16 @@ export default function TracesTable({ ? [] : [ { - ...getScoreGroupColumnProps(isColumnLoading || !traceMetrics.data), + accessorKey: "scores", + header: "Scores", + id: "scores", + enableHiding: true, + defaultHidden: true, + cell: () => { + return isColumnLoading ? ( + + ) : null; + }, columns: scoreColumns, }, ]), @@ -1088,12 +1094,7 @@ export default function TracesTable({ }, tokenDetails: trace.usageDetails, costDetails: trace.costDetails, - scores: trace.scores - ? verifyAndPrefixScoreDataAgainstKeys( - scoreKeysAndProps, - trace.scores, - ) - : undefined, + scores: trace.scores, cost: { inputCost: trace.calculatedInputCost ?? undefined, outputCost: trace.calculatedOutputCost ?? undefined, @@ -1102,7 +1103,7 @@ export default function TracesTable({ }; }) ?? []) : []; - }, [traces.isSuccess, traceRowData?.rows, scoreKeysAndProps]); + }, [traces.isSuccess, traceRowData?.rows]); const setFilterState = useDebounce(setUserFilterState); diff --git a/web/src/constants/VERSION.ts b/web/src/constants/VERSION.ts index 41a2c7706699..2ca23d61aed6 100644 --- a/web/src/constants/VERSION.ts +++ b/web/src/constants/VERSION.ts @@ -1 +1 @@ -export const VERSION = "v3.106.4"; +export const VERSION = "v3.107.0"; diff --git a/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx b/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx index bfbeb8ec4a13..39b837412623 100644 --- a/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx +++ b/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx @@ -10,7 +10,6 @@ import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row- import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder"; import { CreateOrEditAnnotationQueueButton } from "@/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton"; import { type ScoreDataType } from "@langfuse/shared"; -import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { DropdownMenu, DropdownMenuContent, @@ -24,6 +23,7 @@ import TableLink from "@/src/components/table/table-link"; import Link from "next/link"; import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess"; import { DeleteAnnotationQueueButton } from "@/src/features/annotation-queues/components/DeleteAnnotationQueueButton"; +import { getScoreDataTypeIcon } from "@/src/features/scores/lib/scoreColumns"; type RowData = { key: { diff --git a/web/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton.tsx b/web/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton.tsx index d3ad6a9e91c0..fd42f3b70d82 100644 --- a/web/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton.tsx +++ b/web/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton.tsx @@ -31,7 +31,6 @@ import { type ValidatedScoreConfig, } from "@langfuse/shared"; import { api } from "@/src/utils/api"; -import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values"; import { useRouter } from "next/router"; import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture"; @@ -47,6 +46,7 @@ import { import { ChevronDown, ChevronRight } from "lucide-react"; import { UserAssignmentSection } from "@/src/features/annotation-queues/components/UserAssignmentSection"; import { showErrorToast } from "@/src/features/notifications/showErrorToast"; +import { getScoreDataTypeIcon } from "@/src/features/scores/lib/scoreColumns"; export const CreateOrEditAnnotationQueueButton = ({ projectId, diff --git a/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx b/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx index 0fd056ecbf73..2699db955ec5 100644 --- a/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx +++ b/web/src/features/annotation-queues/pages/AnnotationQueueItems.tsx @@ -5,7 +5,6 @@ import { CardDescription } from "@/src/components/ui/card"; import { Button } from "@/src/components/ui/button"; import { ClipboardPen, Lock } from "lucide-react"; import { Badge } from "@/src/components/ui/badge"; -import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import Link from "next/link"; import { CreateOrEditAnnotationQueueButton } from "@/src/features/annotation-queues/components/CreateOrEditAnnotationQueueButton"; import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess"; @@ -19,6 +18,7 @@ import { SidePanelTitle, } from "@/src/components/ui/side-panel"; import { SubHeaderLabel } from "@/src/components/layouts/header"; +import { getScoreDataTypeIcon } from "@/src/features/scores/lib/scoreColumns"; export default function QueueItems() { const router = useRouter(); diff --git a/web/src/features/dashboard/components/ChartScores.tsx b/web/src/features/dashboard/components/ChartScores.tsx index 772ec6f20b86..6a4f64d6b0b9 100644 --- a/web/src/features/dashboard/components/ChartScores.tsx +++ b/web/src/features/dashboard/components/ChartScores.tsx @@ -12,7 +12,7 @@ import { type DashboardDateRangeAggregationOption, dashboardDateRangeAggregationSettings, } from "@/src/utils/date-range-utils"; -import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; +import { getScoreDataTypeIcon } from "@/src/features/scores/lib/scoreColumns"; import { NoDataOrLoading } from "@/src/components/NoDataOrLoading"; import { type QueryType, diff --git a/web/src/features/dashboard/components/ScoresTable.tsx b/web/src/features/dashboard/components/ScoresTable.tsx index b96b82b413ce..c2f053b7670e 100644 --- a/web/src/features/dashboard/components/ScoresTable.tsx +++ b/web/src/features/dashboard/components/ScoresTable.tsx @@ -11,7 +11,7 @@ import { RightAlignedCell } from "./RightAlignedCell"; import { LeftAlignedCell } from "@/src/features/dashboard/components/LeftAlignedCell"; import { TotalMetric } from "./TotalMetric"; import { createTracesTimeFilter } from "@/src/features/dashboard/lib/dashboard-utils"; -import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; +import { getScoreDataTypeIcon } from "@/src/features/scores/lib/scoreColumns"; import { isCategoricalDataType } from "@/src/features/scores/lib/helpers"; import { type DatabaseRow } from "@/src/server/api/services/sqlInterface"; import { NoDataOrLoading } from "@/src/components/NoDataOrLoading"; diff --git a/web/src/features/dashboard/components/score-analytics/ScoreAnalytics.tsx b/web/src/features/dashboard/components/score-analytics/ScoreAnalytics.tsx index 49481d2a1a8e..99e1ce474bbf 100644 --- a/web/src/features/dashboard/components/score-analytics/ScoreAnalytics.tsx +++ b/web/src/features/dashboard/components/score-analytics/ScoreAnalytics.tsx @@ -10,9 +10,7 @@ import { isBooleanDataType, isCategoricalDataType, isNumericDataType, - toOrderedScoresList, } from "@/src/features/scores/lib/helpers"; -import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { NumericScoreTimeSeriesChart } from "@/src/features/dashboard/components/score-analytics/NumericScoreTimeSeriesChart"; import { CategoricalScoreChart } from "@/src/features/dashboard/components/score-analytics/CategoricalScoreChart"; import { NumericScoreHistogram } from "@/src/features/dashboard/components/score-analytics/NumericScoreHistogram"; @@ -20,6 +18,10 @@ import DocPopup from "@/src/components/layouts/doc-popup"; import { NoDataOrLoading } from "@/src/components/NoDataOrLoading"; import { Flex, Text } from "@tremor/react"; import useLocalStorage from "@/src/components/useLocalStorage"; +import { + convertScoreColumnsToAnalyticsData, + getScoreDataTypeIcon, +} from "@/src/features/scores/lib/scoreColumns"; export function ScoreAnalytics(props: { className?: string; @@ -37,38 +39,22 @@ export function ScoreAnalytics(props: { [], ); - const scoreKeysAndProps = api.scores.getScoreKeysAndProps.useQuery( + const scoreKeysAndProps = api.scores.getScoreColumns.useQuery( { projectId: props.projectId, - selectedTimeOption: { option: props.agg, filterSource: "DASHBOARD" }, + fromTimestamp: props.fromTimestamp, + toTimestamp: props.toTimestamp, }, { - trpc: { - context: { - skipBatch: true, - }, - }, enabled: !props.isLoading, }, ); - const { scoreAnalyticsOptions, scoreKeyToData } = useMemo(() => { - const scoreAnalyticsOptions = scoreKeysAndProps.data - ? toOrderedScoresList(scoreKeysAndProps.data).map( - ({ key, name, dataType, source }) => ({ - key, - value: `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`, - }), - ) - : []; - - return { - scoreAnalyticsOptions, - scoreKeyToData: new Map( - scoreKeysAndProps.data?.map((obj) => [obj.key, obj]) ?? [], - ), - }; - }, [scoreKeysAndProps.data]); + const { scoreAnalyticsOptions, scoreKeyToData } = useMemo( + () => + convertScoreColumnsToAnalyticsData(scoreKeysAndProps.data?.scoreColumns), + [scoreKeysAndProps.data], + ); const scoreAnalyticsValues = scoreAnalyticsOptions?.filter((option) => selectedDashboardScoreKeys.includes(option.key), @@ -84,7 +70,7 @@ export function ScoreAnalytics(props: { headerChildren={ !scoreKeysAndProps.isPending && !props.isLoading && - Boolean(scoreKeysAndProps.data?.length) && ( + Boolean(scoreKeysAndProps.data?.scoreColumns.length) && ( { @@ -111,7 +97,7 @@ export function ScoreAnalytics(props: { ) } > - {Boolean(scoreKeysAndProps.data?.length) && + {Boolean(scoreKeysAndProps.data?.scoreColumns.length) && Boolean(scoreAnalyticsValues.length) ? (
{scoreAnalyticsValues.map(({ key: scoreKey }, index) => { @@ -190,7 +176,7 @@ export function ScoreAnalytics(props: { ); })}
- ) : Boolean(scoreKeysAndProps.data?.length) ? ( + ) : Boolean(scoreKeysAndProps.data?.scoreColumns.length) ? ( { if (!scoreKeysAndProps.data) return new Map(); return new Map( - scoreKeysAndProps.data.map(({ key, dataType, source, name }) => [ - key, - `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`, - ]), + scoreKeysAndProps.data.scoreColumns.map( + ({ key, dataType, source, name }) => [ + key, + `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`, + ], + ), ); }, [scoreKeysAndProps.data]); diff --git a/web/src/features/datasets/components/DatasetRunItemsTable.tsx b/web/src/features/datasets/components/DatasetRunItemsTable.tsx index 69c5d700baf9..74824b9e5d90 100644 --- a/web/src/features/datasets/components/DatasetRunItemsTable.tsx +++ b/web/src/features/datasets/components/DatasetRunItemsTable.tsx @@ -14,14 +14,12 @@ import { cn } from "@/src/utils/tailwind"; import { MemoizedIOTableCell } from "@/src/components/ui/IOTableCell"; import { IOTableCell } from "@/src/components/ui/IOTableCell"; import { ListTree } from "lucide-react"; -import { - getScoreGroupColumnProps, - verifyAndPrefixScoreDataAgainstKeys, -} from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { type ScoreAggregate } from "@langfuse/shared"; -import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns"; +import { useScoreColumns } from "@/src/features/scores/hooks/useScoreColumns"; import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder"; import { LocalIsoDate } from "@/src/components/LocalIsoDate"; +import { Skeleton } from "@/src/components/ui/skeleton"; +import { scoreFilters } from "@/src/features/scores/lib/scoreColumns"; export type DatasetRunItemRowData = { id: string; @@ -88,10 +86,20 @@ export function DatasetRunItemsTable( // eslint-disable-next-line react-hooks/exhaustive-deps }, [runItems.isSuccess, runItems.data]); - const { scoreColumns, scoreKeysAndProps, isColumnLoading } = - useIndividualScoreColumns({ + const { scoreColumns, isLoading: isColumnLoading } = + useScoreColumns({ projectId: props.projectId, scoreColumnKey: "scores", + filter: + "datasetRunId" in props + ? scoreFilters.forDatasetRunItems({ + datasetRunIds: [props.datasetRunId], + datasetId: props.datasetId, + }) + : scoreFilters.forDatasetItems({ + datasetItemIds: [props.datasetItemId], + datasetId: props.datasetId, + }), }); const columns: LangfuseColumnDef[] = [ @@ -178,7 +186,17 @@ export function DatasetRunItemsTable( return <>{totalCost}; }, }, - { ...getScoreGroupColumnProps(isColumnLoading), columns: scoreColumns }, + { + accessorKey: "scores", + header: "Scores", + id: "scores", + enableHiding: true, + defaultHidden: true, + cell: () => { + return isColumnLoading ? : null; + }, + columns: scoreColumns, + }, { accessorKey: "input", header: `${"datasetItemId" in props ? "Trace Input" : "Input"}`, @@ -267,10 +285,7 @@ export function DatasetRunItemsTable( observationId: item.observation?.id, } : undefined, - scores: verifyAndPrefixScoreDataAgainstKeys( - scoreKeysAndProps, - item.scores, - ), + scores: item.scores, totalCost: !!item.observation?.calculatedTotalCost ? usdFormatter(item.observation.calculatedTotalCost.toNumber()) : !!item.trace?.totalCost @@ -281,7 +296,7 @@ export function DatasetRunItemsTable( }; }) : []; - }, [runItems, scoreKeysAndProps]); + }, [runItems]); return ( <> diff --git a/web/src/features/datasets/components/DatasetRunsTable.tsx b/web/src/features/datasets/components/DatasetRunsTable.tsx index cf7a6f59c042..5b3e3ed161dc 100644 --- a/web/src/features/datasets/components/DatasetRunsTable.tsx +++ b/web/src/features/datasets/components/DatasetRunsTable.tsx @@ -15,13 +15,7 @@ import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState import { useDebounce } from "@/src/hooks/useDebounce"; import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch"; import { IOTableCell } from "@/src/components/ui/IOTableCell"; -import { - getScoreDataTypeIcon, - getScoreGroupColumnProps, - verifyAndPrefixScoreDataAgainstKeys, -} from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { type ScoreAggregate } from "@langfuse/shared"; -import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns"; import { ChevronDown, Columns3, MoreVertical, Trash } from "lucide-react"; import { DropdownMenu, @@ -61,6 +55,13 @@ import { ResizableHandle, } from "@/src/components/ui/resizable"; import useSessionStorage from "@/src/components/useSessionStorage"; +import { useScoreColumns } from "@/src/features/scores/hooks/useScoreColumns"; +import { + scoreFilters, + addPrefixToScoreKeys, + getScoreDataTypeIcon, + convertScoreColumnsToAnalyticsData, +} from "@/src/features/scores/lib/scoreColumns"; export type DatasetRunRowData = { id: string; @@ -271,34 +272,47 @@ export function DatasetRunsTable(props: { // eslint-disable-next-line react-hooks/exhaustive-deps }, [runs.isSuccess, runs.data]); - const runScoresKeysAndProps = - api.datasets.getRunLevelScoreKeysAndProps.useQuery({ + const { scoreColumns, isLoading: isColumnLoading } = + useScoreColumns({ + scoreColumnKey: "runItemScores", projectId: props.projectId, - datasetId: props.datasetId, + filter: scoreFilters.forDatasetRunItems({ + datasetRunIds: runs.data?.runs.map((r) => r.id) ?? [], + datasetId: props.datasetId, + }), + isFilterDataPending: runs.isPending, }); - const { scoreColumns, scoreKeysAndProps, isColumnLoading } = - useIndividualScoreColumns({ + const { scoreColumns: runScoreColumns, isLoading: isRunScoreColumnLoading } = + useScoreColumns({ + scoreColumnKey: "runScores", projectId: props.projectId, - scoreColumnKey: "runItemScores", - showAggregateViewOnly: false, + filter: scoreFilters.forDatasetRuns({ + datasetRunIds: runs.data?.runs.map((r) => r.id) ?? [], + }), + prefix: "Run-level", + isFilterDataPending: runs.isPending, }); - const { - scoreColumns: runScoreColumns, - scoreKeysAndProps: runScoreKeysAndProps, - isColumnLoading: isRunScoreColumnLoading, - } = useIndividualScoreColumns({ - projectId: props.projectId, - scoreColumnKey: "runScores", - showAggregateViewOnly: false, - scoreColumnPrefix: "Run-level", - scoreKeysAndPropsData: runScoresKeysAndProps.data, - }); + const scoreKeysAndProps = api.scores.getScoreColumns.useQuery( + { + projectId: props.projectId, + filter: scoreFilters.forDatasetRunItems({ + datasetRunIds: runs.data?.runs.map((r) => r.id) ?? [], + datasetId: props.datasetId, + }), + }, + { + enabled: runs.isSuccess, + }, + ); const scoreIdToName = useMemo(() => { - return new Map(scoreKeysAndProps.map((obj) => [obj.key, obj.name]) ?? []); - }, [scoreKeysAndProps]); + return new Map( + scoreKeysAndProps.data?.scoreColumns.map((obj) => [obj.key, obj.name]) ?? + [], + ); + }, [scoreKeysAndProps.data?.scoreColumns]); const runAggregatedMetrics = useMemo(() => { return transformAggregatedRunMetricsToChartData( @@ -307,21 +321,11 @@ export function DatasetRunsTable(props: { ); }, [runsMetrics.data, scoreIdToName]); - const { scoreAnalyticsOptions, scoreKeyToData } = useMemo(() => { - const scoreAnalyticsOptions = scoreKeysAndProps - ? scoreKeysAndProps.map(({ key, name, dataType, source }) => ({ - key, - value: `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`, - })) - : []; - - return { - scoreAnalyticsOptions, - scoreKeyToData: new Map( - scoreKeysAndProps.map((obj) => [obj.key, obj]) ?? [], - ), - }; - }, [scoreKeysAndProps]); + const { scoreAnalyticsOptions, scoreKeyToData } = useMemo( + () => + convertScoreColumnsToAnalyticsData(scoreKeysAndProps.data?.scoreColumns), + [scoreKeysAndProps.data?.scoreColumns], + ); useEffect(() => { setScoreOptions(scoreAnalyticsOptions); @@ -460,19 +464,27 @@ export function DatasetRunsTable(props: { }, }, { - ...getScoreGroupColumnProps(isRunScoreColumnLoading, { - accessorKey: "runScores", - header: "Run-level Scores", - id: "runScores", - }), + accessorKey: "runScores", + header: "Run-Level Scores", + id: "runScores", + enableHiding: true, + defaultHidden: true, + cell: () => { + return isRunScoreColumnLoading ? ( + + ) : null; + }, columns: runScoreColumns, }, { - ...getScoreGroupColumnProps(isColumnLoading, { - accessorKey: "runItemScores", - header: "Run Item Scores", - id: "runItemScores", - }), + accessorKey: "runItemScores", + header: "Run Item Scores", + id: "runItemScores", + enableHiding: true, + defaultHidden: true, + cell: () => { + return isColumnLoading ? : null; + }, columns: scoreColumns, }, { @@ -542,16 +554,10 @@ export function DatasetRunsTable(props: { avgTotalCost: item.avgTotalCost ? usdFormatter(item.avgTotalCost.toNumber()) : usdFormatter(0), - runItemScores: item.scores - ? verifyAndPrefixScoreDataAgainstKeys(scoreKeysAndProps, item.scores) - : undefined, + runItemScores: item.scores, runScores: item.runScores - ? verifyAndPrefixScoreDataAgainstKeys( - runScoreKeysAndProps, - item.runScores, - "Run-level", - ) - : undefined, + ? addPrefixToScoreKeys(item.runScores, "Run-level") + : {}, description: item.description ?? "", metadata: item.metadata, }; diff --git a/web/src/features/datasets/server/dataset-router.ts b/web/src/features/datasets/server/dataset-router.ts index 1b75282ab64e..a8ae2a8bc1b9 100644 --- a/web/src/features/datasets/server/dataset-router.ts +++ b/web/src/features/datasets/server/dataset-router.ts @@ -29,7 +29,6 @@ import { } from "@/src/features/datasets/server/service"; import { logger, - getRunScoresGroupedByNameSourceType, addToDeleteDatasetQueue, getDatasetRunItemsByDatasetIdCh, getDatasetRunItemsCountByDatasetIdCh, @@ -44,10 +43,7 @@ import { validateWebhookURL, } from "@langfuse/shared/src/server"; import { createId as createCuid } from "@paralleldrive/cuid2"; -import { - aggregateScores, - composeAggregateScoreKey, -} from "@/src/features/scores/lib/aggregateScores"; +import { aggregateScores } from "@/src/features/scores/lib/aggregateScores"; const formatDatasetItemData = (data: string | null | undefined) => { if (data === "") return Prisma.DbNull; @@ -1245,24 +1241,7 @@ export const datasetRouter = createTRPCRouter({ return []; } - const res = await getRunScoresGroupedByNameSourceType( - input.projectId, - datasetRuns.map((dr) => dr.id), - [ - { - column: "timestamp", - operator: ">=", - value: dataset.createdAt, - type: "datetime", - }, - ], - ); - return res.map(({ name, source, dataType }) => ({ - key: composeAggregateScoreKey({ name, source, dataType }), - name: name, - source: source, - dataType: dataType, - })); + return []; }), upsertRemoteExperiment: protectedProjectProcedure .input( diff --git a/web/src/features/scores/components/AnnotateDrawerContent.tsx b/web/src/features/scores/components/AnnotateDrawerContent.tsx index 090ac7cc739e..ddc1ea789cf9 100644 --- a/web/src/features/scores/components/AnnotateDrawerContent.tsx +++ b/web/src/features/scores/components/AnnotateDrawerContent.tsx @@ -62,7 +62,7 @@ import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-sel import { useRouter } from "next/router"; import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture"; import { cn } from "@/src/utils/tailwind"; -import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; +import { getScoreDataTypeIcon } from "@/src/features/scores/lib/scoreColumns"; import { DropdownMenuItem } from "@/src/components/ui/dropdown-menu"; import { type ScoreTarget, diff --git a/web/src/features/scores/components/ScoreDetailColumnHelpers.tsx b/web/src/features/scores/components/ScoreDetailColumnHelpers.tsx deleted file mode 100644 index e4f87dedea40..000000000000 --- a/web/src/features/scores/components/ScoreDetailColumnHelpers.tsx +++ /dev/null @@ -1,159 +0,0 @@ -import { ScoresTableCell } from "@/src/components/scores-table-cell"; -import { type LangfuseColumnDef } from "@/src/components/table/types"; -import { type ObservationsTableRow } from "@/src/components/table/use-cases/observations"; -import { type TracesTableRow } from "@/src/components/table/use-cases/traces"; -import { Skeleton } from "@/src/components/ui/skeleton"; -import { type DatasetRunItemRowData } from "@/src/features/datasets/components/DatasetRunItemsTable"; -import { type DatasetRunRowData } from "@/src/features/datasets/components/DatasetRunsTable"; -import { - type CategoricalAggregate, - type NumericAggregate, - type ScoreAggregate, -} from "@langfuse/shared"; -import type { PromptVersionTableRow } from "@/src/pages/project/[projectId]/prompts/metrics"; -import { type ScoreDataType } from "@langfuse/shared"; -import { type Row } from "@tanstack/react-table"; -import React from "react"; -import { - type ScoreData, - type TableRowTypesWithIndividualScoreColumns, -} from "@/src/features/scores/lib/types"; -import { type SessionTableRow } from "@/src/components/table/use-cases/sessions"; - -const prefixScoreColKey = ( - key: string, - prefix: "Trace" | "Generation" | "Run-level", -): string => `${prefix}-${key}`; - -export const getScoreDataTypeIcon = (dataType: ScoreDataType): string => { - switch (dataType) { - case "NUMERIC": - default: - return "#"; - case "CATEGORICAL": - return "Ⓒ"; - case "BOOLEAN": - return "Ⓑ"; - } -}; - -const parseScoreColumn = < - T extends - | TracesTableRow - | ObservationsTableRow - | DatasetRunRowData - | DatasetRunItemRowData - | PromptVersionTableRow - | SessionTableRow, ->( - col: ScoreData, - prefix?: "Trace" | "Generation" | "Run-level", -): LangfuseColumnDef => { - const { key, name, source, dataType } = col; - - if (!!prefix) { - return { - header: `${prefix}: ${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`, - accessorKey: prefixScoreColKey(key, prefix), - id: prefixScoreColKey(key, prefix), - enableHiding: true, - size: 150, - }; - } - - return { - header: `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`, - accessorKey: key, - id: key, - enableHiding: true, - size: 150, - }; -}; - -export function verifyAndPrefixScoreDataAgainstKeys( - scoreKeys: ScoreData[], - scoreData: ScoreAggregate, - prefix?: "Trace" | "Generation" | "Run-level", -): ScoreAggregate { - if (!Boolean(scoreKeys.length)) return {}; - let filteredScores: ScoreAggregate = {}; - - const getScoreKey = (key: string) => - !!prefix ? prefixScoreColKey(key, prefix) : key; - - for (const key in scoreData) { - if (scoreKeys.some((column) => column.key === key)) { - filteredScores[getScoreKey(key)] = scoreData[key]; - } - } - - return filteredScores; -} - -export const constructIndividualScoreColumns = < - T extends TableRowTypesWithIndividualScoreColumns, ->({ - scoreColumnProps, - scoreColumnKey, - showAggregateViewOnly = false, - scoreColumnPrefix, - cellsLoading = false, -}: { - scoreColumnProps: ScoreData[]; - scoreColumnKey: keyof T & string; - showAggregateViewOnly?: boolean; - scoreColumnPrefix?: "Trace" | "Generation" | "Run-level"; - cellsLoading?: boolean; -}): LangfuseColumnDef[] => { - return scoreColumnProps.map((col) => { - const { accessorKey, header, size, enableHiding } = parseScoreColumn( - col, - scoreColumnPrefix, - ); - - return { - accessorKey, - header, - size, - enableHiding, - cell: ({ row }: { row: Row }) => { - const scoresData: ScoreAggregate = row.getValue(scoreColumnKey) ?? {}; - - if (cellsLoading) return ; - - if (!Boolean(Object.keys(scoresData).length)) return null; - if (!scoresData.hasOwnProperty(accessorKey)) return null; - - const value: CategoricalAggregate | NumericAggregate | undefined = - scoresData[accessorKey]; - - if (!value) return null; - return ( - - ); - }, - }; - }); -}; - -export const getScoreGroupColumnProps = ( - isLoading: boolean, - config = { - accessorKey: "scores", - header: "Scores", - id: "scores", - }, -) => ({ - accessorKey: config.accessorKey, - header: config.header, - id: config.id, - enableHiding: true, - hideByDefault: true, - cell: () => { - return isLoading ? : null; - }, -}); diff --git a/web/src/features/scores/hooks/useIndividualScoreColumns.ts b/web/src/features/scores/hooks/useIndividualScoreColumns.ts deleted file mode 100644 index ae9e956a2f46..000000000000 --- a/web/src/features/scores/hooks/useIndividualScoreColumns.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { useMemo } from "react"; -import { api } from "@/src/utils/api"; -import { type TableRowTypesWithIndividualScoreColumns } from "@/src/features/scores/lib/types"; -import { constructIndividualScoreColumns } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; -import { type TableDateRangeOptions } from "@/src/utils/date-range-utils"; -import { toOrderedScoresList } from "@/src/features/scores/lib/helpers"; -import { type RouterOutputs } from "@/src/utils/api"; - -export function useIndividualScoreColumns< - T extends TableRowTypesWithIndividualScoreColumns, ->({ - projectId, - scoreColumnKey, - selectedFilterOption, - showAggregateViewOnly = false, - scoreColumnPrefix, - cellsLoading = false, - scoreKeysAndPropsData, -}: { - projectId: string; - scoreColumnKey: keyof T & string; - selectedFilterOption?: TableDateRangeOptions; - showAggregateViewOnly?: boolean; - scoreColumnPrefix?: "Trace" | "Generation" | "Run-level"; - cellsLoading?: boolean; - scoreKeysAndPropsData?: RouterOutputs["scores"]["getScoreKeysAndProps"]; -}) { - const scoreKeysAndProps = api.scores.getScoreKeysAndProps.useQuery( - { - projectId, - ...(selectedFilterOption - ? { - selectedTimeOption: { - option: selectedFilterOption, - filterSource: "TABLE", - }, - } - : {}), - }, - { - trpc: { - context: { - skipBatch: true, - }, - }, - refetchOnMount: false, // prevents refetching loops - }, - ); - - const relevantData = scoreKeysAndPropsData ?? scoreKeysAndProps.data; - - const scoreColumns = useMemo(() => { - return constructIndividualScoreColumns({ - scoreColumnProps: relevantData ? toOrderedScoresList(relevantData) : [], - scoreColumnKey, - scoreColumnPrefix, - showAggregateViewOnly, - cellsLoading, - }); - }, [ - relevantData, - scoreColumnKey, - showAggregateViewOnly, - scoreColumnPrefix, - cellsLoading, - ]); - - return { - scoreColumns, - scoreKeysAndProps: relevantData ?? [], - // temporary workaround to show loading state until we have full data - isColumnLoading: scoreKeysAndProps.isLoading, - }; -} diff --git a/web/src/features/scores/hooks/useScoreColumns.ts b/web/src/features/scores/hooks/useScoreColumns.ts new file mode 100644 index 000000000000..1bb9811a9347 --- /dev/null +++ b/web/src/features/scores/hooks/useScoreColumns.ts @@ -0,0 +1,98 @@ +import { useMemo } from "react"; +import { api } from "@/src/utils/api"; +import { + type ScoreDataType, + type FilterCondition, + type ScoreAggregate, +} from "@langfuse/shared"; +import { type LangfuseColumnDef } from "@/src/components/table/types"; +import { ScoresTableCell } from "@/src/components/scores-table-cell"; +import { toOrderedScoresList } from "@/src/features/scores/lib/helpers"; +import { getScoreDataTypeIcon } from "@/src/features/scores/lib/scoreColumns"; + +// Simple score column creation +function createScoreColumns>( + scoreColumns: Array<{ + key: string; + name: string; + source: string; + dataType: ScoreDataType; + }>, + scoreColumnKey: keyof T & string, + prefix?: string, +): LangfuseColumnDef[] { + return scoreColumns.map(({ key, name, source, dataType }) => { + // Apply prefix to both column ID/accessor and header + const accessorKey = prefix ? `${prefix}-${key}` : key; + const header = prefix + ? `${prefix}: ${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})` + : `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`; + + return { + accessorKey, + header, + id: accessorKey, + enableHiding: true, + size: 150, + cell: ({ row }) => { + // Handle both prefixed and non-prefixed score data access + const scoresData: ScoreAggregate = row.getValue(scoreColumnKey) ?? {}; + const value = scoresData[accessorKey]; + + if (!value) return null; + + return ScoresTableCell({ + aggregate: value, + showSingleValue: true, + hasMetadata: value.hasMetadata ?? false, + }); + }, + }; + }); +} + +export function useScoreColumns>({ + projectId, + scoreColumnKey, + filter, + fromTimestamp, + toTimestamp, + prefix, + isFilterDataPending = false, +}: { + projectId: string; + scoreColumnKey: keyof T & string; + filter?: FilterCondition[]; + fromTimestamp?: Date; + toTimestamp?: Date; + prefix?: string; + isFilterDataPending?: boolean; +}) { + const scoreColumnsQuery = api.scores.getScoreColumns.useQuery( + { + projectId, + filter: filter || [], + fromTimestamp, + toTimestamp, + }, + { + enabled: !isFilterDataPending, + }, + ); + + const scoreColumns = useMemo(() => { + if (!scoreColumnsQuery.data?.scoreColumns) return []; + + return createScoreColumns( + toOrderedScoresList(scoreColumnsQuery.data.scoreColumns), + scoreColumnKey, + prefix, + ); + }, [scoreColumnsQuery.data?.scoreColumns, scoreColumnKey, prefix]); + + return { + scoreColumns, + isLoading: scoreColumnsQuery.isPending, + error: scoreColumnsQuery.error, + }; +} diff --git a/web/src/features/scores/lib/scoreColumns.ts b/web/src/features/scores/lib/scoreColumns.ts new file mode 100644 index 000000000000..b498cec1ef35 --- /dev/null +++ b/web/src/features/scores/lib/scoreColumns.ts @@ -0,0 +1,154 @@ +import { + type ScoreAggregate, + type FilterCondition, + type ScoreDataType, + type ScoreSourceType, +} from "@langfuse/shared"; + +export const scoreFilters = { + // Filter for trace level scores + forTraces: (): FilterCondition[] => [ + { + type: "null", + column: "traceId", + operator: "is not null", + value: "", + }, + { + type: "null", + column: "observationId", + operator: "is null", + value: "", + }, + ], + + // Filter for session level scores + forSessions: (): FilterCondition[] => [ + { + type: "null", + column: "traceId", + operator: "is null", + value: "", + }, + { + type: "null", + column: "sessionId", + operator: "is not null", + value: "", + }, + ], + + // Filter for observation level scores + forObservations: (): FilterCondition[] => [ + { + type: "null", + column: "observationId", + operator: "is not null", + value: "", + }, + ], + + // Filter for dataset run level scores + forDatasetRuns: ({ + datasetRunIds, + }: { + datasetRunIds: string[]; + }): FilterCondition[] => [ + { + type: "stringOptions", + column: "datasetRunIds", + operator: "any of", + value: datasetRunIds, + }, + ], + + // Filter for dataset run item scores via dataset_run_items_rmt + forDatasetRunItems: ({ + datasetRunIds, + datasetId, + }: { + datasetRunIds: string[]; + datasetId: string; + }): FilterCondition[] => [ + { + type: "stringOptions", + column: "datasetRunItemRunIds", + operator: "any of", + value: datasetRunIds, + }, + { + type: "string", + column: "datasetId", + operator: "=", + value: datasetId, + }, + ], + + // Filter for dataset item scores via dataset_run_items_rmt + forDatasetItems: ({ + datasetItemIds, + datasetId, + }: { + datasetItemIds: string[]; + datasetId: string; + }): FilterCondition[] => [ + { + type: "stringOptions", + column: "datasetItemIds", + operator: "any of", + value: datasetItemIds, + }, + { + type: "string", + column: "datasetId", + operator: "=", + value: datasetId, + }, + ], +}; + +export const addPrefixToScoreKeys = ( + scores: ScoreAggregate, + prefix: string, +) => { + const prefixed: ScoreAggregate = {}; + for (const [key, value] of Object.entries(scores)) { + prefixed[`${prefix}-${key}`] = value; + } + return prefixed; +}; + +export const getScoreDataTypeIcon = (dataType: ScoreDataType): string => { + switch (dataType) { + case "NUMERIC": + default: + return "#"; + case "CATEGORICAL": + return "Ⓒ"; + case "BOOLEAN": + return "Ⓑ"; + } +}; + +// Utility function (could go in a utils file) +export const convertScoreColumnsToAnalyticsData = ( + scoreColumns: + | { + key: string; + name: string; + dataType: ScoreDataType; + source: ScoreSourceType; + }[] + | undefined, +) => { + const scoreAnalyticsOptions = + scoreColumns?.map(({ key, name, dataType, source }) => ({ + key, + value: `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`, + })) ?? []; + + return { + scoreAnalyticsOptions, + scoreKeyToData: new Map(scoreColumns?.map((obj) => [obj.key, obj]) ?? []), + }; +}; diff --git a/web/src/pages/project/[projectId]/datasets/[datasetId]/compare.tsx b/web/src/pages/project/[projectId]/datasets/[datasetId]/compare.tsx index fc3c8944ba5e..e79fdc19b1f5 100644 --- a/web/src/pages/project/[projectId]/datasets/[datasetId]/compare.tsx +++ b/web/src/pages/project/[projectId]/datasets/[datasetId]/compare.tsx @@ -15,12 +15,8 @@ import { import { CreateExperimentsForm } from "@/src/features/experiments/components/CreateExperimentsForm"; import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess"; import { DatasetAnalytics } from "@/src/features/datasets/components/DatasetAnalytics"; -import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { TimeseriesChart } from "@/src/features/scores/components/TimeseriesChart"; -import { - isNumericDataType, - toOrderedScoresList, -} from "@/src/features/scores/lib/helpers"; +import { isNumericDataType } from "@/src/features/scores/lib/helpers"; import { CompareViewAdapter } from "@/src/features/scores/adapters"; import { RESOURCE_METRICS, @@ -36,6 +32,11 @@ import { SidePanelTitle, } from "@/src/components/ui/side-panel"; import useLocalStorage from "@/src/components/useLocalStorage"; +import { + convertScoreColumnsToAnalyticsData, + getScoreDataTypeIcon, + scoreFilters, +} from "@/src/features/scores/lib/scoreColumns"; export default function DatasetCompare() { const router = useRouter(); @@ -92,11 +93,13 @@ export default function DatasetCompare() { }, ); - // LFE-3236: refactor to filter query to only include scores for runs in runIds - const scoreKeysAndProps = api.scores.getScoreKeysAndProps.useQuery( + const scoreKeysAndProps = api.scores.getScoreColumns.useQuery( { projectId: projectId, - selectedTimeOption: { option: "All time", filterSource: "TABLE" }, + filter: scoreFilters.forDatasetRunItems({ + datasetRunIds: runIds ?? [], + datasetId, + }), }, { enabled: runIds && runIds.length > 1, @@ -105,7 +108,8 @@ export default function DatasetCompare() { const scoreIdToName = useMemo(() => { return new Map( - scoreKeysAndProps.data?.map((obj) => [obj.key, obj.name]) ?? [], + scoreKeysAndProps.data?.scoreColumns.map((obj) => [obj.key, obj.name]) ?? + [], ); }, [scoreKeysAndProps.data]); @@ -116,23 +120,11 @@ export default function DatasetCompare() { ); }, [runMetrics.data, runIds, scoreIdToName]); - const { scoreAnalyticsOptions, scoreKeyToData } = useMemo(() => { - const scoreAnalyticsOptions = scoreKeysAndProps.data - ? toOrderedScoresList(scoreKeysAndProps.data).map( - ({ key, name, dataType, source }) => ({ - key, - value: `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`, - }), - ) - : []; - - return { - scoreAnalyticsOptions, - scoreKeyToData: new Map( - scoreKeysAndProps.data?.map((obj) => [obj.key, obj]) ?? [], - ), - }; - }, [scoreKeysAndProps.data]); + const { scoreAnalyticsOptions, scoreKeyToData } = useMemo( + () => + convertScoreColumnsToAnalyticsData(scoreKeysAndProps.data?.scoreColumns), + [scoreKeysAndProps.data], + ); const handleExperimentSettled = async (data?: { success: boolean; diff --git a/web/src/pages/project/[projectId]/prompts/metrics.tsx b/web/src/pages/project/[projectId]/prompts/metrics.tsx index 8b8d8f883362..efd8001c3e3c 100644 --- a/web/src/pages/project/[projectId]/prompts/metrics.tsx +++ b/web/src/pages/project/[projectId]/prompts/metrics.tsx @@ -12,9 +12,7 @@ import { numberFormatter, usdFormatter } from "@/src/utils/numbers"; import { formatIntervalSeconds } from "@/src/utils/dates"; import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility"; import { Skeleton } from "@/src/components/ui/skeleton"; -import { verifyAndPrefixScoreDataAgainstKeys } from "@/src/features/scores/components/ScoreDetailColumnHelpers"; import { type ScoreAggregate } from "@langfuse/shared"; -import { useIndividualScoreColumns } from "@/src/features/scores/hooks/useIndividualScoreColumns"; import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder"; import Page from "@/src/components/layouts/page"; import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav"; @@ -23,6 +21,11 @@ import { getPromptTabs, PROMPT_TABS, } from "@/src/features/navigation/utils/prompt-tabs"; +import { useScoreColumns } from "@/src/features/scores/hooks/useScoreColumns"; +import { + scoreFilters, + addPrefixToScoreKeys, +} from "@/src/features/scores/lib/scoreColumns"; export type PromptVersionTableRow = { version: number; @@ -129,25 +132,22 @@ export default function PromptVersionTable({ }, ); - const { - scoreColumns: traceScoreColumns, - scoreKeysAndProps, - isColumnLoading: isTraceColumnLoading, - } = useIndividualScoreColumns({ - projectId, - scoreColumnPrefix: "Trace", - scoreColumnKey: "traceScores", - showAggregateViewOnly: true, - }); + const { scoreColumns: traceScoreColumns, isLoading: isTraceColumnLoading } = + useScoreColumns({ + scoreColumnKey: "traceScores", + projectId: projectId, + filter: scoreFilters.forTraces(), + prefix: "Trace", + }); const { scoreColumns: generationScoreColumns, - isColumnLoading: isGenerationColumnLoading, - } = useIndividualScoreColumns({ - projectId, - scoreColumnPrefix: "Generation", + isLoading: isGenerationColumnLoading, + } = useScoreColumns({ scoreColumnKey: "generationScores", - showAggregateViewOnly: true, + projectId: projectId, + filter: scoreFilters.forObservations(), + prefix: "Generation", }); const columns: LangfuseColumnDef[] = [ @@ -361,13 +361,11 @@ export default function PromptVersionTable({ medianOutputTokens: prompt.medianOutputTokens, medianCost: prompt.medianTotalCost, generationCount: prompt.observationCount, - traceScores: verifyAndPrefixScoreDataAgainstKeys( - scoreKeysAndProps, + traceScores: addPrefixToScoreKeys( prompt.traceScores ?? {}, "Trace", ), - generationScores: verifyAndPrefixScoreDataAgainstKeys( - scoreKeysAndProps, + generationScores: addPrefixToScoreKeys( prompt.observationScores ?? {}, "Generation", ), diff --git a/web/src/server/api/routers/scores.ts b/web/src/server/api/routers/scores.ts index b372e98791e4..5d9cb141afa1 100644 --- a/web/src/server/api/routers/scores.ts +++ b/web/src/server/api/routers/scores.ts @@ -501,6 +501,9 @@ export const scoresRouter = createTRPCRouter({ return validateDbScore(clickhouseScore); }), + /** + * @deprecated, use getScoreColumns instead + */ getScoreKeysAndProps: protectedProjectProcedure .input( z.object({ @@ -510,7 +513,11 @@ export const scoresRouter = createTRPCRouter({ ) .query(async ({ input }) => { const date = getDateFromOption(input.selectedTimeOption); - const res = await getScoresGroupedByNameSourceType(input.projectId, date); + const res = await getScoresGroupedByNameSourceType({ + projectId: input.projectId, + fromTimestamp: date, + filter: [], + }); return res.map(({ name, source, dataType }) => ({ key: composeAggregateScoreKey({ name, source, dataType }), name: name, @@ -518,6 +525,34 @@ export const scoresRouter = createTRPCRouter({ dataType: dataType, })); }), + getScoreColumns: protectedProjectProcedure + .input( + z.object({ + projectId: z.string(), + filter: z.array(singleFilter).optional(), + fromTimestamp: z.date().optional(), + toTimestamp: z.date().optional(), + }), + ) + .query(async ({ input }) => { + const { projectId, filter, fromTimestamp, toTimestamp } = input; + + const groupedScores = await getScoresGroupedByNameSourceType({ + projectId, + filter: filter || [], + fromTimestamp, + toTimestamp, + }); + + const scoreColumns = groupedScores.map(({ name, source, dataType }) => ({ + key: composeAggregateScoreKey({ name, source, dataType }), + name, + source, + dataType, + })); + + return { scoreColumns }; + }), hasAny: protectedProjectProcedure .input( z.object({ diff --git a/worker/.eslintrc.js b/worker/.eslintrc.js index 71d1424720b9..c4f29421bb12 100644 --- a/worker/.eslintrc.js +++ b/worker/.eslintrc.js @@ -5,5 +5,5 @@ module.exports = { parserOptions: { project: true, }, - ignorePatterns: ["**/*test*.*"], + ignorePatterns: ["**/*test*.*", "**/worker-thread.js"], }; diff --git a/worker/package.json b/worker/package.json index 886cdb619117..6eaa5d7e8318 100644 --- a/worker/package.json +++ b/worker/package.json @@ -1,6 +1,6 @@ { "name": "worker", - "version": "3.106.4", + "version": "3.107.0", "description": "", "license": "MIT", "private": true, diff --git a/worker/src/constants/VERSION.ts b/worker/src/constants/VERSION.ts index 41a2c7706699..2ca23d61aed6 100644 --- a/worker/src/constants/VERSION.ts +++ b/worker/src/constants/VERSION.ts @@ -1 +1 @@ -export const VERSION = "v3.106.4"; +export const VERSION = "v3.107.0"; diff --git a/worker/src/env.ts b/worker/src/env.ts index f9e9397d60fd..8487ac64e816 100644 --- a/worker/src/env.ts +++ b/worker/src/env.ts @@ -266,6 +266,10 @@ const EnvSchema = z.object({ .positive() .default(2), LANGFUSE_DELETE_BATCH_SIZE: z.coerce.number().positive().default(2000), + LANGFUSE_TOKEN_COUNT_WORKER_POOL_SIZE: z.coerce + .number() + .positive() + .default(2), }); export const env: z.infer = diff --git a/worker/src/features/tokenisation/async-usage.ts b/worker/src/features/tokenisation/async-usage.ts new file mode 100644 index 000000000000..3978ee643fbd --- /dev/null +++ b/worker/src/features/tokenisation/async-usage.ts @@ -0,0 +1,181 @@ +import { Worker } from "worker_threads"; +import { Model } from "@langfuse/shared"; +import { logger } from "@langfuse/shared/src/server"; +import path from "path"; +import { env } from "../../env"; + +interface TokenCountWorkerPool { + workers: Worker[]; + currentWorkerIndex: number; + pendingRequests: Map< + string, + { + resolve: (value: number | undefined) => void; // eslint-disable-line no-unused-vars + reject: (error: Error) => void; // eslint-disable-line no-unused-vars + timeout: NodeJS.Timeout; + } + >; +} + +class TokenCountWorkerManager { + private pool: TokenCountWorkerPool; + private readonly workerPath: string; + private readonly poolSize: number; + private requestCounter = 0; + + constructor(poolSize: number) { + this.poolSize = poolSize; + // Use compiled JavaScript file + this.workerPath = path.join(__dirname, "worker-thread.js"); + this.pool = { + workers: [], + currentWorkerIndex: 0, + pendingRequests: new Map(), + }; + this.initializeWorkers(); + } + + private initializeWorkers() { + for (let i = 0; i < this.poolSize; i++) { + this.createWorker(); + } + } + + private createWorker() { + const worker = this.createWorkerWithListeners(); + this.pool.workers.push(worker); + } + + private createWorkerWithListeners(): Worker { + const worker = new Worker(this.workerPath); + + worker.on( + "message", + (data: { + id: string; + result: number | undefined; + error: string | null; + }) => { + const request = this.pool.pendingRequests.get(data.id); + if (request) { + clearTimeout(request.timeout); + this.pool.pendingRequests.delete(data.id); + + if (data.error) { + request.reject(new Error(data.error)); + } else { + request.resolve(data.result); + } + } + }, + ); + + worker.on("error", (error) => { + logger.error("Worker thread error:", error); + // Recreate worker on error + this.replaceWorker(worker); + }); + + worker.on("exit", (code) => { + if (code !== 0) { + logger.error(`Worker stopped with exit code ${code}`); + this.replaceWorker(worker); + } + }); + + return worker; + } + + private cleanupPendingRequests() { + // Reject all pending requests to provide faster error feedback + for (const [, request] of this.pool.pendingRequests.entries()) { + clearTimeout(request.timeout); + request.reject(new Error("Worker failed and is being replaced")); + } + this.pool.pendingRequests.clear(); + } + + private replaceWorker(deadWorker: Worker) { + const index = this.pool.workers.indexOf(deadWorker); + if (index !== -1) { + // Clean up any pending requests for the dead worker + this.cleanupPendingRequests(); + + // Create a new worker with proper event listeners + this.pool.workers[index] = this.createWorkerWithListeners(); + } + } + + private getNextWorker(): Worker { + const worker = this.pool.workers[this.pool.currentWorkerIndex]; + this.pool.currentWorkerIndex = + (this.pool.currentWorkerIndex + 1) % this.poolSize; + return worker; + } + + async tokenCount( + params: { model: Model; text: unknown }, + timeoutMs = 30000, + ): Promise { + return new Promise((resolve, reject) => { + const id = `token-count-${++this.requestCounter}-${Date.now()}`; + const worker = this.getNextWorker(); + + const timeout = setTimeout(() => { + this.pool.pendingRequests.delete(id); + reject( + new Error(`Token count operation timed out after ${timeoutMs}ms`), + ); + }, timeoutMs); + + this.pool.pendingRequests.set(id, { resolve, reject, timeout }); + + // Serialize the data to ensure no complex objects like Decimal are passed + const serializedParams = { + model: JSON.parse(JSON.stringify(params.model)), + text: params.text, + id, + }; + + worker.postMessage(serializedParams); + }); + } + + async terminate() { + // Clear all pending requests + for (const [, request] of this.pool.pendingRequests.entries()) { + clearTimeout(request.timeout); + request.reject(new Error("Worker pool is terminating")); + } + this.pool.pendingRequests.clear(); + + // Terminate all workers + await Promise.all(this.pool.workers.map((worker) => worker.terminate())); + this.pool.workers = []; + } +} + +// Singleton instance +let workerManager: TokenCountWorkerManager | null = null; + +export function getTokenCountWorkerManager( + poolSize?: number, +): TokenCountWorkerManager { + if (!workerManager) { + workerManager = new TokenCountWorkerManager( + poolSize ?? env.LANGFUSE_TOKEN_COUNT_WORKER_POOL_SIZE, + ); + } + return workerManager; +} + +export async function tokenCountAsync( + params: { + model: Model; + text: unknown; + }, + timeoutMs?: number, +): Promise { + const manager = getTokenCountWorkerManager(); + return manager.tokenCount(params, timeoutMs); +} diff --git a/worker/src/features/tokenisation/usage.ts b/worker/src/features/tokenisation/usage.ts index 714162514ca9..42760688e51f 100644 --- a/worker/src/features/tokenisation/usage.ts +++ b/worker/src/features/tokenisation/usage.ts @@ -10,11 +10,7 @@ import { } from "tiktoken"; import { z } from "zod/v4"; -import { - instrumentSync, - logger, - recordIncrement, -} from "@langfuse/shared/src/server"; +import { logger } from "@langfuse/shared/src/server"; const OpenAiTokenConfig = z.object({ tokenizerModel: z.string().refine(isTiktokenModel, { @@ -32,53 +28,30 @@ const OpenAiChatTokenConfig = z.object({ tokensPerName: z.number(), }); -const tokenCountMetric = "langfuse.tokenisedTokens"; - export function tokenCount(p: { model: Model; text: unknown; }): number | undefined { - return instrumentSync( - { - name: "token-count", - }, - (span) => { - if ( - p.text === null || - p.text === undefined || - (Array.isArray(p.text) && p.text.length === 0) - ) { - return undefined; - } - - if (p.model.tokenizerId === "openai") { - const count = openAiTokenCount({ - model: p.model, - text: p.text, - }); - - count ? span.setAttribute("token-count", count) : undefined; - count ? span.setAttribute("tokenizer", "openai") : undefined; - count ? recordIncrement(tokenCountMetric, count) : undefined; - - return count; - } else if (p.model.tokenizerId === "claude") { - const count = claudeTokenCount(p.text); - - count ? span.setAttribute("token-count", count) : undefined; - count ? span.setAttribute("tokenizer", "claude") : undefined; - count ? recordIncrement(tokenCountMetric, count) : undefined; - - return count; - } else { - if (p.model.tokenizerId) { - logger.error(`Unknown tokenizer ${p.model.tokenizerId}`); - } + if ( + p.text === null || + p.text === undefined || + (Array.isArray(p.text) && p.text.length === 0) + ) { + return undefined; + } - return undefined; - } - }, - ); + if (p.model.tokenizerId === "openai") { + return openAiTokenCount({ + model: p.model, + text: p.text, + }); + } else if (p.model.tokenizerId === "claude") { + return claudeTokenCount(p.text); + } + if (p.model.tokenizerId) { + logger.error(`Unknown tokenizer ${p.model.tokenizerId}`); + } + return undefined; } type ChatMessage = { diff --git a/worker/src/features/tokenisation/worker-thread.js b/worker/src/features/tokenisation/worker-thread.js new file mode 100644 index 000000000000..304f7a11857c --- /dev/null +++ b/worker/src/features/tokenisation/worker-thread.js @@ -0,0 +1,20 @@ +// Ensure to keep this file 100% compatible with worker-thread.ts + +const { parentPort } = require("worker_threads"); +const { tokenCount } = require("../../../dist/features/tokenisation/usage.js"); + +// Worker thread entry point +if (parentPort) { + parentPort.on("message", (data) => { + try { + const result = tokenCount({ model: data.model, text: data.text }); + parentPort.postMessage({ id: data.id, result, error: null }); + } catch (error) { + parentPort.postMessage({ + id: data.id, + result: undefined, + error: error instanceof Error ? error.message : "Unknown error", + }); + } + }); +} diff --git a/worker/src/features/tokenisation/worker-thread.ts b/worker/src/features/tokenisation/worker-thread.ts new file mode 100644 index 000000000000..a3d779e7fa8b --- /dev/null +++ b/worker/src/features/tokenisation/worker-thread.ts @@ -0,0 +1,20 @@ +// Ensure to keep this file 100% compatible with worker-thread.js + +const { parentPort } = require("worker_threads"); +const { tokenCount } = require("../../../dist/features/tokenisation/usage.js"); + +// Worker thread entry point +if (parentPort) { + parentPort.on("message", (data: any) => { + try { + const result = tokenCount({ model: data.model, text: data.text }); + parentPort.postMessage({ id: data.id, result, error: null }); + } catch (error) { + parentPort.postMessage({ + id: data.id, + result: undefined, + error: error instanceof Error ? error.message : "Unknown error", + }); + } + }); +} diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts index 107efc7cc8ca..7dbb21a4963a 100644 --- a/worker/src/services/IngestionService/index.ts +++ b/worker/src/services/IngestionService/index.ts @@ -39,6 +39,7 @@ import { hasNoJobConfigsCache, } from "@langfuse/shared/src/server"; +import { tokenCountAsync } from "../../features/tokenisation/async-usage"; import { tokenCount } from "../../features/tokenisation/usage"; import { ClickhouseWriter, TableName } from "../ClickhouseWriter"; import { @@ -791,7 +792,7 @@ export class IngestionService { }) : null; - const final_usage_details = this.getUsageUnits( + const final_usage_details = await this.getUsageUnits( observationRecord, internalModel, ); @@ -824,12 +825,14 @@ export class IngestionService { : []; } - private getUsageUnits( + private async getUsageUnits( observationRecord: ObservationRecordInsertType, model: Model | null | undefined, - ): Pick< - ObservationRecordInsertType, - "usage_details" | "provided_usage_details" + ): Promise< + Pick< + ObservationRecordInsertType, + "usage_details" | "provided_usage_details" + > > { const providedUsageDetails = Object.fromEntries( Object.entries(observationRecord.provided_usage_details).filter( @@ -842,14 +845,66 @@ export class IngestionService { model && Object.keys(providedUsageDetails).length === 0 ) { - const newInputCount = tokenCount({ - text: observationRecord.input, - model, - }); - const newOutputCount = tokenCount({ - text: observationRecord.output, - model, - }); + let newInputCount: number | undefined; + let newOutputCount: number | undefined; + await instrumentAsync( + { + name: "token-count", + }, + async (span) => { + try { + [newInputCount, newOutputCount] = await Promise.all([ + tokenCountAsync({ + text: observationRecord.input, + model, + }), + tokenCountAsync({ + text: observationRecord.output, + model, + }), + ]); + } catch (error) { + logger.warn( + `Async tokenization has failed. Falling back to synchronous tokenization`, + error, + ); + newInputCount = tokenCount({ + text: observationRecord.input, + model, + }); + newOutputCount = tokenCount({ + text: observationRecord.output, + model, + }); + } + + // Tracing + newInputCount + ? span.setAttribute( + "langfuse.tokenization.input-count", + newInputCount, + ) + : undefined; + newOutputCount + ? span.setAttribute( + "langfuse.tokenization.output-count", + newOutputCount, + ) + : undefined; + newInputCount || newOutputCount + ? span.setAttribute( + "langfuse.tokenization.tokenizer", + model.tokenizerId || "unknown", + ) + : undefined; + newInputCount + ? recordIncrement("langfuse.tokenisedTokens", newInputCount) + : undefined; + newOutputCount + ? recordIncrement("langfuse.tokenisedTokens", newOutputCount) + : undefined; + }, + ); logger.debug( `Tokenized observation ${observationRecord.id} with model ${model.id}, input: ${newInputCount}, output: ${newOutputCount}`, diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts index 0b751ba2e125..e9b9a186621e 100644 --- a/worker/src/utils/shutdown.ts +++ b/worker/src/utils/shutdown.ts @@ -5,6 +5,7 @@ import { ClickhouseWriter } from "../services/ClickhouseWriter"; import { setSigtermReceived } from "../features/health"; import { server } from "../index"; import { freeAllTokenizers } from "../features/tokenisation/usage"; +import { getTokenCountWorkerManager } from "../features/tokenisation/async-usage"; import { WorkerManager } from "../queues/workerManager"; import { prisma } from "@langfuse/shared/src/db"; import { BackgroundMigrationManager } from "../backgroundMigrations/backgroundMigrationManager"; @@ -39,5 +40,13 @@ export const onShutdown: NodeJS.SignalsListener = async (signal) => { freeAllTokenizers(); logger.info("All tokenizers are cleaned up from memory."); + // Shutdown tokenization worker threads + try { + await getTokenCountWorkerManager().terminate(); + logger.info("Token count worker threads have been terminated."); + } catch (error) { + logger.error("Error terminating token count worker threads", error); + } + logger.info("Shutdown complete, exiting process..."); };