diff --git a/gateway/src/routes/internal.rs b/gateway/src/routes/internal.rs index cb168916da..63802fddbd 100644 --- a/gateway/src/routes/internal.rs +++ b/gateway/src/routes/internal.rs @@ -33,6 +33,10 @@ pub fn build_internal_non_otel_enabled_routes() -> Router { "/internal/functions/{function_name}/inference-stats/{metric_name}", get(endpoints::internal::inference_stats::get_inference_with_feedback_stats_handler), ) + .route( + "/internal/feedback/{target_id}/latest-id-by-metric", + get(endpoints::feedback::internal::get_latest_feedback_id_by_metric_handler), + ) .route( "/internal/model_inferences/{inference_id}", get(endpoints::internal::model_inferences::get_model_inferences_handler), diff --git a/internal/tensorzero-node/lib/bindings/LatestFeedbackIdByMetricResponse.ts b/internal/tensorzero-node/lib/bindings/LatestFeedbackIdByMetricResponse.ts new file mode 100644 index 0000000000..3fb1c3e37e --- /dev/null +++ b/internal/tensorzero-node/lib/bindings/LatestFeedbackIdByMetricResponse.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type LatestFeedbackIdByMetricResponse = { + feedback_id_by_metric: { [key in string]?: string }; +}; diff --git a/internal/tensorzero-node/lib/bindings/index.ts b/internal/tensorzero-node/lib/bindings/index.ts index cbf99124ec..649cd47832 100644 --- a/internal/tensorzero-node/lib/bindings/index.ts +++ b/internal/tensorzero-node/lib/bindings/index.ts @@ -155,6 +155,7 @@ export * from "./LLMJudgeIncludeConfig"; export * from "./LLMJudgeInputFormat"; export * from "./LLMJudgeOptimize"; export * from "./LLMJudgeOutputType"; +export * from "./LatestFeedbackIdByMetricResponse"; export * from "./LaunchOptimizationParams"; export * from "./LaunchOptimizationWorkflowParams"; export * from "./ListDatapointsRequest"; diff --git a/tensorzero-core/src/db/clickhouse/feedback.rs b/tensorzero-core/src/db/clickhouse/feedback.rs index a336af1aa7..83c1a2ea5e 100644 --- a/tensorzero-core/src/db/clickhouse/feedback.rs +++ b/tensorzero-core/src/db/clickhouse/feedback.rs @@ -9,7 +9,7 @@ use crate::{ feedback::{ BooleanMetricFeedbackRow, CommentFeedbackRow, CumulativeFeedbackTimeSeriesPoint, DemonstrationFeedbackRow, FeedbackBounds, FeedbackBoundsByType, FeedbackByVariant, - FeedbackRow, FloatMetricFeedbackRow, MetricWithFeedback, + FeedbackRow, FloatMetricFeedbackRow, LatestFeedbackRow, MetricWithFeedback, }, }, error::{Error, ErrorDetails}, @@ -491,6 +491,35 @@ fn build_metrics_with_feedback_query( (query, query_params) } +/// Builds the SQL query for getting the latest feedback ID for each metric for a target +fn build_latest_feedback_id_by_metric_query(target_id: Uuid) -> (String, HashMap) { + let mut query_params = HashMap::new(); + query_params.insert("target_id".to_string(), target_id.to_string()); + + let query = r" + SELECT + metric_name, + argMax(id, toUInt128(id)) as latest_id + FROM BooleanMetricFeedbackByTargetId + WHERE target_id = {target_id:String} + GROUP BY metric_name + + UNION ALL + + SELECT + metric_name, + argMax(id, toUInt128(id)) as latest_id + FROM FloatMetricFeedbackByTargetId + WHERE target_id = {target_id:String} + GROUP BY metric_name + + ORDER BY metric_name + FORMAT JSONEachRow" + .to_string(); + + (query, query_params) +} + // Implementation of FeedbackQueries trait #[async_trait] impl FeedbackQueries for ClickHouseConnectionInfo { @@ -940,6 +969,22 @@ impl FeedbackQueries for ClickHouseConnectionInfo { parse_json_rows(response.response.as_str()) } + + async fn query_latest_feedback_id_by_metric( + &self, + target_id: Uuid, + ) -> Result, Error> { + let (query, params_owned) = build_latest_feedback_id_by_metric_query(target_id); + + let query_params: HashMap<&str, &str> = params_owned + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + + let response = self.run_query_synchronous(query, &query_params).await?; + + parse_json_rows(response.response.as_str()) + } } fn parse_table_bounds(response: &str) -> Result { @@ -1952,4 +1997,24 @@ mod tests { Some(&"test_variant".to_string()) ); } + + #[test] + fn test_build_latest_feedback_id_by_metric_query() { + let target_id = Uuid::now_v7(); + let (query, query_params) = build_latest_feedback_id_by_metric_query(target_id); + + // Verify the query structure + assert_query_contains(&query, "SELECT"); + assert_query_contains(&query, "FROM BooleanMetricFeedbackByTargetId"); + assert_query_contains(&query, "FROM FloatMetricFeedbackByTargetId"); + assert_query_contains(&query, "argMax(id, toUInt128(id)) as latest_id"); + assert_query_contains(&query, "WHERE target_id = {target_id:String}"); + assert_query_contains(&query, "UNION ALL"); + assert_query_contains(&query, "GROUP BY metric_name"); + assert_query_contains(&query, "ORDER BY metric_name"); + assert_query_contains(&query, "FORMAT JSONEachRow"); + + // Verify params + assert_eq!(query_params.get("target_id"), Some(&target_id.to_string())); + } } diff --git a/tensorzero-core/src/db/feedback.rs b/tensorzero-core/src/db/feedback.rs index 1f5ea7c138..615e99e41b 100644 --- a/tensorzero-core/src/db/feedback.rs +++ b/tensorzero-core/src/db/feedback.rs @@ -6,10 +6,14 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +#[cfg(test)] +use mockall::automock; + use super::TableBounds; use crate::serde_util::deserialize_u64; #[async_trait] +#[cfg_attr(test, automock)] pub trait FeedbackQueries { /// Retrieves cumulative feedback statistics for a given metric and function, optionally filtered by variant names. async fn get_feedback_by_variant( @@ -91,6 +95,12 @@ pub trait FeedbackQueries { inference_table: &str, variant_name: Option<&str>, ) -> Result, Error>; + + /// Query the latest feedback ID for each metric for a given target + async fn query_latest_feedback_id_by_metric( + &self, + target_id: Uuid, + ) -> Result, Error>; } #[derive(Debug, Serialize, Deserialize, ts_rs::TS)] @@ -239,3 +249,9 @@ pub enum MetricType { Float, Demonstration, } + +#[derive(Debug, Deserialize)] +pub struct LatestFeedbackRow { + pub metric_name: String, + pub latest_id: String, +} diff --git a/tensorzero-core/src/endpoints/feedback/internal/latest_feedback_by_metric.rs b/tensorzero-core/src/endpoints/feedback/internal/latest_feedback_by_metric.rs new file mode 100644 index 0000000000..685ef114e7 --- /dev/null +++ b/tensorzero-core/src/endpoints/feedback/internal/latest_feedback_by_metric.rs @@ -0,0 +1,125 @@ +//! Feedback endpoint for querying feedback-related information + +use std::collections::HashMap; + +use axum::extract::{Path, State}; +use axum::{Json, debug_handler}; +use serde::{Deserialize, Serialize}; +use tracing::instrument; +use uuid::Uuid; + +use crate::db::feedback::FeedbackQueries; +use crate::error::Error; +use crate::utils::gateway::{AppState, AppStateData}; + +#[derive(Debug, Serialize, Deserialize, ts_rs::TS)] +#[ts(export)] +pub struct LatestFeedbackIdByMetricResponse { + pub feedback_id_by_metric: HashMap, +} + +/// HTTP handler for getting the latest feedback ID for each metric for a target +#[debug_handler(state = AppStateData)] +#[instrument( + name = "get_latest_feedback_id_by_metric_handler", + skip_all, + fields( + target_id = %target_id, + ) +)] +pub async fn get_latest_feedback_id_by_metric_handler( + State(app_state): AppState, + Path(target_id): Path, +) -> Result, Error> { + let response = + get_latest_feedback_id_by_metric(&app_state.clickhouse_connection_info, target_id).await?; + Ok(Json(response)) +} + +/// Core business logic for getting the latest feedback ID for each metric +pub async fn get_latest_feedback_id_by_metric( + clickhouse: &impl FeedbackQueries, + target_id: Uuid, +) -> Result { + let latest_feedback_rows = clickhouse + .query_latest_feedback_id_by_metric(target_id) + .await?; + + let feedback_id_by_metric = latest_feedback_rows + .into_iter() + .map(|row| (row.metric_name, row.latest_id)) + .collect(); + + Ok(LatestFeedbackIdByMetricResponse { + feedback_id_by_metric, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::feedback::{LatestFeedbackRow, MockFeedbackQueries}; + use std::collections::HashMap; + + #[tokio::test] + async fn test_get_latest_feedback_id_by_metric_calls_clickhouse() { + let mut mock_clickhouse = MockFeedbackQueries::new(); + + let target_id = Uuid::now_v7(); + let accuracy_id = Uuid::now_v7().to_string(); + let quality_id = Uuid::now_v7().to_string(); + + let mut expected_map = HashMap::new(); + expected_map.insert("accuracy".to_string(), accuracy_id.clone()); + expected_map.insert("quality".to_string(), quality_id.clone()); + + mock_clickhouse + .expect_query_latest_feedback_id_by_metric() + .withf(move |id| *id == target_id) + .times(1) + .returning({ + let accuracy_id = accuracy_id.clone(); + let quality_id = quality_id.clone(); + move |_| { + let rows = vec![ + LatestFeedbackRow { + metric_name: "accuracy".to_string(), + latest_id: accuracy_id.clone(), + }, + LatestFeedbackRow { + metric_name: "quality".to_string(), + latest_id: quality_id.clone(), + }, + ]; + Box::pin(async move { Ok(rows) }) + } + }); + + let result = get_latest_feedback_id_by_metric(&mock_clickhouse, target_id) + .await + .unwrap(); + + assert_eq!(result.feedback_id_by_metric, expected_map); + } + + #[tokio::test] + async fn test_get_latest_feedback_id_by_metric_empty_result() { + let mut mock_clickhouse = MockFeedbackQueries::new(); + + let target_id = Uuid::now_v7(); + let expected_map = HashMap::new(); + + mock_clickhouse + .expect_query_latest_feedback_id_by_metric() + .withf(move |id| *id == target_id) + .times(1) + .returning(move |_| Box::pin(async move { Ok(vec![]) })); + + let result = get_latest_feedback_id_by_metric(&mock_clickhouse, target_id) + .await + .unwrap(); + + assert_eq!(result.feedback_id_by_metric, expected_map); + assert!(result.feedback_id_by_metric.is_empty()); + } +} diff --git a/tensorzero-core/src/endpoints/feedback/internal/mod.rs b/tensorzero-core/src/endpoints/feedback/internal/mod.rs new file mode 100644 index 0000000000..73dea3c006 --- /dev/null +++ b/tensorzero-core/src/endpoints/feedback/internal/mod.rs @@ -0,0 +1,3 @@ +mod latest_feedback_by_metric; + +pub use latest_feedback_by_metric::*; diff --git a/tensorzero-core/src/endpoints/feedback/mod.rs b/tensorzero-core/src/endpoints/feedback/mod.rs index dec9162ef4..1b76c291c5 100644 --- a/tensorzero-core/src/endpoints/feedback/mod.rs +++ b/tensorzero-core/src/endpoints/feedback/mod.rs @@ -33,7 +33,9 @@ use tensorzero_auth::middleware::RequestApiKeyExtension; use tracing_opentelemetry::OpenTelemetrySpanExt; use super::validate_tags; + pub mod human_feedback; +pub mod internal; /// There is a potential issue here where if we write an inference and then immediately write feedback for it, /// we might not be able to find the inference in the database because it hasn't been written yet. diff --git a/tensorzero-core/tests/e2e/endpoints/internal/feedback.rs b/tensorzero-core/tests/e2e/endpoints/internal/feedback.rs new file mode 100644 index 0000000000..77fd08999b --- /dev/null +++ b/tensorzero-core/tests/e2e/endpoints/internal/feedback.rs @@ -0,0 +1,185 @@ +//! E2E tests for the feedback endpoints. + +use reqwest::Client; +use serde_json::json; +use std::collections::HashMap; +use tensorzero_core::endpoints::feedback::internal::LatestFeedbackIdByMetricResponse; +use uuid::Uuid; + +use crate::common::get_gateway_endpoint; + +/// Helper function to create an inference and return its inference_id +async fn create_inference(client: &Client, function_name: &str) -> Uuid { + let inference_payload = json!({ + "function_name": function_name, + "input": { + "system": {"assistant_name": "TestBot"}, + "messages": [{"role": "user", "content": "Hello"}] + }, + "stream": false, + }); + + let response = client + .post(get_gateway_endpoint("/inference")) + .json(&inference_payload) + .send() + .await + .unwrap(); + + assert!( + response.status().is_success(), + "Failed to create inference: {}", + response.status() + ); + + let response_json: serde_json::Value = response.json().await.unwrap(); + Uuid::parse_str(response_json["inference_id"].as_str().unwrap()).unwrap() +} + +/// Helper function to submit feedback for an inference +async fn submit_inference_feedback( + client: &Client, + inference_id: Uuid, + metric_name: &str, + value: serde_json::Value, +) -> Uuid { + let payload = json!({ + "inference_id": inference_id, + "metric_name": metric_name, + "value": value, + }); + + let response = client + .post(get_gateway_endpoint("/feedback")) + .json(&payload) + .send() + .await + .unwrap(); + + assert!( + response.status().is_success(), + "Failed to submit feedback: {}", + response.status() + ); + + let response_json: serde_json::Value = response.json().await.unwrap(); + Uuid::parse_str(response_json["feedback_id"].as_str().unwrap()).unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_get_latest_feedback_id_by_metric_with_data() { + let http_client = Client::new(); + + // Create an inference + let inference_id = create_inference(&http_client, "basic_test").await; + + // Submit feedback for task_success metric + let feedback_id = + submit_inference_feedback(&http_client, inference_id, "task_success", json!(true)).await; + + // Wait for ClickHouse to process + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + // Query latest feedback by metric for the inference + let url = get_gateway_endpoint(&format!( + "/internal/feedback/{inference_id}/latest-id-by-metric" + )); + let resp = http_client.get(url).send().await.unwrap(); + + assert!( + resp.status().is_success(), + "get_latest_feedback_id_by_metric request failed: status={:?}", + resp.status() + ); + + let response: LatestFeedbackIdByMetricResponse = resp.json().await.unwrap(); + + // Should have task_success metric + assert!( + response.feedback_id_by_metric.contains_key("task_success"), + "Expected to find task_success in response" + ); + + // Verify the feedback ID matches + assert_eq!( + response.feedback_id_by_metric.get("task_success"), + Some(&feedback_id.to_string()) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_get_latest_feedback_id_by_metric_multiple_feedback_same_metric() { + let http_client = Client::new(); + + // Create an inference + let inference_id = create_inference(&http_client, "basic_test").await; + + // Submit multiple feedback entries for the same metric + let _feedback_id_1 = + submit_inference_feedback(&http_client, inference_id, "task_success", json!(false)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + let feedback_id_2 = + submit_inference_feedback(&http_client, inference_id, "task_success", json!(true)).await; + + // Wait for ClickHouse to process + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + // Query latest feedback by metric + let url = get_gateway_endpoint(&format!( + "/internal/feedback/{inference_id}/latest-id-by-metric" + )); + let resp = http_client.get(url).send().await.unwrap(); + + assert!(resp.status().is_success()); + let response: LatestFeedbackIdByMetricResponse = resp.json().await.unwrap(); + + // Should only have the latest feedback ID (feedback_id_2) + assert_eq!( + response.feedback_id_by_metric.get("task_success"), + Some(&feedback_id_2.to_string()), + "Expected to get the latest feedback ID" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_get_latest_feedback_id_by_metric_nonexistent_target() { + let http_client = Client::new(); + + // Use a UUID that likely doesn't exist + let nonexistent_id = Uuid::now_v7(); + let url = get_gateway_endpoint(&format!( + "/internal/feedback/{nonexistent_id}/latest-id-by-metric" + )); + + let resp = http_client.get(url).send().await.unwrap(); + + assert!( + resp.status().is_success(), + "Should return success even for nonexistent target" + ); + + let response: LatestFeedbackIdByMetricResponse = resp.json().await.unwrap(); + + // Should return empty map + assert_eq!( + response.feedback_id_by_metric, + HashMap::new(), + "Expected empty map for nonexistent target" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_get_latest_feedback_id_by_metric_invalid_uuid() { + let http_client = Client::new(); + + // Use an invalid UUID + let url = get_gateway_endpoint("/internal/feedback/not-a-valid-uuid/latest-id-by-metric"); + + let resp = http_client.get(url).send().await.unwrap(); + + assert_eq!( + resp.status(), + reqwest::StatusCode::BAD_REQUEST, + "Expected 400 for invalid UUID" + ); +} diff --git a/tensorzero-core/tests/e2e/endpoints/internal/mod.rs b/tensorzero-core/tests/e2e/endpoints/internal/mod.rs index 0981948ea9..f271d3154f 100644 --- a/tensorzero-core/tests/e2e/endpoints/internal/mod.rs +++ b/tensorzero-core/tests/e2e/endpoints/internal/mod.rs @@ -2,6 +2,7 @@ mod action; mod config; mod datapoint_count; mod episodes; +mod feedback; mod functions; mod inference_metadata; mod inference_stats; diff --git a/ui/app/routes/api/inference/$inference_id/route.ts b/ui/app/routes/api/inference/$inference_id/route.ts index 5a9d76ee95..df09261263 100644 --- a/ui/app/routes/api/inference/$inference_id/route.ts +++ b/ui/app/routes/api/inference/$inference_id/route.ts @@ -1,8 +1,5 @@ import { data, type LoaderFunctionArgs } from "react-router"; -import { - pollForFeedbackItem, - queryLatestFeedbackIdByMetric, -} from "~/utils/clickhouse/feedback"; +import { pollForFeedbackItem } from "~/utils/clickhouse/feedback"; import { getNativeDatabaseClient } from "~/utils/tensorzero/native_client.server"; import { resolveModelInferences } from "~/utils/resolve.server"; import { getUsedVariants } from "~/utils/clickhouse/function"; @@ -72,7 +69,7 @@ export async function loader({ // Query these after polling completes to avoid race condition with materialized views [feedback_bounds, latestFeedbackByMetric] = await Promise.all([ dbClient.queryFeedbackBoundsByTargetId({ target_id: inference_id }), - queryLatestFeedbackIdByMetric({ target_id: inference_id }), + client.getLatestFeedbackIdByMetric(inference_id), ]); } else { // Normal case: execute all queries in parallel @@ -89,7 +86,7 @@ export async function loader({ demonstrationFeedbackPromise, dbClient.queryFeedbackBoundsByTargetId({ target_id: inference_id }), feedbackDataPromise, - queryLatestFeedbackIdByMetric({ target_id: inference_id }), + client.getLatestFeedbackIdByMetric(inference_id), ]); } diff --git a/ui/app/routes/observability/episodes/$episode_id/route.tsx b/ui/app/routes/observability/episodes/$episode_id/route.tsx index 2bed46073f..70d4f8fa94 100644 --- a/ui/app/routes/observability/episodes/$episode_id/route.tsx +++ b/ui/app/routes/observability/episodes/$episode_id/route.tsx @@ -1,9 +1,7 @@ import { listInferencesWithPagination } from "~/utils/clickhouse/inference.server"; -import { - pollForFeedbackItem, - queryLatestFeedbackIdByMetric, -} from "~/utils/clickhouse/feedback"; +import { pollForFeedbackItem } from "~/utils/clickhouse/feedback"; import { getNativeDatabaseClient } from "~/utils/tensorzero/native_client.server"; +import { getTensorZeroClient } from "~/utils/tensorzero.server"; import type { Route } from "./+types/route"; import { data, @@ -44,7 +42,6 @@ import { TableHeader, TableRow, } from "~/components/ui/table"; -import { getTensorZeroClient } from "~/utils/get-tensorzero-client.server"; export type InferencesData = { inferences: StoredInference[]; @@ -98,7 +95,7 @@ export async function loader({ request, params }: Route.LoaderArgs) { } const dbClient = await getNativeDatabaseClient(); - const tensorZeroClient = await getTensorZeroClient(); + const tensorZeroClient = getTensorZeroClient(); // Start count queries early - these will be streamed to section headers const numInferencesPromise = tensorZeroClient @@ -139,7 +136,7 @@ export async function loader({ request, params }: Route.LoaderArgs) { async (feedbacks) => { const [bounds, latestFeedbackByMetric] = await Promise.all([ dbClient.queryFeedbackBoundsByTargetId({ target_id: episode_id }), - queryLatestFeedbackIdByMetric({ target_id: episode_id }), + tensorZeroClient.getLatestFeedbackIdByMetric(episode_id), ]); return { feedbacks, bounds, latestFeedbackByMetric }; }, @@ -153,7 +150,7 @@ export async function loader({ request, params }: Route.LoaderArgs) { limit, }), dbClient.queryFeedbackBoundsByTargetId({ target_id: episode_id }), - queryLatestFeedbackIdByMetric({ target_id: episode_id }), + tensorZeroClient.getLatestFeedbackIdByMetric(episode_id), ]).then(([feedbacks, bounds, latestFeedbackByMetric]) => ({ feedbacks, bounds, diff --git a/ui/app/routes/observability/inferences/$inference_id/route.tsx b/ui/app/routes/observability/inferences/$inference_id/route.tsx index 74a1a9074f..dfb84ba3f9 100644 --- a/ui/app/routes/observability/inferences/$inference_id/route.tsx +++ b/ui/app/routes/observability/inferences/$inference_id/route.tsx @@ -1,7 +1,4 @@ -import { - pollForFeedbackItem, - queryLatestFeedbackIdByMetric, -} from "~/utils/clickhouse/feedback"; +import { pollForFeedbackItem } from "~/utils/clickhouse/feedback"; import { getNativeDatabaseClient } from "~/utils/tensorzero/native_client.server"; import { getTensorZeroClient } from "~/utils/tensorzero.server"; import { @@ -99,7 +96,7 @@ export async function loader({ request, params }: Route.LoaderArgs) { // Query these after polling completes to avoid race condition with materialized views [feedback_bounds, latestFeedbackByMetric] = await Promise.all([ dbClient.queryFeedbackBoundsByTargetId({ target_id: inference_id }), - queryLatestFeedbackIdByMetric({ target_id: inference_id }), + tensorZeroClient.getLatestFeedbackIdByMetric(inference_id), ]); } else { // Normal case: execute all queries in parallel @@ -116,7 +113,7 @@ export async function loader({ request, params }: Route.LoaderArgs) { demonstrationFeedbackPromise, dbClient.queryFeedbackBoundsByTargetId({ target_id: inference_id }), feedbackDataPromise, - queryLatestFeedbackIdByMetric({ target_id: inference_id }), + tensorZeroClient.getLatestFeedbackIdByMetric(inference_id), ]); } diff --git a/ui/app/utils/clickhouse/feedback.ts b/ui/app/utils/clickhouse/feedback.ts index c542ab254f..8fe62dc794 100644 --- a/ui/app/utils/clickhouse/feedback.ts +++ b/ui/app/utils/clickhouse/feedback.ts @@ -1,7 +1,4 @@ -import { data } from "react-router"; -import { getClickhouseClient } from "./client.server"; import { getNativeDatabaseClient } from "../tensorzero/native_client.server"; -import { z } from "zod"; import { logger } from "~/utils/logger"; import type { FeedbackRow } from "~/types/tensorzero"; @@ -48,54 +45,3 @@ export async function pollForFeedbackItem( } return feedback; } - -export async function queryLatestFeedbackIdByMetric(params: { - target_id: string; -}): Promise> { - const { target_id } = params; - - const query = ` - SELECT - metric_name, - argMax(id, toUInt128(id)) as latest_id - FROM BooleanMetricFeedbackByTargetId - WHERE target_id = {target_id:String} - GROUP BY metric_name - - UNION ALL - - SELECT - metric_name, - argMax(id, toUInt128(id)) as latest_id - FROM FloatMetricFeedbackByTargetId - WHERE target_id = {target_id:String} - GROUP BY metric_name - - ORDER BY metric_name - `; - - try { - const resultSet = await getClickhouseClient().query({ - query, - format: "JSONEachRow", - query_params: { target_id }, - }); - const rows = await resultSet.json(); - - const latestFeedbackByMetric = z - .array( - z.object({ - metric_name: z.string(), - latest_id: z.string().uuid(), - }), - ) - .parse(rows); - - return Object.fromEntries( - latestFeedbackByMetric.map((item) => [item.metric_name, item.latest_id]), - ); - } catch (error) { - logger.error("ERROR", error); - throw data("Error querying latest feedback by metric", { status: 500 }); - } -} diff --git a/ui/app/utils/tensorzero/tensorzero.ts b/ui/app/utils/tensorzero/tensorzero.ts index 06b292c2be..e1b862e087 100644 --- a/ui/app/utils/tensorzero/tensorzero.ts +++ b/ui/app/utils/tensorzero/tensorzero.ts @@ -34,6 +34,7 @@ import type { GetInferencesResponse, GetModelInferencesResponse, InferenceStatsResponse, + LatestFeedbackIdByMetricResponse, ListDatapointsRequest, ListDatasetsResponse, ListEvaluationRunsResponse, @@ -1060,6 +1061,30 @@ export class TensorZeroClient { return (await response.json()) as GetEpisodeInferenceCountResponse; } + /** + * Queries the latest feedback ID for each metric for a given target. + * @param targetId - The target ID (inference_id or episode_id) to query feedback for + * @returns A promise that resolves with a mapping of metric names to their latest feedback IDs + * @throws Error if the request fails + */ + async getLatestFeedbackIdByMetric( + targetId: string, + ): Promise> { + const endpoint = `/internal/feedback/${encodeURIComponent(targetId)}/latest-id-by-metric`; + const response = await this.fetch(endpoint, { method: "GET" }); + if (!response.ok) { + const message = await this.getErrorText(response); + this.handleHttpError({ message, response }); + } + const body = (await response.json()) as LatestFeedbackIdByMetricResponse; + // Convert optional values to non-optional (ts-rs generates HashMap as optional, but values are always present) + return Object.fromEntries( + Object.entries(body.feedback_id_by_metric).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ); + } + private async fetch( path: string, init: {