//! Golden transcript capture + cross-provider diff tooling (bd-3uqg.8.9).

//!

//! Captures normalized event streams from multiple provider families using

//! identical prompts, diffs them across providers, and produces JSONL/Markdown

//! reports for drift detection.

//!

//! # Approach

//!

//! 1. Each provider family has a "golden transcript" — a normalized event

//! sequence captured from a deterministic mock response.

//! 2. Cross-provider diffs compare text extraction, tool-call shape, stop

//! reasons, and event sequence structure.

//! 3. All output is JSONL for machine consumption + human-readable summaries.

//!

//! Run:

//! ```bash

//! cargo test --test e2e_golden_transcript_diff

//! ```

#![allow(clippy::too_many_lines)]

#![allow(clippy::doc_markdown)]

#![allow(clippy::items_after_statements)]

#![allow(clippy::similar_names)]

#![allow(clippy::cast_precision_loss)]

#![allow(clippy::missing_const_for_fn)]

#![allow(clippy::redundant_clone)]

#![allow(clippy::type_complexity)]

#![allow(clippy::redundant_closure)]

mod common;

use common::{MockHttpResponse, MockHttpServer, TestHarness};

use futures::StreamExt;

use pi::model::{Message, StreamEvent, UserContent, UserMessage};

use pi::models::ModelEntry;

use pi::provider::{Context, InputType, Model, ModelCost, StreamOptions, ToolDef};

use pi::providers::create_provider;

use serde::Serialize;

use serde_json::json;

use std::collections::HashMap;

use std::fmt::Write as _;

use std::sync::Arc;

// ═══════════════════════════════════════════════════════════════════════

// Core types

// ═══════════════════════════════════════════════════════════════════════

/// Schema for golden transcript JSONL artifacts.

const TRANSCRIPT_SCHEMA: &str = "pi.golden_transcript.v1";

/// A single normalized event in the golden transcript.

#[derive(Debug, Clone, Serialize)]

struct NormalizedEvent {

/// Event kind (Start, TextDelta, TextEnd, ToolCallEnd, Done, Error, etc.)

kind: String,

/// Content index for block events.

content_index: Option<usize>,

/// Text content (for TextDelta/TextEnd).

text: Option<String>,

/// Tool call details (for ToolCallEnd).

tool_name: Option<String>,

tool_arguments: Option<String>,

/// Stop reason (for Done/Error).

stop_reason: Option<String>,

/// Error message (for Error events).

error_message: Option<String>,

}

/// Complete golden transcript for one provider.

#[derive(Debug, Clone, Serialize)]

struct GoldenTranscript {

/// Provider family name.

family: String,

/// Provider ID.

provider: String,

/// Scenario name (e.g., "text", "tool_call").

scenario: String,

/// Normalized event sequence.

events: Vec<NormalizedEvent>,

/// Extracted final text content.

final_text: String,

/// Number of tool calls.

tool_call_count: usize,

/// Stop reason string.

stop_reason: Option<String>,

/// Whether the event sequence passed structural validation.

sequence_valid: bool,

/// Validation error, if any.

sequence_error: Option<String>,

/// Total event count.

event_count: usize,

}

/// A difference found between two provider transcripts.

#[derive(Debug, Clone, Serialize)]

struct TranscriptDiff {

/// Field that differs.

field: String,

/// Baseline provider family.

baseline_family: String,

/// Baseline value.

baseline_value: String,

/// Comparison provider family.

compare_family: String,

/// Comparison value.

compare_value: String,

/// Severity: "structural" (event shape), "semantic" (content), "cosmetic" (non-critical).

severity: String,

}

/// Cross-provider diff report.

#[derive(Debug, Clone, Serialize)]

struct DiffReport {

schema: String,

scenario: String,

baseline_family: String,

families_compared: Vec<String>,

diffs: Vec<TranscriptDiff>,

all_match: bool,

}

// ═══════════════════════════════════════════════════════════════════════

// Helpers

// ═══════════════════════════════════════════════════════════════════════

fn make_entry(provider: &str, model_id: &str, base_url: &str) -> ModelEntry {

ModelEntry {

model: Model {

id: model_id.to_string(),

name: format!("{provider} golden model"),

api: String::new(),

provider: provider.to_string(),

base_url: base_url.to_string(),

reasoning: false,

input: vec![InputType::Text],

cost: ModelCost {

input: 0.0,

output: 0.0,

cache_read: 0.0,

cache_write: 0.0,

},

context_window: 8192,

max_tokens: 4096,

headers: HashMap::new(),

},

api_key: None,

headers: HashMap::new(),

auth_header: false,

compat: None,

oauth_config: None,

}

}

fn simple_context() -> Context<'static> {

Context::owned(

Some("You are a helpful assistant.".to_string()),

vec![Message::User(UserMessage {

content: UserContent::Text("Say hello world.".to_string()),

timestamp: 0,

})],

Vec::new(),

)

}

fn tool_context() -> Context<'static> {

Context::owned(

Some("You are a helpful assistant.".to_string()),

vec![Message::User(UserMessage {

content: UserContent::Text("Call the echo function with text hello.".to_string()),

timestamp: 0,

})],

vec![ToolDef {

name: "echo".to_string(),

description: "Echo text back".to_string(),

parameters: json!({

"type": "object",

"properties": {

"text": {"type": "string", "description": "text to echo"}

},

"required": ["text"],

}),

}],

)

}

fn default_options() -> StreamOptions {

StreamOptions {

api_key: Some("golden-test-key".to_string()),

max_tokens: Some(64),

..Default::default()

}

}

fn make_sse_response(body: &str) -> MockHttpResponse {

MockHttpResponse {

status: 200,

headers: vec![("Content-Type".to_string(), "text/event-stream".to_string())],

body: body.as_bytes().to_vec(),

}

}

fn collect_events(

provider: Arc<dyn pi::provider::Provider>,

context: Context<'static>,

options: StreamOptions,

) -> Result<Vec<StreamEvent>, String> {

common::run_async(async move {

let stream = provider

.stream(&context, &options)

.await

.map_err(|e| e.to_string())?;

let mut pinned = std::pin::pin!(stream);

let mut events = Vec::new();

while let Some(item) = pinned.next().await {

let event = item.map_err(|e| e.to_string())?;

let terminal = matches!(event, StreamEvent::Done { .. } | StreamEvent::Error { .. });

events.push(event);

if terminal {

break;

}

}

Ok(events)

})

}

// ═══════════════════════════════════════════════════════════════════════

// Normalization

// ═══════════════════════════════════════════════════════════════════════

fn event_kind(event: &StreamEvent) -> &'static str {

match event {

StreamEvent::Start { .. } => "Start",

StreamEvent::TextStart { .. } => "TextStart",

StreamEvent::TextDelta { .. } => "TextDelta",

StreamEvent::TextEnd { .. } => "TextEnd",

StreamEvent::ThinkingStart { .. } => "ThinkingStart",

StreamEvent::ThinkingDelta { .. } => "ThinkingDelta",

StreamEvent::ThinkingEnd { .. } => "ThinkingEnd",

StreamEvent::ToolCallStart { .. } => "ToolCallStart",

StreamEvent::ToolCallDelta { .. } => "ToolCallDelta",

StreamEvent::ToolCallEnd { .. } => "ToolCallEnd",

StreamEvent::Done { .. } => "Done",

StreamEvent::Error { .. } => "Error",

}

}

fn normalize_event(event: &StreamEvent) -> NormalizedEvent {

match event {

StreamEvent::Start { .. } => NormalizedEvent {

kind: "Start".to_string(),

content_index: None,

text: None,

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

StreamEvent::TextDelta {

content_index,

delta,

..

} => NormalizedEvent {

kind: "TextDelta".to_string(),

content_index: Some(*content_index),

text: Some(delta.clone()),

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

StreamEvent::TextEnd {

content_index,

content,

..

} => NormalizedEvent {

kind: "TextEnd".to_string(),

content_index: Some(*content_index),

text: Some(content.clone()),

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

StreamEvent::TextStart { content_index, .. } => NormalizedEvent {

kind: "TextStart".to_string(),

content_index: Some(*content_index),

text: None,

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

StreamEvent::ToolCallStart { content_index, .. } => NormalizedEvent {

kind: "ToolCallStart".to_string(),

content_index: Some(*content_index),

text: None,

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

StreamEvent::ToolCallDelta {

content_index,

delta,

..

} => NormalizedEvent {

kind: "ToolCallDelta".to_string(),

content_index: Some(*content_index),

text: Some(delta.clone()),

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

StreamEvent::ToolCallEnd {

content_index,

tool_call,

..

} => NormalizedEvent {

kind: "ToolCallEnd".to_string(),

content_index: Some(*content_index),

text: None,

tool_name: Some(tool_call.name.clone()),

tool_arguments: Some(tool_call.arguments.to_string()),

stop_reason: None,

error_message: None,

},

StreamEvent::Done { reason, .. } => NormalizedEvent {

kind: "Done".to_string(),

content_index: None,

text: None,

tool_name: None,

tool_arguments: None,

stop_reason: Some(format!("{reason:?}")),

error_message: None,

},

StreamEvent::Error { reason, error, .. } => NormalizedEvent {

kind: "Error".to_string(),

content_index: None,

text: None,

tool_name: None,

tool_arguments: None,

stop_reason: Some(format!("{reason:?}")),

error_message: error.error_message.clone(),

},

StreamEvent::ThinkingStart { content_index, .. } => NormalizedEvent {

kind: "ThinkingStart".to_string(),

content_index: Some(*content_index),

text: None,

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

StreamEvent::ThinkingDelta {

content_index,

delta,

..

} => NormalizedEvent {

kind: "ThinkingDelta".to_string(),

content_index: Some(*content_index),

text: Some(delta.clone()),

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

StreamEvent::ThinkingEnd {

content_index,

content,

..

} => NormalizedEvent {

kind: "ThinkingEnd".to_string(),

content_index: Some(*content_index),

text: Some(content.clone()),

tool_name: None,

tool_arguments: None,

stop_reason: None,

error_message: None,

},

}

}

fn validate_sequence(events: &[StreamEvent]) -> Result<(), String> {

if events.is_empty() {

return Err("no events emitted".to_string());

}

if !matches!(events.first(), Some(StreamEvent::Start { .. })) {

return Err("first event must be Start".to_string());

}

if !matches!(

events.last(),

Some(StreamEvent::Done { .. } | StreamEvent::Error { .. })

) {

return Err("last event must be Done or Error".to_string());

}

Ok(())

}

fn build_transcript(

family: &str,

provider: &str,

scenario: &str,

result: &Result<Vec<StreamEvent>, String>,

) -> GoldenTranscript {

match result {

Ok(events) => {

let normalized: Vec<NormalizedEvent> = events.iter().map(normalize_event).collect();

let final_text = extract_final_text(events);

let tool_call_count = events

.iter()

.filter(|e| matches!(e, StreamEvent::ToolCallEnd { .. }))

.count();

let stop_reason = events.iter().find_map(|e| match e {

StreamEvent::Done { reason, .. } | StreamEvent::Error { reason, .. } => {

Some(format!("{reason:?}"))

}

_ => None,

});

let validation = validate_sequence(events);

GoldenTranscript {

family: family.to_string(),

provider: provider.to_string(),

scenario: scenario.to_string(),

event_count: normalized.len(),

events: normalized,

final_text,

tool_call_count,

stop_reason,

sequence_valid: validation.is_ok(),

sequence_error: validation.err(),

}

}

Err(e) => GoldenTranscript {

family: family.to_string(),

provider: provider.to_string(),

scenario: scenario.to_string(),

events: Vec::new(),

final_text: String::new(),

tool_call_count: 0,

stop_reason: None,

sequence_valid: false,

sequence_error: Some(e.clone()),

event_count: 0,

},

}

}

fn extract_final_text(events: &[StreamEvent]) -> String {

// Prefer TextEnd content; fall back to concatenating TextDelta.

let from_end: Option<String> = events.iter().find_map(|e| match e {

StreamEvent::TextEnd { content, .. } => Some(content.clone()),

_ => None,

});

if let Some(text) = from_end {

return text;

}

let mut buf = String::new();

for e in events {

if let StreamEvent::TextDelta { delta, .. } = e {

buf.push_str(delta);

}

}

buf

}

// ═══════════════════════════════════════════════════════════════════════

// Diff engine

// ═══════════════════════════════════════════════════════════════════════

fn diff_transcripts(

baseline: &GoldenTranscript,

compare: &GoldenTranscript,

) -> Vec<TranscriptDiff> {

let mut diffs = Vec::new();

// 1. Final text content

if baseline.final_text != compare.final_text {

diffs.push(TranscriptDiff {

field: "final_text".to_string(),

baseline_family: baseline.family.clone(),

baseline_value: baseline.final_text.clone(),

compare_family: compare.family.clone(),

compare_value: compare.final_text.clone(),

severity: "semantic".to_string(),

});

}

// 2. Tool call count

if baseline.tool_call_count != compare.tool_call_count {

diffs.push(TranscriptDiff {

field: "tool_call_count".to_string(),

baseline_family: baseline.family.clone(),

baseline_value: baseline.tool_call_count.to_string(),

compare_family: compare.family.clone(),

compare_value: compare.tool_call_count.to_string(),

severity: "structural".to_string(),

});

}

// 3. Stop reason

if baseline.stop_reason != compare.stop_reason {

diffs.push(TranscriptDiff {

field: "stop_reason".to_string(),

baseline_family: baseline.family.clone(),

baseline_value: baseline.stop_reason.clone().unwrap_or_default(),

compare_family: compare.family.clone(),

compare_value: compare.stop_reason.clone().unwrap_or_default(),

severity: "structural".to_string(),

});

}

// 4. Event kind sequence (abstract shape)

let baseline_kinds: Vec<&str> = baseline.events.iter().map(|e| e.kind.as_str()).collect();

let compare_kinds: Vec<&str> = compare.events.iter().map(|e| e.kind.as_str()).collect();

let baseline_shape = abstract_event_shape(&baseline_kinds);

let compare_shape = abstract_event_shape(&compare_kinds);

if baseline_shape != compare_shape {

diffs.push(TranscriptDiff {

field: "event_shape".to_string(),

baseline_family: baseline.family.clone(),

baseline_value: baseline_shape,

compare_family: compare.family.clone(),

compare_value: compare_shape,

severity: "structural".to_string(),

});

}

// 5. Sequence validity

if baseline.sequence_valid != compare.sequence_valid {

diffs.push(TranscriptDiff {

field: "sequence_valid".to_string(),

baseline_family: baseline.family.clone(),

baseline_value: baseline.sequence_valid.to_string(),

compare_family: compare.family.clone(),

compare_value: compare.sequence_valid.to_string(),

severity: "structural".to_string(),

});

}

// 6. Tool call names (if both have tool calls)

if baseline.tool_call_count > 0 && compare.tool_call_count > 0 {

let baseline_names: Vec<&str> = baseline

.events

.iter()

.filter_map(|e| e.tool_name.as_deref())

.collect();

let compare_names: Vec<&str> = compare

.events

.iter()

.filter_map(|e| e.tool_name.as_deref())

.collect();

if baseline_names != compare_names {

diffs.push(TranscriptDiff {

field: "tool_names".to_string(),

baseline_family: baseline.family.clone(),

baseline_value: format!("{baseline_names:?}"),

compare_family: compare.family.clone(),

compare_value: format!("{compare_names:?}"),

severity: "semantic".to_string(),

});

}

}

diffs

}

/// Collapse event kinds into an abstract shape, merging consecutive deltas.

fn abstract_event_shape(kinds: &[&str]) -> String {

let mut shape = Vec::new();

let mut last = "";

for kind in kinds {

if *kind == last && kind.contains("Delta") {

// Merge consecutive deltas

continue;

}

shape.push(*kind);

last = kind;

}

shape.join(" → ")

}

fn build_diff_report(

scenario: &str,

baseline: &GoldenTranscript,

others: &[GoldenTranscript],

) -> DiffReport {

let mut all_diffs = Vec::new();

let mut families = Vec::new();

for other in others {

families.push(other.family.clone());

all_diffs.extend(diff_transcripts(baseline, other));

}

DiffReport {

schema: TRANSCRIPT_SCHEMA.to_string(),

scenario: scenario.to_string(),

baseline_family: baseline.family.clone(),

families_compared: families,

all_match: all_diffs.is_empty(),

diffs: all_diffs,

}

}

// ═══════════════════════════════════════════════════════════════════════

// Output

// ═══════════════════════════════════════════════════════════════════════

fn write_transcript_jsonl(harness: &TestHarness, name: &str, transcripts: &[GoldenTranscript]) {

let mut buf = String::new();

for t in transcripts {

let _ = writeln!(buf, "{}", serde_json::to_string(t).unwrap_or_default());

}

let path = harness.temp_path(format!("{name}_transcripts.jsonl"));

std::fs::write(&path, &buf).expect("write transcript JSONL");

harness.record_artifact(format!("{name}_transcripts.jsonl"), &path);

}

fn write_diff_report_jsonl(harness: &TestHarness, name: &str, report: &DiffReport) {

let content = serde_json::to_string_pretty(report).unwrap_or_default();

let path = harness.temp_path(format!("{name}_diff.json"));

std::fs::write(&path, &content).expect("write diff report");

harness.record_artifact(format!("{name}_diff.json"), &path);

}

fn write_markdown_summary(harness: &TestHarness, name: &str, report: &DiffReport) {

let mut md = String::new();

let _ = writeln!(md, "# Golden Transcript Diff: {}", report.scenario);

let _ = writeln!(md, "\nBaseline: **{}**", report.baseline_family);

let _ = writeln!(md, "Compared: {}", report.families_compared.join(", "));

let _ = writeln!(

md,

"Result: **{}**\n",

if report.all_match {

"ALL MATCH"

} else {

"DIFFS FOUND"

}

);

if !report.diffs.is_empty() {

let _ = writeln!(md, "| Field | Baseline | Compare | Severity |");

let _ = writeln!(md, "|-------|----------|---------|----------|");

for d in &report.diffs {

let _ = writeln!(

md,

"| {} | {}={} | {}={} | {} |",

d.field,

d.baseline_family,

truncate_for_table(&d.baseline_value, 30),

d.compare_family,

truncate_for_table(&d.compare_value, 30),

d.severity,

);

}

}

let path = harness.temp_path(format!("{name}_diff.md"));

std::fs::write(&path, &md).expect("write diff markdown");

harness.record_artifact(format!("{name}_diff.md"), &path);

}

fn truncate_for_table(s: &str, max: usize) -> String {

if s.len() <= max {

s.to_string()

} else {

// Find a char boundary at or before `max` to avoid panicking on

// multi-byte UTF-8 sequences.

let mut end = max.min(s.len());

while end > 0 && !s.is_char_boundary(end) {

end -= 1;

}

format!("{}...", &s[..end])

}

}

// ═══════════════════════════════════════════════════════════════════════

// SSE fixtures (reused from e2e_provider_scenarios patterns)

// ═══════════════════════════════════════════════════════════════════════

fn openai_responses_text_sse() -> String {

[

r#"data: {"type":"response.output_text.delta","item_id":"msg_1","content_index":0,"delta":"Hello"}"#,

"",

r#"data: {"type":"response.output_text.delta","item_id":"msg_1","content_index":0,"delta":" world!"}"#,

"",

r#"data: {"type":"response.completed","response":{"incomplete_details":null,"usage":{"input_tokens":10,"output_tokens":5,"total_tokens":15}}}"#,

"",

]

.join("\n")

}

fn anthropic_text_sse() -> String {

[

r"event: message_start",

r#"data: {"type":"message_start","message":{"id":"msg_001","type":"message","role":"assistant","content":[],"model":"claude-test","stop_reason":null,"usage":{"input_tokens":10,"output_tokens":0}}}"#,

"",

r"event: content_block_start",

r#"data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#,

"",

r"event: content_block_delta",

r#"data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello world!"}}"#,

"",

r"event: content_block_stop",

r#"data: {"type":"content_block_stop","index":0}"#,

"",

r"event: message_delta",

r#"data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}"#,

"",

r"event: message_stop",

r#"data: {"type":"message_stop"}"#,

"",

]

.join("\n")

}

fn gemini_text_sse() -> String {

[

r#"data: {"candidates":[{"content":{"parts":[{"text":"Hello world!"}],"role":"model"},"finishReason":"STOP","index":0}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":5,"totalTokenCount":15}}"#,

"",

]

.join("\n")

}

fn openai_completions_text_sse() -> String {

[

r#"data: {"id":"oai-001","object":"chat.completion.chunk","created":1700000000,"model":"gpt-test","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]}"#,

"",

r#"data: {"id":"oai-001","object":"chat.completion.chunk","created":1700000000,"model":"gpt-test","choices":[{"index":0,"delta":{"content":" world!"},"finish_reason":null}]}"#,

"",

r#"data: {"id":"oai-001","object":"chat.completion.chunk","created":1700000000,"model":"gpt-test","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}"#,

"",

"data: [DONE]",

"",

]

.join("\n")

}

fn openai_responses_tool_sse() -> String {

[

r#"data: {"type":"response.output_item.added","output_index":0,"item":{"type":"function_call","id":"fc_1","call_id":"call_001","name":"echo","arguments":""}}"#,

"",

r#"data: {"type":"response.function_call_arguments.delta","item_id":"fc_1","output_index":0,"delta":"{\"text\":\"hello\"}"}"#,

"",

r#"data: {"type":"response.output_item.done","output_index":0,"item":{"type":"function_call","id":"fc_1","call_id":"call_001","name":"echo","arguments":"{\"text\":\"hello\"}","status":"completed"}}"#,

"",

r#"data: {"type":"response.completed","response":{"incomplete_details":null,"usage":{"input_tokens":15,"output_tokens":12,"total_tokens":27}}}"#,

"",

]

.join("\n")

}

fn anthropic_tool_sse() -> String {

[

r"event: message_start",

r#"data: {"type":"message_start","message":{"id":"msg_tool","type":"message","role":"assistant","content":[],"model":"claude-test","stop_reason":null,"usage":{"input_tokens":15,"output_tokens":0}}}"#,

"",

r"event: content_block_start",

r#"data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_001","name":"echo","input":{}}}"#,

"",

r"event: content_block_delta",

r#"data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"text\":\"hello\"}"}}"#,

"",

r"event: content_block_stop",

r#"data: {"type":"content_block_stop","index":0}"#,

"",

r"event: message_delta",

r#"data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":12}}"#,

"",

r"event: message_stop",

r#"data: {"type":"message_stop"}"#,

"",

]

.join("\n")

}

fn gemini_tool_sse() -> String {

let chunk = json!({

"candidates": [{

"content": {

"role": "model",

"parts": [{"functionCall": {"name": "echo", "args": {"text": "hello"}}}]

},

"finishReason": "STOP"

}],

"usageMetadata": {

"promptTokenCount": 15,

"candidatesTokenCount": 12,

"totalTokenCount": 27

}

});

format!("data: {}\n\n", serde_json::to_string(&chunk).unwrap())

}

// ═══════════════════════════════════════════════════════════════════════

// Provider setup helpers

// ═══════════════════════════════════════════════════════════════════════

fn setup_openai_responses(

harness: &TestHarness,

sse: &str,

) -> (Arc<dyn pi::provider::Provider>, MockHttpServer) {

let server = harness.start_mock_http_server();

server.add_route("POST", "/v1/responses", make_sse_response(sse));

let base_url = format!("{}/v1", server.base_url());

let mut entry = make_entry("openai", "golden-gpt", &base_url);

entry.model.api.clear();

let provider = create_provider(&entry, None).expect("create openai provider");

(provider, server)

}

fn setup_openai_completions(

harness: &TestHarness,

sse: &str,

) -> (Arc<dyn pi::provider::Provider>, MockHttpServer) {

let server = harness.start_mock_http_server();

server.add_route(

"POST",

"/openai/v1/chat/completions",

make_sse_response(sse),

);

let base_url = format!("{}/openai/v1", server.base_url());

let mut entry = make_entry("groq", "golden-llama", &base_url);

entry.model.api.clear();

let provider = create_provider(&entry, None).expect("create openai-compat provider");

(provider, server)

}

fn setup_anthropic(

harness: &TestHarness,

sse: &str,

) -> (Arc<dyn pi::provider::Provider>, MockHttpServer) {

let server = harness.start_mock_http_server();

server.add_route("POST", "/v1/messages", make_sse_response(sse));

let base_url = format!("{}/v1/messages", server.base_url());

let mut entry = make_entry("anthropic", "golden-claude", &base_url);

entry.model.api.clear();

let provider = create_provider(&entry, None).expect("create anthropic provider");

(provider, server)

}

fn setup_gemini(

harness: &TestHarness,

sse: &str,

) -> (Arc<dyn pi::provider::Provider>, MockHttpServer) {

let server = harness.start_mock_http_server();

let route = "/v1beta/models/golden-gemini:streamGenerateContent?alt=sse";

server.add_route("POST", route, make_sse_response(sse));

let base_url = format!("{}/v1beta", server.base_url());

let mut entry = make_entry("google", "golden-gemini", &base_url);

entry.model.api.clear();

let provider = create_provider(&entry, None).expect("create gemini provider");

(provider, server)

}

// ═══════════════════════════════════════════════════════════════════════

// Section 1: Golden transcript capture for text responses

// ═══════════════════════════════════════════════════════════════════════

#[test]

fn capture_text_golden_transcripts() {

let harness = TestHarness::new("capture_text_golden_transcripts");

let ctx = simple_context();

let opts = default_options();

let mut transcripts = Vec::new();

// OpenAI Responses API

let (provider, _server) = setup_openai_responses(&harness, &openai_responses_text_sse());

let result = collect_events(provider, ctx.clone(), opts.clone());

transcripts.push(build_transcript(

"openai-responses",

"openai",

"text",

&result,

));

// Anthropic

let (provider, _server) = setup_anthropic(&harness, &anthropic_text_sse());

let result = collect_events(provider, ctx.clone(), opts.clone());

transcripts.push(build_transcript(

"anthropic-messages",

"anthropic",

"text",

&result,

));

// Gemini

let (provider, _server) = setup_gemini(&harness, &gemini_text_sse());

let result = collect_events(provider, ctx.clone(), opts.clone());

transcripts.push(build_transcript(

"gemini-generative",

"google",

"text",

&result,

));

// OpenAI Completions (via groq)

let (provider, _server) = setup_openai_completions(&harness, &openai_completions_text_sse());

let result = collect_events(provider, ctx.clone(), opts.clone());

transcripts.push(build_transcript(

"openai-completions",

"groq",

"text",

&result,

));

// All should produce valid transcripts

for t in &transcripts {

assert!(

t.sequence_valid,

"{} transcript should be valid: {:?}",

t.family, t.sequence_error

);

}

write_transcript_jsonl(&harness, "text", &transcripts);

}

#[test]

fn text_transcripts_all_extract_hello_world() {

let harness = TestHarness::new("text_transcripts_all_extract_hello_world");

let ctx = simple_context();

let opts = default_options();

let families: Vec<(

&str,

&str,

String,

Box<dyn Fn(&TestHarness, &str) -> (Arc<dyn pi::provider::Provider>, MockHttpServer)>,

)> = vec![

(

"openai-responses",

"openai",

openai_responses_text_sse(),

Box::new(|h, s| setup_openai_responses(h, s)),

),

(

"anthropic-messages",

"anthropic",

anthropic_text_sse(),

Box::new(|h, s| setup_anthropic(h, s)),

),

(

"gemini-generative",

"google",

gemini_text_sse(),

Box::new(|h, s| setup_gemini(h, s)),

),

(

"openai-completions",

"groq",

openai_completions_text_sse(),

Box::new(|h, s| setup_openai_completions(h, s)),

),

];

for (family, provider, sse, setup) in &families {

let (prov, _server) = setup(&harness, sse);

let result = collect_events(prov, ctx.clone(), opts.clone());

let transcript = build_transcript(family, provider, "text", &result);

assert!(

transcript.final_text.contains("Hello") && transcript.final_text.contains("world"),

"{family}: expected 'Hello world' but got '{}'",

transcript.final_text

);

}

}

#[test]

fn text_transcripts_all_have_stop_reason() {

let harness = TestHarness::new("text_transcripts_all_have_stop_reason");

let ctx = simple_context();

let opts = default_options();

let setups: Vec<(

&str,

String,

Box<dyn Fn(&TestHarness, &str) -> (Arc<dyn pi::provider::Provider>, MockHttpServer)>,

)> = vec![

(

"openai-responses",

openai_responses_text_sse(),

Box::new(|h, s| setup_openai_responses(h, s)),

),

(

"anthropic-messages",

anthropic_text_sse(),

Box::new(|h, s| setup_anthropic(h, s)),

),

(

"gemini-generative",

gemini_text_sse(),

Box::new(|h, s| setup_gemini(h, s)),

),

(

"openai-completions",

openai_completions_text_sse(),

Box::new(|h, s| setup_openai_completions(h, s)),

),

];