from scripts.hallucination_toolkit import (
    OpenAIBackend, OpenAIItem, OpenAIPlanner,
    make_sla_certificate, save_sla_certificate_json,
    generate_answer_if_allowed
)

# Initialize
backend = OpenAIBackend(model="gpt-4o-mini")
planner = OpenAIPlanner(backend, temperature=0.3)

# Prepare items
items = [
    OpenAIItem(
        prompt="Who won the 2019 Nobel Prize in Physics?",
        n_samples=7,
        m=6,
        skeleton_policy="closed_book"
    ),
    OpenAIItem(
        prompt="""Task: Answer based on the evidence.
        Question: What is the speed of light?
        Evidence: The speed of light in vacuum is 299,792,458 m/s.""",
        n_samples=5,
        m=6,
        fields_to_erase=["Evidence"],
        skeleton_policy="evidence_erase"
    )
]

# Run evaluation
metrics = planner.run(
    items,
    h_star=0.05,           # Target max 5% hallucination
    isr_threshold=1.0,     # Information sufficiency threshold
    margin_extra_bits=0.2, # Safety margin (nats)
    B_clip=12.0,          # Clipping bound
    clip_mode="one-sided" # Conservative clipping
)

# Generate report and SLA certificate
report = planner.aggregate(
    items, 
    metrics, 
    alpha=0.05,  # 95% confidence
    h_star=0.05
)

cert = make_sla_certificate(report, model_name="GPT-4o-mini")
save_sla_certificate_json(cert, "sla_certificate.json")

print(f"Answer rate: {report.answer_rate:.1%}")
print(f"Empirical hallucination: {report.empirical_hallucination_rate:.1%}" 
      if report.empirical_hallucination_rate else "No labels")
print(f"Worst-case RoH bound: {report.worst_item_roh_bound:.3f}")

# Generate answers for allowed items
for item, metric in zip(items, metrics):
    if metric.decision_answer:
        answer = generate_answer_if_allowed(backend, item, metric)
        print(f"\nQ: {item.prompt[:50]}...")
        print(f"A: {answer}")
    else:
        print(f"\n[REFUSED] {item.prompt[:50]}...")

from scripts.hallucination_toolkit_semantic import (
    OpenAIChatBackend, SemanticPlanner, OpenAIItem
)

# Initialize with semantic features
backend = OpenAIChatBackend(model="gpt-4o-mini", debug=True)
planner = SemanticPlanner(
    backend,
    temperature=1.0,
    top_p=0.9,
    max_tokens_answer=48,
    use_llr_top_meaning_budget=True,  # Recommended
    max_workers_skeletons=6,           # Parallel processing
    kl_smoothing_eps=1e-2
)

# Standard factual query
factual_item = OpenAIItem(
    prompt="Who won the Nobel Prize in Physics in 2014?",
    M_full=10,
    m_skeletons=6,
    n_samples_per_skeleton=6
)

# Creative/Code mode example
code_item = OpenAIItem(
    prompt="Write a Python function to calculate fibonacci numbers",
    M_full=16,  # More samples for style variation
    m_skeletons=6,
    n_samples_per_skeleton=4
)

# Run evaluations
factual_metrics = planner.run([factual_item], h_star=0.05, isr_threshold=1.0)
code_metrics = planner.run([code_item], h_star=0.30, isr_threshold=0.8)  # Relaxed for code

# Display semantic analysis
for metric in factual_metrics:
    print(f"Decision: {'ANSWER' if metric.decision_answer else 'REFUSE'}")
    print(f"Top meaning: {metric.top_meaning}")
    print(f"Posterior confidence: {metric.P_full_top:.1%}")
    print(f"Semantic entropy: {metric.H_sem:.3f}")
    print(f"RoH bound: {metric.roh_bound:.3f}")

item = OpenAIItem(
    prompt="Question: X\nEvidence: Y\nAnswer based on evidence.",
    fields_to_erase=["Evidence"],
    skeleton_policy="evidence_erase"
)

item = OpenAIItem(
    prompt="What happened in Paris in 1889?",
    skeleton_policy="closed_book"  # Masks entities, dates, numbers
)

# Set h_star=0.30 (70% confidence) instead of 0.05 (95%)
# Still blocks fabricated facts about real entities
metrics = planner.run(items, h_star=0.30, isr_threshold=0.8)

# Backend
backend = OpenAIBackend(model="gpt-4o-mini", api_key=None)

# Item
item = OpenAIItem(
    prompt="...",
    n_samples=7,
    m=6,
    skeleton_policy="auto",  # "auto", "evidence_erase", "closed_book"
    fields_to_erase=["Evidence"],  # For evidence mode
)

# Planner
planner = OpenAIPlanner(backend, temperature=0.3, q_floor=None)
metrics = planner.run(items, h_star=0.05, isr_threshold=1.0)
report = planner.aggregate(items, metrics, alpha=0.05)

# SLA Certificate
cert = make_sla_certificate(report, model_name="gpt-4o-mini")
save_sla_certificate_json(cert, "path.json")

# Answer Generation
answer = generate_answer_if_allowed(backend, item, metric)

# Backend with logprobs
backend = OpenAIChatBackend(
    model="gpt-4o-mini",
    debug=True,
    strict=False
)

# Planner with semantic features
planner = SemanticPlanner(
    backend,
    entailer=None,  # Auto-creates OpenAILLMEntailer
    temperature=1.0,
    use_llr_top_meaning_budget=True,
    max_workers_skeletons=6,
    max_entail_calls=120
)

# Item with semantic parameters
item = OpenAIItem(
    prompt="...",
    M_full=10,
    m_skeletons=6,
    n_samples_per_skeleton=6
)

# Run with parallel item processing
metrics = planner.run(items, h_star=0.05, max_workers_items=8)