Evaluations

from checkthat_ai import CheckThatAI
import os

client = CheckThatAI(api_key=os.getenv("OPENAI_API_KEY"))

# Evaluations run automatically during claim processing
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {"role": "user", "content": "Extract and evaluate claims from: 'Studies show that 95% of doctors recommend this new treatment for diabetes management.'"}
    ],
    # Optional: Include Confident AI key for dashboard access
    extra_headers={
        "X-Confident-API-Key": os.getenv("CONFIDENT_API_KEY")
    }
)

print(response.choices[0].message.content)

curl -X POST 'https://api.checkthat-ai.com/v1/chat/completions' \
  -H 'Content-Type: application/json' \
  -H 'X-Confident-API-Key: your-confident-ai-key' \
  -d '{
    "api_key": "your-provider-api-key",
    "model": "gpt-4o",
    "messages": [
      {
        "role": "user",
        "content": "Analyze and evaluate claims in this text: '\''The new vaccine shows 98% efficacy in preventing severe illness.'\''"
      }
    ],
    "enable_evaluations": true
  }'

import requests
import csv
from datetime import datetime

def download_evaluation_report(evaluation_id, api_key):
    """Download evaluation report as CSV"""
    
    response = requests.get(
        f'https://api.checkthat-ai.com/v1/evaluations/{evaluation_id}/report',
        headers={
            'Authorization': f'Bearer {api_key}',
            'Accept': 'text/csv'
        }
    )
    
    if response.status_code == 200:
        # Save report locally
        filename = f"evaluation_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
        with open(filename, 'w', newline='', encoding='utf-8') as file:
            file.write(response.text)
        
        print(f"✅ Report saved as {filename}")
        return filename
    else:
        print(f"❌ Failed to download report: {response.status_code}")
        return None

# Usage
report_file = download_evaluation_report("eval_123abc", "your-api-key")

# Analyze report
if report_file:
    with open(report_file, 'r', encoding='utf-8') as file:
        reader = csv.DictReader(file)
        for row in reader:
            print(f"Claim: {row['extracted_claim']}")
            print(f"Accuracy: {row['accuracy_score']}")
            print(f"Verifiable: {row['verifiable']}")
            print("---")

import pandas as pd
import matplotlib.pyplot as plt

def analyze_evaluation_metrics(csv_file):
    """Analyze evaluation metrics from CSV report"""
    
    df = pd.read_csv(csv_file)
    
    # Calculate summary statistics
    print("📊 Evaluation Summary")
    print(f"Total Claims Processed: {len(df)}")
    print(f"Average Accuracy: {df['accuracy_score'].mean():.3f}")
    print(f"Average Completeness: {df['completeness_score'].mean():.3f}")
    print(f"Average Coherence: {df['coherence_score'].mean():.3f}")
    print(f"Verifiable Claims: {df['verifiable'].sum()}/{len(df)} ({df['verifiable'].mean()*100:.1f}%)")
    
    # Category breakdown
    print("\n📋 Claims by Category:")
    category_counts = df['category'].value_counts()
    for category, count in category_counts.items():
        avg_accuracy = df[df['category'] == category]['accuracy_score'].mean()
        print(f"  {category}: {count} claims (avg accuracy: {avg_accuracy:.3f})")
    
    # Quality distribution
    high_quality = df[df['accuracy_score'] >= 0.9]
    medium_quality = df[(df['accuracy_score'] >= 0.7) & (df['accuracy_score'] < 0.9)]
    low_quality = df[df['accuracy_score'] < 0.7]
    
    print(f"\n🎯 Quality Distribution:")
    print(f"  High Quality (≥0.9): {len(high_quality)} claims")
    print(f"  Medium Quality (0.7-0.9): {len(medium_quality)} claims")
    print(f"  Low Quality (<0.7): {len(low_quality)} claims")

# Usage
analyze_evaluation_metrics("evaluation_report_20240115_103000.csv")

from checkthat_ai import CheckThatAI
import csv
import os

client = CheckThatAI(api_key=os.getenv("OPENAI_API_KEY"))

def evaluate_claims_batch(texts, output_file="batch_evaluation.csv"):
    """Process multiple texts and generate evaluation report"""
    
    results = []
    
    for i, text in enumerate(texts):
        print(f"Processing text {i+1}/{len(texts)}...")
        
        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[
                {"role": "user", "content": f"Extract and evaluate claims: {text}"}
            ],
            extra_headers={
                "X-Confident-API-Key": os.getenv("CONFIDENT_API_KEY")
            }
        )
        
        # Parse response and collect evaluation data
        # (Response would include evaluation metrics)
        results.append({
            'text_id': i+1,
            'original_text': text,
            'response': response.choices[0].message.content,
            'evaluation_id': response.id  # Use response ID to fetch detailed eval
        })
    
    # Save batch results
    with open(output_file, 'w', newline='', encoding='utf-8') as file:
        if results:
            writer = csv.DictWriter(file, fieldnames=results[0].keys())
            writer.writeheader()
            writer.writerows(results)
    
    print(f"✅ Batch evaluation complete. Results saved to {output_file}")
    return results

# Usage
sample_texts = [
    "Clinical trials show 95% efficacy in reducing symptoms.",
    "The new policy will create 1 million jobs by 2025.",
    "Scientists confirm that this treatment is 100% safe."
]

batch_results = evaluate_claims_batch(sample_texts)

import time
import json

def monitor_evaluation_quality(client, texts, threshold=0.8):
    """Monitor evaluation quality in real-time"""
    
    low_quality_alerts = []
    
    for text in texts:
        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[{"role": "user", "content": f"Evaluate: {text}"}],
            extra_headers={"X-Confident-API-Key": os.getenv("CONFIDENT_API_KEY")}
        )
        
        # In a real implementation, you'd parse evaluation metrics from response
        # For demonstration, we'll simulate evaluation scores
        simulated_accuracy = 0.75  # This would come from actual evaluation
        
        if simulated_accuracy < threshold:
            alert = {
                'text': text,
                'accuracy': simulated_accuracy,
                'timestamp': time.time(),
                'response_id': response.id
            }
            low_quality_alerts.append(alert)
            print(f"⚠️  Low quality alert: {simulated_accuracy:.3f} < {threshold}")
        else:
            print(f"✅ Quality check passed: {simulated_accuracy:.3f}")
        
        time.sleep(1)  # Rate limiting
    
    return low_quality_alerts

# Usage
monitoring_texts = [
    "Some vague claim about health benefits",
    "Precisely documented clinical trial results show 94.7% efficacy"
]

alerts = monitor_evaluation_quality(client, monitoring_texts)
print(f"Generated {len(alerts)} quality alerts")