Evaluations

from checkthat_ai import CheckThatAI
import os

client = CheckThatAI(api_key=os.getenv("OPENAI_API_KEY"))

# Evaluations run automatically during claim processing
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {"role": "user", "content": "Extract and evaluate claims from: 'Studies show that 95% of doctors recommend this new treatment for diabetes management.'"}
    ],
    # Optional: Include Confident AI key for dashboard access
    extra_headers={
        "X-Confident-API-Key": os.getenv("CONFIDENT_API_KEY")
    }
)

print(response.choices[0].message.content)

import requests
import csv
from datetime import datetime

def download_evaluation_report(evaluation_id, api_key):
    """Download evaluation report as CSV"""
    
    response = requests.get(
        f'https://api.checkthat-ai.com/v1/evaluations/{evaluation_id}/report',
        headers={
            'Authorization': f'Bearer {api_key}',
            'Accept': 'text/csv'
        }
    )
    
    if response.status_code == 200:
        # Save report locally
        filename = f"evaluation_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
        with open(filename, 'w', newline='', encoding='utf-8') as file:
            file.write(response.text)
        
        print(f"✅ Report saved as {filename}")
        return filename
    else:
        print(f"❌ Failed to download report: {response.status_code}")
        return None

# Usage
report_file = download_evaluation_report("eval_123abc", "your-api-key")

# Analyze report
if report_file:
    with open(report_file, 'r', encoding='utf-8') as file:
        reader = csv.DictReader(file)
        for row in reader:
            print(f"Claim: {row['extracted_claim']}")
            print(f"Accuracy: {row['accuracy_score']}")
            print(f"Verifiable: {row['verifiable']}")
            print("---")

from checkthat_ai import CheckThatAI
import csv
import os

client = CheckThatAI(api_key=os.getenv("OPENAI_API_KEY"))

def evaluate_claims_batch(texts, output_file="batch_evaluation.csv"):
    """Process multiple texts and generate evaluation report"""
    
    results = []
    
    for i, text in enumerate(texts):
        print(f"Processing text {i+1}/{len(texts)}...")
        
        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[
                {"role": "user", "content": f"Extract and evaluate claims: {text}"}
            ],
            extra_headers={
                "X-Confident-API-Key": os.getenv("CONFIDENT_API_KEY")
            }
        )
        
        # Parse response and collect evaluation data
        # (Response would include evaluation metrics)
        results.append({
            'text_id': i+1,
            'original_text': text,
            'response': response.choices[0].message.content,
            'evaluation_id': response.id  # Use response ID to fetch detailed eval
        })
    
    # Save batch results
    with open(output_file, 'w', newline='', encoding='utf-8') as file:
        if results:
            writer = csv.DictWriter(file, fieldnames=results[0].keys())
            writer.writeheader()
            writer.writerows(results)
    
    print(f"✅ Batch evaluation complete. Results saved to {output_file}")
    return results

# Usage
sample_texts = [
    "Clinical trials show 95% efficacy in reducing symptoms.",
    "The new policy will create 1 million jobs by 2025.",
    "Scientists confirm that this treatment is 100% safe."
]

batch_results = evaluate_claims_batch(sample_texts)