LLM Fine-Tuning

Real-World Scenario

A legal firm wants Claude to write contract summaries in their specific format — with standard section headings, risk labels, and their house style. Prompting works 70% of the time but requires a 2,000-token few-shot example on every call. Fine-tuning on 300 human-reviewed examples achieves 94% format compliance with a 150-token system prompt, cutting costs by 85%.

When to Fine-Tune

Preparing Fine-Tuning Data

The quality and format of training data matters far more than the quantity. 200 excellent examples beat 2,000 mediocre ones.

import json
import random
from pathlib import Path

# Fine-tuning data format: list of conversation examples
# Each example is a complete conversation the model should learn from

def create_training_example(
    system: str,
    user_message: str,
    ideal_response: str,
) -> dict:
    """Create one fine-tuning example in Anthropic's Messages format."""
    return {
        "system": system,
        "messages": [
            {"role": "user",      "content": user_message},
            {"role": "assistant", "content": ideal_response},
        ]
    }

# Example: training a contract classification model
SYSTEM = """You are a contract analyst. Classify contracts and extract key terms.
Output format:
CONTRACT_TYPE: [type]
RISK_LEVEL: [LOW|MEDIUM|HIGH|CRITICAL]
KEY_TERMS: [comma-separated list]
SUMMARY: [one sentence]"""

training_examples = [
    create_training_example(
        system=SYSTEM,
        user_message="""SERVICE AGREEMENT between Acme Corp and CloudVendor Inc.
CloudVendor will provide SaaS platform access for $5,000/month.
Either party may terminate with 30 days notice. Auto-renews annually.
Liability capped at 3 months of fees paid.""",
        ideal_response="""CONTRACT_TYPE: SaaS Service Agreement
RISK_LEVEL: LOW
KEY_TERMS: $5,000/month, 30-day termination, annual auto-renewal, liability cap 3 months
SUMMARY: Standard SaaS agreement with monthly fees, mutual termination rights, and liability protection."""
    ),
    create_training_example(
        system=SYSTEM,
        user_message="""MASTER SERVICES AGREEMENT - CONFIDENTIAL
Vendor will develop custom software with unlimited liability for defects.
Client owns all IP. Vendor may not compete for 10 years. 
Payment contingent on client satisfaction — no definition provided.""",
        ideal_response="""CONTRACT_TYPE: Custom Software Development Agreement
RISK_LEVEL: CRITICAL
KEY_TERMS: unlimited liability, IP assignment, 10-year non-compete, contingent payment
SUMMARY: High-risk development contract with unlimited vendor liability, broad IP transfer, and vague payment terms."""
    ),
    # Add 198+ more high-quality examples...
]

# Validate the dataset
def validate_dataset(examples: list[dict]) -> dict:
    issues = []
    for i, ex in enumerate(examples):
        if "messages" not in ex:
            issues.append(f"Example {i}: missing 'messages'")
        elif not any(m["role"] == "assistant" for m in ex["messages"]):
            issues.append(f"Example {i}: no assistant response")
        response_len = len(ex["messages"][-1]["content"])
        if response_len < 10:
            issues.append(f"Example {i}: response too short ({response_len} chars)")

    return {
        "total": len(examples),
        "valid": len(examples) - len(issues),
        "issues": issues[:5],   # first 5 issues
    }

report = validate_dataset(training_examples)
print(json.dumps(report, indent=2))

# Split into train/validation sets
random.seed(42)
random.shuffle(training_examples)
split = int(len(training_examples) * 0.9)
train_set = training_examples[:split]
val_set   = training_examples[split:]

# Write JSONL files (one JSON object per line)
Path("fine_tune_train.jsonl").write_text(
    "\n".join(json.dumps(ex) for ex in train_set)
)
Path("fine_tune_val.jsonl").write_text(
    "\n".join(json.dumps(ex) for ex in val_set)
)
print(f"Train: {len(train_set)} examples, Val: {len(val_set)} examples")

Running a Fine-Tuning Job (Anthropic)

import anthropic
import time

client = anthropic.Anthropic()

# Upload training data
with open("fine_tune_train.jsonl", "rb") as f:
    train_file = client.beta.files.upload(
        file=("fine_tune_train.jsonl", f, "application/jsonl"),
    )

with open("fine_tune_val.jsonl", "rb") as f:
    val_file = client.beta.files.upload(
        file=("fine_tune_val.jsonl", f, "application/jsonl"),
    )

print(f"Train file ID: {train_file.id}")
print(f"Val file ID:   {val_file.id}")

# Create the fine-tuning job
job = client.beta.fine_tuning.jobs.create(
    model="claude-haiku-4-5-20251001",   # base model to fine-tune
    training_file=train_file.id,
    validation_file=val_file.id,
    hyperparameters={
        "n_epochs": 3,           # number of passes through the training data
        "batch_size": 8,
        "learning_rate_multiplier": 1.0,
    },
    suffix="contract-classifier",    # appended to the fine-tuned model name
)
print(f"Job ID: {job.id}, Status: {job.status}")

# Poll for completion
def wait_for_job(job_id: str, poll_interval: int = 30) -> dict:
    while True:
        job = client.beta.fine_tuning.jobs.retrieve(job_id)
        print(f"Status: {job.status}")
        if job.status in ("succeeded", "failed", "cancelled"):
            return job
        time.sleep(poll_interval)

completed_job = wait_for_job(job.id)
if completed_job.status == "succeeded":
    model_id = completed_job.fine_tuned_model
    print(f"Fine-tuned model: {model_id}")

Evaluating the Fine-Tuned Model

import anthropic
import json
from sklearn.metrics import accuracy_score

client = anthropic.Anthropic()
FINE_TUNED_MODEL = "your-fine-tuned-model-id"
BASE_MODEL       = "claude-haiku-4-5-20251001"

SYSTEM = """You are a contract analyst. Classify contracts.
Output format:
CONTRACT_TYPE: [type]
RISK_LEVEL: [LOW|MEDIUM|HIGH|CRITICAL]"""

# Held-out evaluation set (not seen during training)
eval_examples = [
    {
        "input": "5-year licensing agreement for proprietary software. $50k/year. "
                 "Licensor may revoke for breach. No modifications allowed.",
        "expected_risk": "MEDIUM"
    },
    {
        "input": "Data processing agreement. Vendor processes EU personal data. "
                 "No DPA signed. No data deletion timeline specified.",
        "expected_risk": "CRITICAL"
    },
    # Add 50+ evaluation examples
]

def extract_risk(response_text: str) -> str:
    for line in response_text.split("\n"):
        if line.startswith("RISK_LEVEL:"):
            return line.split(":")[1].strip()
    return "UNKNOWN"

def evaluate_model(model_id: str, examples: list[dict]) -> dict:
    predicted = []
    for ex in examples:
        response = client.messages.create(
            model=model_id,
            max_tokens=200,
            system=SYSTEM,
            messages=[{"role": "user", "content": ex["input"]}]
        )
        predicted.append(extract_risk(response.content[0].text))

    expected = [ex["expected_risk"] for ex in examples]
    accuracy = sum(p == e for p, e in zip(predicted, expected)) / len(examples)
    return {"accuracy": accuracy, "predicted": predicted}

# Compare base model vs fine-tuned
print("Evaluating base model...")
base_results = evaluate_model(BASE_MODEL, eval_examples[:10])
print(f"Base model accuracy:        {base_results['accuracy']:.1%}")

print("Evaluating fine-tuned model...")
ft_results = evaluate_model(FINE_TUNED_MODEL, eval_examples[:10])
print(f"Fine-tuned model accuracy:  {ft_results['accuracy']:.1%}")
print(f"Improvement:                {(ft_results['accuracy'] - base_results['accuracy']):.1%}")

Fine-Tuning Best Practices

# Data quality checklist

BEST_PRACTICES = """
1. DATA QUALITY (most important)
   - Each example should be something you'd show a new employee as correct
   - Responses must be internally consistent (same format, tone, terminology)
   - Cover all edge cases in your training set, not just the easy ones
   - Have domain experts review at least 20% of examples

2. DATASET SIZE
   - Start with 50-100 examples to prove the concept
   - 200-500 examples is usually enough for format/style tasks
   - 1,000+ for complex domain knowledge tasks
   - Quality beats quantity: 200 excellent > 2,000 mediocre

3. VALIDATION SET
   - Hold out 10-15% for validation (never train on these)
   - Validation loss should decrease — if it diverges, you're overfitting

4. AVOID OVERFITTING
   - Don't repeat examples identically
   - Add paraphrase variations of similar inputs
   - Monitor val loss vs train loss during training

5. EVALUATION
   - Define your success metric BEFORE training
   - Use human evaluation on 50+ holdout examples
   - Compare against strong prompt engineering baseline first

6. ITERATION
   - Analyze failure cases: are errors systematic or random?
   - Add more examples for failure categories
   - Adjust hyperparameters (more epochs if underfitting, fewer if overfitting)
"""
print(BEST_PRACTICES)