CVE Vulnerability Patching Benchmark

Import your evaluation results into W&B for centralized tracking.

# Pseudocode — implement these functions for your agent

import wandb

def upload_to_wandb(results):
    with wandb.init(
        project="cve-bench",
        entity="tobias_xor-xor",
        job_type="evaluation"
    ):
        wandb.log({
            "pass_rate": results.pass_rate,
            "total_evals": results.total,
            "cost_usd": results.cost
        })

upload_to_wandb(evaluation_results)

Run your agent against the benchmark and log results to W&B.

# Pseudocode — implement these functions for your agent

def evaluate_agent(agent, samples):
    results = {
        "pass": 0,
        "fail": 0,
        "build": 0,
        "infra": 0
    }

    for sample in samples:
        outcome = run_agent(agent, sample)
        results[outcome] += 1

    return results

# Log to W&B with detailed metrics
results = evaluate_agent(my_agent, cve_samples)
wandb.log(results)

Expected schema for evaluation results.

{
  "agent_model": "string",
  "sample_id": "string",
  "outcome": "pass" | "fail" | "build" | "infra",
  "time_seconds": number,
  "cost_usd": number,
  "tokens_in": number,
  "tokens_out": number
}