{
  "version": "v2026-05-04",
  "releasedAt": "2026-05-04",
  "paperLabel": "OenoBench release_v1.2",
  "totalQuestions": 3266,
  "totalEvaluations": 52256,
  "totalCostUsd": 98.33,
  "scoringNotes": "Evaluation slate over 16 model configurations on 3,266 multiple-choice wine-knowledge questions (release_v1.2). Bootstrap 95% CIs use 1,000 resamples. CB-fail = closed-book solvable (parametric knowledge); CB-pass = contextual reasoning required.",
  "questions": {
    "byDomain": {
      "wine-regions": 1108,
      "grape-varieties": 766,
      "producers": 515,
      "viticulture": 502,
      "winemaking": 188,
      "wine-business": 250
    },
    "byDifficulty": {
      "1": 694,
      "2": 927,
      "3": 698,
      "4": 1010
    },
    "closedBook": {
      "cbFail": 1601,
      "cbPass": 1665
    }
  },
  "scores": [
    {
      "model": "o3",
      "displayName": "o3",
      "provider": "OpenAI",
      "family": "openai",
      "reasoning": "effort",
      "twin": "gpt-5",
      "isGenerator": true,
      "overallAccuracy": 0.836,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.884 },
        { "domain": "grape-varieties", "accuracy": 0.813 },
        { "domain": "producers", "accuracy": 0.909 },
        { "domain": "viticulture", "accuracy": 0.803 },
        { "domain": "winemaking", "accuracy": 0.765 },
        { "domain": "wine-business", "accuracy": 0.654 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.984 },
        { "level": 2, "accuracy": 0.814 },
        { "level": 3, "accuracy": 0.894 },
        { "level": 4, "accuracy": 0.712 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.974 },
        "cbPass": { "n": 1665, "accuracy": 0.703 },
        "delta": 0.271,
        "ci": [0.248, 0.294]
      },
      "selfPreference": {
        "ownAccuracy": 0.819,
        "otherAccuracy": 0.821,
        "delta": -0.001,
        "ci": [-0.039, 0.033]
      },
      "cost": {
        "totalUsd": 11.80,
        "correct": 2729,
        "costPerCorrect": 0.0043,
        "p50LatencyMs": 5697,
        "p95LatencyMs": 22366,
        "tokensIn": 377942,
        "tokensOut": 1381104,
        "parseRate": 0.982
      }
    },
    {
      "model": "gpt-5",
      "displayName": "GPT-5",
      "provider": "OpenAI",
      "family": "openai",
      "reasoning": "standard",
      "twin": "o3",
      "isGenerator": true,
      "overallAccuracy": 0.828,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.881 },
        { "domain": "grape-varieties", "accuracy": 0.782 },
        { "domain": "producers", "accuracy": 0.884 },
        { "domain": "viticulture", "accuracy": 0.828 },
        { "domain": "winemaking", "accuracy": 0.807 },
        { "domain": "wine-business", "accuracy": 0.630 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.983 },
        { "level": 2, "accuracy": 0.824 },
        { "level": 3, "accuracy": 0.875 },
        { "level": 4, "accuracy": 0.692 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.964 },
        "cbPass": { "n": 1665, "accuracy": 0.697 },
        "delta": 0.266,
        "ci": [0.243, 0.292]
      },
      "selfPreference": {
        "ownAccuracy": 0.797,
        "otherAccuracy": 0.814,
        "delta": -0.017,
        "ci": [-0.054, 0.021]
      },
      "cost": {
        "totalUsd": 21.90,
        "correct": 2704,
        "costPerCorrect": 0.0081,
        "p50LatencyMs": 10406,
        "p95LatencyMs": 48978,
        "tokensIn": 377889,
        "tokensOut": 2142407,
        "parseRate": 0.960
      }
    },
    {
      "model": "gemini-2.5-pro-thinking",
      "displayName": "Gemini 2.5 Pro (thinking)",
      "provider": "Google",
      "family": "google",
      "reasoning": "explicit_budget",
      "twin": "gemini-2.5-pro",
      "isGenerator": true,
      "overallAccuracy": 0.826,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.865 },
        { "domain": "grape-varieties", "accuracy": 0.805 },
        { "domain": "producers", "accuracy": 0.900 },
        { "domain": "viticulture", "accuracy": 0.805 },
        { "domain": "winemaking", "accuracy": 0.834 },
        { "domain": "wine-business", "accuracy": 0.602 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.991 },
        { "level": 2, "accuracy": 0.802 },
        { "level": 3, "accuracy": 0.883 },
        { "level": 4, "accuracy": 0.694 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.974 },
        "cbPass": { "n": 1665, "accuracy": 0.684 },
        "delta": 0.290,
        "ci": [0.267, 0.313]
      },
      "selfPreference": {
        "ownAccuracy": 0.740,
        "otherAccuracy": 0.820,
        "delta": -0.080,
        "ci": [-0.123, -0.038]
      },
      "cost": {
        "totalUsd": 13.06,
        "correct": 2698,
        "costPerCorrect": 0.0048,
        "p50LatencyMs": 5650,
        "p95LatencyMs": 6606,
        "tokensIn": 361492,
        "tokensOut": 1261267,
        "parseRate": 1.0
      }
    },
    {
      "model": "gemini-2.5-pro",
      "displayName": "Gemini 2.5 Pro",
      "provider": "Google",
      "family": "google",
      "reasoning": "standard",
      "twin": "gemini-2.5-pro-thinking",
      "isGenerator": true,
      "overallAccuracy": 0.817,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.870 },
        { "domain": "grape-varieties", "accuracy": 0.788 },
        { "domain": "producers", "accuracy": 0.890 },
        { "domain": "viticulture", "accuracy": 0.785 },
        { "domain": "winemaking", "accuracy": 0.775 },
        { "domain": "wine-business", "accuracy": 0.618 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.986 },
        { "level": 2, "accuracy": 0.792 },
        { "level": 3, "accuracy": 0.873 },
        { "level": 4, "accuracy": 0.685 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.970 },
        "cbPass": { "n": 1665, "accuracy": 0.670 },
        "delta": 0.300,
        "ci": [0.275, 0.325]
      },
      "selfPreference": {
        "ownAccuracy": 0.750,
        "otherAccuracy": 0.811,
        "delta": -0.061,
        "ci": [-0.107, -0.018]
      },
      "cost": {
        "totalUsd": 29.47,
        "correct": 2669,
        "costPerCorrect": 0.0110,
        "p50LatencyMs": 9004,
        "p95LatencyMs": 20478,
        "tokensIn": 359476,
        "tokensOut": 2901683,
        "parseRate": 0.970
      }
    },
    {
      "model": "claude-opus-4.7",
      "displayName": "Claude Opus 4.7",
      "provider": "Anthropic",
      "family": "anthropic",
      "reasoning": "standard",
      "twin": "claude-opus-4.7-thinking",
      "isGenerator": true,
      "overallAccuracy": 0.810,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.832 },
        { "domain": "grape-varieties", "accuracy": 0.802 },
        { "domain": "producers", "accuracy": 0.868 },
        { "domain": "viticulture", "accuracy": 0.779 },
        { "domain": "winemaking", "accuracy": 0.866 },
        { "domain": "wine-business", "accuracy": 0.642 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.980 },
        { "level": 2, "accuracy": 0.770 },
        { "level": 3, "accuracy": 0.867 },
        { "level": 4, "accuracy": 0.691 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.973 },
        "cbPass": { "n": 1665, "accuracy": 0.654 },
        "delta": 0.319,
        "ci": [0.295, 0.342]
      },
      "selfPreference": {
        "ownAccuracy": 0.866,
        "otherAccuracy": 0.775,
        "delta": 0.091,
        "ci": [0.061, 0.121]
      },
      "cost": {
        "totalUsd": 3.35,
        "correct": 2647,
        "costPerCorrect": 0.0013,
        "p50LatencyMs": 1278,
        "p95LatencyMs": 2114,
        "tokensIn": 588831,
        "tokensOut": 16254,
        "parseRate": 0.994
      }
    },
    {
      "model": "claude-opus-4.7-thinking",
      "displayName": "Claude Opus 4.7 (thinking)",
      "provider": "Anthropic",
      "family": "anthropic",
      "reasoning": "explicit_budget",
      "twin": "claude-opus-4.7",
      "isGenerator": true,
      "overallAccuracy": 0.810,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.832 },
        { "domain": "grape-varieties", "accuracy": 0.809 },
        { "domain": "producers", "accuracy": 0.870 },
        { "domain": "viticulture", "accuracy": 0.787 },
        { "domain": "winemaking", "accuracy": 0.840 },
        { "domain": "wine-business", "accuracy": 0.614 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.983 },
        { "level": 2, "accuracy": 0.768 },
        { "level": 3, "accuracy": 0.870 },
        { "level": 4, "accuracy": 0.686 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.974 },
        "cbPass": { "n": 1665, "accuracy": 0.652 },
        "delta": 0.323,
        "ci": [0.300, 0.347]
      },
      "selfPreference": {
        "ownAccuracy": 0.869,
        "otherAccuracy": 0.772,
        "delta": 0.097,
        "ci": [0.065, 0.131]
      },
      "cost": {
        "totalUsd": 3.53,
        "correct": 2645,
        "costPerCorrect": 0.0013,
        "p50LatencyMs": 1279,
        "p95LatencyMs": 2145,
        "tokensIn": 588798,
        "tokensOut": 23317,
        "parseRate": 0.994
      }
    },
    {
      "model": "gpt-5-mini",
      "displayName": "GPT-5 mini",
      "provider": "OpenAI",
      "family": "openai",
      "reasoning": "standard",
      "isGenerator": true,
      "overallAccuracy": 0.784,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.821 },
        { "domain": "grape-varieties", "accuracy": 0.744 },
        { "domain": "producers", "accuracy": 0.835 },
        { "domain": "viticulture", "accuracy": 0.775 },
        { "domain": "winemaking", "accuracy": 0.775 },
        { "domain": "wine-business", "accuracy": 0.663 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.980 },
        { "level": 2, "accuracy": 0.737 },
        { "level": 3, "accuracy": 0.847 },
        { "level": 4, "accuracy": 0.648 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.947 },
        "cbPass": { "n": 1665, "accuracy": 0.628 },
        "delta": 0.319,
        "ci": [0.295, 0.345]
      },
      "selfPreference": {
        "ownAccuracy": 0.779,
        "otherAccuracy": 0.761,
        "delta": 0.017,
        "ci": [-0.024, 0.054]
      },
      "cost": {
        "totalUsd": 2.82,
        "correct": 2561,
        "costPerCorrect": 0.0011,
        "p50LatencyMs": 7389,
        "p95LatencyMs": 24192,
        "tokensIn": 379342,
        "tokensOut": 1364211,
        "parseRate": 0.998
      }
    },
    {
      "model": "deepseek-r1",
      "displayName": "DeepSeek-R1",
      "provider": "DeepSeek",
      "family": "deepseek",
      "reasoning": "explicit_budget",
      "twin": "deepseek-v3",
      "isGenerator": false,
      "overallAccuracy": 0.771,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.805 },
        { "domain": "grape-varieties", "accuracy": 0.736 },
        { "domain": "producers", "accuracy": 0.856 },
        { "domain": "viticulture", "accuracy": 0.722 },
        { "domain": "winemaking", "accuracy": 0.765 },
        { "domain": "wine-business", "accuracy": 0.646 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.975 },
        { "level": 2, "accuracy": 0.742 },
        { "level": 3, "accuracy": 0.820 },
        { "level": 4, "accuracy": 0.621 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.943 },
        "cbPass": { "n": 1665, "accuracy": 0.605 },
        "delta": 0.337,
        "ci": [0.311, 0.365]
      },
      "cost": {
        "totalUsd": 10.54,
        "correct": 2517,
        "costPerCorrect": 0.0042,
        "p50LatencyMs": 35687,
        "p95LatencyMs": 130266,
        "tokensIn": 365667,
        "tokensOut": 4112254,
        "parseRate": 0.984
      }
    },
    {
      "model": "gemini-2.5-flash",
      "displayName": "Gemini 2.5 Flash",
      "provider": "Google",
      "family": "google",
      "reasoning": "standard",
      "isGenerator": false,
      "overallAccuracy": 0.751,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.769 },
        { "domain": "grape-varieties", "accuracy": 0.729 },
        { "domain": "producers", "accuracy": 0.823 },
        { "domain": "viticulture", "accuracy": 0.736 },
        { "domain": "winemaking", "accuracy": 0.775 },
        { "domain": "wine-business", "accuracy": 0.606 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.954 },
        { "level": 2, "accuracy": 0.709 },
        { "level": 3, "accuracy": 0.822 },
        { "level": 4, "accuracy": 0.601 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.926 },
        "cbPass": { "n": 1665, "accuracy": 0.583 },
        "delta": 0.343,
        "ci": [0.316, 0.370]
      },
      "selfPreference": {
        "ownAccuracy": 0.643,
        "otherAccuracy": 0.747,
        "delta": -0.104,
        "ci": [-0.153, -0.056]
      },
      "cost": {
        "totalUsd": 0.12,
        "correct": 2454,
        "costPerCorrect": 0.00005,
        "p50LatencyMs": 403,
        "p95LatencyMs": 777,
        "tokensIn": 361492,
        "tokensOut": 3266,
        "parseRate": 1.0
      }
    },
    {
      "model": "deepseek-v3",
      "displayName": "DeepSeek-V3",
      "provider": "DeepSeek",
      "family": "deepseek",
      "reasoning": "standard",
      "twin": "deepseek-r1",
      "isGenerator": false,
      "overallAccuracy": 0.703,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.744 },
        { "domain": "grape-varieties", "accuracy": 0.685 },
        { "domain": "producers", "accuracy": 0.732 },
        { "domain": "viticulture", "accuracy": 0.708 },
        { "domain": "winemaking", "accuracy": 0.690 },
        { "domain": "wine-business", "accuracy": 0.512 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.958 },
        { "level": 2, "accuracy": 0.659 },
        { "level": 3, "accuracy": 0.763 },
        { "level": 4, "accuracy": 0.524 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.904 },
        "cbPass": { "n": 1665, "accuracy": 0.509 },
        "delta": 0.396,
        "ci": [0.366, 0.425]
      },
      "cost": {
        "totalUsd": 0.12,
        "correct": 2295,
        "costPerCorrect": 0.0001,
        "p50LatencyMs": 656,
        "p95LatencyMs": 1141,
        "tokensIn": 363308,
        "tokensOut": 6528,
        "parseRate": 1.0
      }
    },
    {
      "model": "mistral-large-2411",
      "displayName": "Mistral Large 2411",
      "provider": "Mistral AI",
      "family": "mistral",
      "reasoning": "standard",
      "isGenerator": false,
      "overallAccuracy": 0.691,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.715 },
        { "domain": "grape-varieties", "accuracy": 0.656 },
        { "domain": "producers", "accuracy": 0.754 },
        { "domain": "viticulture", "accuracy": 0.694 },
        { "domain": "winemaking", "accuracy": 0.733 },
        { "domain": "wine-business", "accuracy": 0.516 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.944 },
        { "level": 2, "accuracy": 0.631 },
        { "level": 3, "accuracy": 0.733 },
        { "level": 4, "accuracy": 0.540 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.873 },
        "cbPass": { "n": 1665, "accuracy": 0.515 },
        "delta": 0.358,
        "ci": [0.328, 0.389]
      },
      "cost": {
        "totalUsd": 0.87,
        "correct": 2256,
        "costPerCorrect": 0.0004,
        "p50LatencyMs": 294,
        "p95LatencyMs": 2498,
        "tokensIn": 419508,
        "tokensOut": 6532,
        "parseRate": 1.0
      }
    },
    {
      "model": "qwen-2.5-72b",
      "displayName": "Qwen 2.5 72B",
      "provider": "Alibaba",
      "family": "qwen",
      "reasoning": "standard",
      "isGenerator": true,
      "overallAccuracy": 0.674,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.690 },
        { "domain": "grape-varieties", "accuracy": 0.651 },
        { "domain": "producers", "accuracy": 0.734 },
        { "domain": "viticulture", "accuracy": 0.675 },
        { "domain": "winemaking", "accuracy": 0.701 },
        { "domain": "wine-business", "accuracy": 0.528 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.932 },
        { "level": 2, "accuracy": 0.605 },
        { "level": 3, "accuracy": 0.737 },
        { "level": 4, "accuracy": 0.514 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.868 },
        "cbPass": { "n": 1665, "accuracy": 0.488 },
        "delta": 0.381,
        "ci": [0.351, 0.408]
      },
      "selfPreference": {
        "ownAccuracy": 0.661,
        "otherAccuracy": 0.641,
        "delta": 0.020,
        "ci": [-0.021, 0.060]
      },
      "cost": {
        "totalUsd": 0.15,
        "correct": 2202,
        "costPerCorrect": 0.0001,
        "p50LatencyMs": 532,
        "p95LatencyMs": 2728,
        "tokensIn": 401567,
        "tokensOut": 3266,
        "parseRate": 1.0
      }
    },
    {
      "model": "llama-3.3-70b",
      "displayName": "Llama 3.3 70B",
      "provider": "Meta",
      "family": "meta",
      "reasoning": "standard",
      "isGenerator": true,
      "overallAccuracy": 0.671,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.689 },
        { "domain": "grape-varieties", "accuracy": 0.629 },
        { "domain": "producers", "accuracy": 0.772 },
        { "domain": "viticulture", "accuracy": 0.657 },
        { "domain": "winemaking", "accuracy": 0.706 },
        { "domain": "wine-business", "accuracy": 0.504 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.919 },
        { "level": 2, "accuracy": 0.606 },
        { "level": 3, "accuracy": 0.748 },
        { "level": 4, "accuracy": 0.503 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.856 },
        "cbPass": { "n": 1665, "accuracy": 0.492 },
        "delta": 0.364,
        "ci": [0.333, 0.393]
      },
      "selfPreference": {
        "ownAccuracy": 0.655,
        "otherAccuracy": 0.636,
        "delta": 0.019,
        "ci": [-0.023, 0.061]
      },
      "cost": {
        "totalUsd": 0.04,
        "correct": 2190,
        "costPerCorrect": 0.00002,
        "p50LatencyMs": 488,
        "p95LatencyMs": 3103,
        "tokensIn": 403332,
        "tokensOut": 6532,
        "parseRate": 1.0
      }
    },
    {
      "model": "llama-3.1-8b",
      "displayName": "Llama 3.1 8B",
      "provider": "Meta",
      "family": "meta",
      "reasoning": "standard",
      "isGenerator": false,
      "overallAccuracy": 0.605,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.622 },
        { "domain": "grape-varieties", "accuracy": 0.556 },
        { "domain": "producers", "accuracy": 0.695 },
        { "domain": "viticulture", "accuracy": 0.617 },
        { "domain": "winemaking", "accuracy": 0.594 },
        { "domain": "wine-business", "accuracy": 0.476 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.848 },
        { "level": 2, "accuracy": 0.554 },
        { "level": 3, "accuracy": 0.653 },
        { "level": 4, "accuracy": 0.450 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.760 },
        "cbPass": { "n": 1665, "accuracy": 0.456 },
        "delta": 0.304,
        "ci": [0.271, 0.336]
      },
      "selfPreference": {
        "ownAccuracy": 0.542,
        "otherAccuracy": 0.572,
        "delta": -0.030,
        "ci": [-0.073, 0.014]
      },
      "cost": {
        "totalUsd": 0.01,
        "correct": 1976,
        "costPerCorrect": 0.000005,
        "p50LatencyMs": 374,
        "p95LatencyMs": 883,
        "tokensIn": 403288,
        "tokensOut": 6532,
        "parseRate": 1.0
      }
    },
    {
      "model": "qwen-2.5-7b",
      "displayName": "Qwen 2.5 7B",
      "provider": "Alibaba",
      "family": "qwen",
      "reasoning": "standard",
      "isGenerator": true,
      "overallAccuracy": 0.570,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.591 },
        { "domain": "grape-varieties", "accuracy": 0.556 },
        { "domain": "producers", "accuracy": 0.594 },
        { "domain": "viticulture", "accuracy": 0.574 },
        { "domain": "winemaking", "accuracy": 0.578 },
        { "domain": "wine-business", "accuracy": 0.447 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.820 },
        { "level": 2, "accuracy": 0.506 },
        { "level": 3, "accuracy": 0.600 },
        { "level": 4, "accuracy": 0.433 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.728 },
        "cbPass": { "n": 1665, "accuracy": 0.417 },
        "delta": 0.311,
        "ci": [0.277, 0.343]
      },
      "selfPreference": {
        "ownAccuracy": 0.609,
        "otherAccuracy": 0.519,
        "delta": 0.090,
        "ci": [0.045, 0.132]
      },
      "cost": {
        "totalUsd": 0.12,
        "correct": 1860,
        "costPerCorrect": 0.0001,
        "p50LatencyMs": 416,
        "p95LatencyMs": 808,
        "tokensIn": 401787,
        "tokensOut": 6521,
        "parseRate": 1.0
      }
    },
    {
      "model": "claude-haiku-4.5",
      "displayName": "Claude Haiku 4.5",
      "provider": "Anthropic",
      "family": "anthropic",
      "reasoning": "standard",
      "isGenerator": true,
      "overallAccuracy": 0.533,
      "byDomain": [
        { "domain": "wine-regions", "accuracy": 0.558 },
        { "domain": "grape-varieties", "accuracy": 0.497 },
        { "domain": "producers", "accuracy": 0.587 },
        { "domain": "viticulture", "accuracy": 0.542 },
        { "domain": "winemaking", "accuracy": 0.567 },
        { "domain": "wine-business", "accuracy": 0.378 }
      ],
      "byDifficulty": [
        { "level": 1, "accuracy": 0.743 },
        { "level": 2, "accuracy": 0.483 },
        { "level": 3, "accuracy": 0.599 },
        { "level": 4, "accuracy": 0.388 }
      ],
      "closedBook": {
        "cbFail": { "n": 1601, "accuracy": 0.702 },
        "cbPass": { "n": 1665, "accuracy": 0.371 },
        "delta": 0.331,
        "ci": [0.300, 0.368]
      },
      "selfPreference": {
        "ownAccuracy": 0.588,
        "otherAccuracy": 0.488,
        "delta": 0.100,
        "ci": [0.057, 0.145]
      },
      "cost": {
        "totalUsd": 0.44,
        "correct": 1741,
        "costPerCorrect": 0.0003,
        "p50LatencyMs": 1277,
        "p95LatencyMs": 20989,
        "tokensIn": 331007,
        "tokensOut": 21328,
        "parseRate": 0.803
      }
    }
  ],
  "reasoningPairs": [
    {
      "label": "DeepSeek-R1 vs DeepSeek-V3",
      "thinkingModel": "deepseek-r1",
      "standardModel": "deepseek-v3",
      "thinkingAccuracy": 0.771,
      "standardAccuracy": 0.703,
      "delta": 0.068,
      "ci": [0.046, 0.088]
    },
    {
      "label": "Gemini 2.5 Pro: thinking vs standard",
      "thinkingModel": "gemini-2.5-pro-thinking",
      "standardModel": "gemini-2.5-pro",
      "thinkingAccuracy": 0.826,
      "standardAccuracy": 0.817,
      "delta": 0.009,
      "ci": [-0.010, 0.028]
    },
    {
      "label": "o3 vs GPT-5",
      "thinkingModel": "o3",
      "standardModel": "gpt-5",
      "thinkingAccuracy": 0.836,
      "standardAccuracy": 0.828,
      "delta": 0.008,
      "ci": [-0.010, 0.026]
    },
    {
      "label": "Claude Opus 4.7: thinking vs standard",
      "thinkingModel": "claude-opus-4.7-thinking",
      "standardModel": "claude-opus-4.7",
      "thinkingAccuracy": 0.810,
      "standardAccuracy": 0.810,
      "delta": -0.001,
      "ci": [-0.019, 0.018]
    }
  ]
}