{
  "benchmark": "HumanEval-164 Code Decisions Pilot",
  "hardware": "Apple M5 Max (128 GiB unified memory)",
  "date": "2026-10-07",
  "dataset": {
    "total_tasks": 120,
    "calibration_tasks": 30,
    "evaluation_tasks": 90,
    "sha256": "2f7994d4be5d3b80a040dde3422d1dd1dda2422536d3dd79409598769dbe87f9"
  },
  "methodology": {
    "forward_passes_per_task": 12,
    "total_forwards_per_quant": 1440,
    "debiasing_group": "Z3 cyclic rotations",
    "tournament_model": "Bradley-Terry MLE"
  },
  "quant_comparison": [
    {
      "quant_level": "BF16",
      "precision": "Unquantized 16-bit",
      "weights_size_gb": 7.83,
      "answer_smoke": "PASS",
      "single_pass": {
        "correct": 85,
        "total": 90,
        "accuracy_percent": 94.4,
        "median_ms": 98.7
      },
      "cyclic_rotations": {
        "correct": 89,
        "total": 90,
        "accuracy_percent": 98.9,
        "median_ms": 294.8,
        "ece_percent": 0.79,
        "orbit_flips": "2/90 (2.2%)"
      },
      "pairwise_tournament": {
        "correct": 88,
        "total": 90,
        "accuracy_percent": 97.8
      }
    },
    {
      "quant_level": "Q8",
      "precision": "MLX affine 8-bit, g64",
      "weights_size_gb": 4.16,
      "answer_smoke": "PASS",
      "single_pass": {
        "correct": 85,
        "total": 90,
        "accuracy_percent": 94.4,
        "median_ms": 112.7
      },
      "cyclic_rotations": {
        "correct": 90,
        "total": 90,
        "accuracy_percent": 100.0,
        "median_ms": 341.9,
        "ece_percent": 0.57,
        "orbit_flips": "2/90 (2.2%)"
      },
      "pairwise_tournament": {
        "correct": 88,
        "total": 90,
        "accuracy_percent": 97.8
      }
    },
    {
      "quant_level": "Q6",
      "precision": "MLX affine 6-bit, g64",
      "weights_size_gb": 3.18,
      "answer_smoke": "PASS",
      "single_pass": {
        "correct": 85,
        "total": 90,
        "accuracy_percent": 94.4,
        "median_ms": 124.2
      },
      "cyclic_rotations": {
        "correct": 89,
        "total": 90,
        "accuracy_percent": 98.9,
        "median_ms": 370.7,
        "ece_percent": 1.25,
        "orbit_flips": "1/90 (1.1%)"
      },
      "pairwise_tournament": {
        "correct": 88,
        "total": 90,
        "accuracy_percent": 97.8
      }
    },
    {
      "quant_level": "Q4",
      "precision": "MLX affine 4-bit, g64",
      "weights_size_gb": 2.20,
      "answer_smoke": "PASS",
      "single_pass": {
        "correct": 85,
        "total": 90,
        "accuracy_percent": 94.4,
        "median_ms": 97.7
      },
      "cyclic_rotations": {
        "correct": 89,
        "total": 90,
        "accuracy_percent": 98.9,
        "median_ms": 291.4,
        "ece_percent": 1.60,
        "orbit_flips": "1/90 (1.1%)"
      },
      "pairwise_tournament": {
        "correct": 87,
        "total": 90,
        "accuracy_percent": 96.7
      }
    },
    {
      "quant_level": "Q3",
      "precision": "MLX affine 3-bit, g64",
      "weights_size_gb": 1.72,
      "answer_smoke": "PASS",
      "single_pass": {
        "correct": 80,
        "total": 90,
        "accuracy_percent": 88.9,
        "median_ms": 136.5
      },
      "cyclic_rotations": {
        "correct": 87,
        "total": 90,
        "accuracy_percent": 96.7,
        "median_ms": 371.7,
        "ece_percent": 2.45,
        "orbit_flips": "8/90 (8.9%)"
      },
      "pairwise_tournament": {
        "correct": 84,
        "total": 90,
        "accuracy_percent": 93.3
      }
    },
    {
      "quant_level": "Q2",
      "precision": "MLX affine 2-bit, g64",
      "weights_size_gb": 1.23,
      "answer_smoke": "FAIL",
      "single_pass": {
        "correct": 32,
        "total": 90,
        "accuracy_percent": 35.6,
        "median_ms": 112.6
      },
      "cyclic_rotations": {
        "correct": 20,
        "total": 90,
        "accuracy_percent": 22.2,
        "median_ms": 334.9,
        "ece_percent": 11.51,
        "orbit_flips": "55/90 (61.1%)"
      },
      "pairwise_tournament": {
        "correct": 24,
        "total": 90,
        "accuracy_percent": 26.7
      }
    }
  ],
  "statistical_findings": {
    "q4_vs_bf16_single_pass_mcnemar_p": 1.0,
    "q4_vs_bf16_cyclic_mcnemar_p": 1.0,
    "q4_vs_bf16_tournament_mcnemar_p": 1.0,
    "q4_memory_savings_percent": 71.9,
    "q8_cyclic_accuracy_percent": 100.0,
    "q8_holdout_ece_percent": 0.57,
    "q2_collapse_p_vs_bf16": 3.4e-21
  }
}