{
  "schemaVersion": "1.0",
  "study": "Punk public GitHub issue-plan reuse benchmark — 2026-07-13",
  "reproductionLevel": "partial",
  "summary": "Public files support offline verification of the frozen identifier dataset, artifact checksums, and disclosed aggregate arithmetic. They do not recreate or independently establish the original provider-backed run.",
  "exactScope": {
    "workload": "A fully specified, read-only first tool-plan: github_get_issue(owner, repo, number).",
    "source": "100 historical public vLLM GitHub issue identifiers selected on 2026-07-13.",
    "measured": "Planning route, planning latency, recorded model-token cost, exact plan agreement, and a separately verified fresh GitHub GET on the treatment path.",
    "notMeasured": [
      "Issue triage, issue resolution, coding quality, or ambiguous plan discovery",
      "Customer deployment, organic production traffic, realized customer savings, runtime infrastructure cost, or GitHub API cost",
      "Provider invoice amounts, provider response reproducibility, availability, or performance outside the disclosed run"
    ]
  },
  "historicalEnvironment": {
    "punkCommit": "56df8f7bba87626316c33d1e4e8119c622d4de3e",
    "worktreeDirty": false,
    "provider": "anthropic",
    "requestedModel": "claude-haiku-4-5",
    "servedModelSnapshot": "claude-haiku-4-5-20251001",
    "runWindowUtc": "2026-07-13T17:10:44.483Z through 2026-07-13T17:12:39.359Z",
    "caveat": "The commit identifies the historical run but is not a public source release. Provider models, pricing, GitHub source records, dependencies, and network conditions can change. A later run cannot be expected to reproduce the original timing, population, outputs, or raw-report hash."
  },
  "historicalSourceRepository": {
    "availability": "private_at_publication",
    "publicDependency": false,
    "caveat": "The original runner source and exact historical runtime are not independently auditable from this public bundle. Do not interpret the commit identifier as a public-code link."
  },
  "currentPublicMechanismRunner": {
    "availability": "public_after_publication",
    "url": "https://github.com/PunkTechnologies/punk/blob/main/scripts/github-issue-plan-benchmark.ts",
    "command": "bun run bench:github-plan -- --base-url <url> --api-key <key> --github-token <token> --model <model>",
    "purpose": "Run a new benchmark with the same pinned workload shape and publication gate against a current Punk deployment.",
    "caveat": "This improves mechanism reproducibility for new runs. It does not supply the private historical source, raw report, credentials, provider responses, or July 13 environment and therefore does not change the historical study's partial reproduction level."
  },
  "rawPerRequestEvidence": {
    "availability": "private",
    "sha256Commitment": "654d87cda56e4a7b33d8696ba0d739759796e7883db6fc1bf01f690b815d0f13",
    "reason": "The raw report contains internal run identifiers.",
    "caveat": "Its hash is an integrity commitment only. Without the report, readers cannot independently verify each route, event, model response, or GitHub GET."
  },
  "nonCustomerBoundary": {
    "customerData": "none",
    "productionTraffic": "none",
    "affiliation": "none",
    "disclosure": "Punk selected and operated this public reference study using public vLLM issue identifiers. vLLM is not represented as a Punk customer, affiliate, endorser, or participant."
  },
  "inputs": {
    "datasetSelection": "First 100 GitHub Issues API results for repo:vllm-project/vllm is:issue created:2026-06-13..2026-07-12, ordered created ascending; split 30 observation, 20 shadow, 50 evaluation.",
    "datasetCollectedAtUtc": "2026-07-13T17:10:43.131Z",
    "requiredExternalAccessForANewRun": [
      "An authorized GitHub token and access subject to GitHub terms, API behavior, and rate limits",
      "A separately authorized compatible live model-provider account and keys",
      "A compatible Punk build and a new deployment; the historical source is not publicly supplied"
    ]
  },
  "artifacts": {
    "dataset": { "path": "dataset-2026-07-13.json", "sha256": "b9c2e148582ce3586d877cb791496d756c3bba99689df09c6257d7cd51845985" },
    "result": { "path": "result-2026-07-13.json", "sha256": "6e530e31a6b2817035faabf2b2ba19995097494594f1ca3f2155959ccee92bc5" },
    "protocol": { "path": "protocol-2026-07-13.txt", "sha256": "fa9410a8b9e054326d1a101935096553fa85c6e0d6ad53091a5cb5afee1c2426" },
    "verifier": { "path": "verify-2026-07-13.mjs", "sha256": "d5269f774ebb1eaf2ba2f41b41c396f759d88dfb8bfba7082beb83d5e046c2f3" }
  },
  "validation": {
    "download": "Download all five files in this directory without editing their contents.",
    "command": "node verify-2026-07-13.mjs /path/to/downloaded-evidence",
    "checks": [
      "SHA-256 checksums for the dataset, result, protocol, and verifier",
      "The dataset SHA-256 algorithm and the result's selected issue numbers",
      "Traffic totals, paired denominators, exact-plan/read/model-call totals, cost arithmetic, replay/shadow counts, and positive-gate integrity counters"
    ],
    "doesNotCheck": [
      "Private raw per-request evidence or the raw-report SHA-256 preimage",
      "The original private source repository, historical deployment, provider responses, credentials, or GitHub response bodies",
      "Whether the original external systems were truthful or remain unchanged"
    ]
  }
}