{"evidenceLabel":"Punk-sponsored measured public reference study; not a customer deployment","status":"complete_positive","sponsor":"Punk Technologies","affiliationDisclosure":"Not affiliated with or endorsed by the vLLM project.","scope":"Read-only GitHub issue-retrieval planning only; not issue resolution or coding quality.","source":{"repository":"vllm-project/vllm","query":"repo:vllm-project/vllm is:issue created:2026-06-13..2026-07-12","order":"created ascending","collectedAtUtc":"2026-07-13T17:10:43.131Z","totalMatchingAtCollection":519,"selectedUniqueIssues":100,"selectedIssueNumbers":[45482,45492,45493,45496,45500,45507,45515,45519,45520,45531,45533,45538,45540,45543,45562,45568,45570,45572,45585,45591,45592,45593,45596,45597,45598,45604,45605,45608,45624,45626,45633,45647,45658,45661,45663,45667,45668,45669,45670,45687,45689,45691,45698,45699,45702,45704,45709,45719,45733,45734,45736,45741,45742,45749,45751,45750,45752,45760,45769,45771,45775,45784,45785,45800,45806,45808,45814,45815,45830,45834,45837,45847,45856,45859,45861,45866,45871,45872,45878,45884,45885,45888,45902,45904,45911,45920,45922,45929,45938,45943,45974,45987,45995,45997,46014,46016,46027,46028,46036,46040],"selectedIssueCreatedAtUtc":{"first":"2026-06-13T01:25:50Z","last":"2026-06-18T11:39:58Z"},"datasetSha256":"f9a612ebf19dcc6408315552ba15c4f7cc081ead8d55b6494abcc95713a401ce"},"run":{"startedAtUtc":"2026-07-13T17:10:44.483Z","endedAtUtc":"2026-07-13T17:12:39.359Z","punkCommit":"56df8f7bba87626316c33d1e4e8119c622d4de3e","worktreeDirty":false,"liveProvider":"anthropic","requestedModel":"claude-haiku-4-5","servedModelSnapshot":"claude-haiku-4-5-20251001","rawReportSha256":"654d87cda56e4a7b33d8696ba0d739759796e7883db6fc1bf01f690b815d0f13","orderedTreatmentRunIdsSha256":"ec8e7c48b201bbc5ec1d3817e346092fb5c83572bcbcc9e2e5abb844c5cdc60c","orderedControlRunIdsSha256":"5b8d607f6990b231b25bb170416bb4dd1646887c8e454cefbcc0f832a284bcdc"},"traffic":{"uniqueIssues":100,"gatewayRequests":150,"observationBenchmarkLiveModelRequests":30,"shadowBenchmarkLiveModelRequests":20,"heldOutControlRequests":50,"heldOutTreatmentRequests":50,"totalLiveModelRequests":100,"totalArtifactRequests":50},"proof":{"replayPasses":20,"replayFails":0,"shadowPasses":20,"shadowFails":0,"artifactType":"tool_plan","artifactConfidence":0.9635561503461537},"heldOutPairedEvaluation":{"pairs":50,"control":{"liveRoutes":50,"exactPlans":50,"modelRequests":50,"inputTokens":48100,"outputTokens":4050,"recordedModelCostUsd":0.06835,"clientObservedPlanningLatencyMs":{"p50":871.55,"p95":1526.46},"persistedPlanningLatencyMs":{"p50":870,"p95":1521},"providerModels":{"anthropic/claude-haiku-4-5-20251001":50}},"treatment":{"artifactRoutes":50,"exactPlans":50,"modelRequests":0,"freshGitHubReads":50,"inputTokens":0,"deterministicPlanOutputTokens":1200,"recordedModelCostUsd":0,"clientObservedPlanningLatencyMs":{"p50":8.8,"p95":18.49},"persistedPlanningLatencyMs":{"p50":6,"p95":16}},"pairedRecordedModelCostAvoidedUsd":0.06835,"controlRecordedModelCostPer1000RequestsUsd":1.367,"medianClientObservedPlanningLatencyDeltaMs":-869.67,"p95ClientObservedPlanningLatencyDeltaMs":-720.56,"errors":0,"fallbacks":0,"malformedPlans":0,"exclusions":0},"costDisclosure":"Recorded model cost uses Punk's pricing table and token telemetry, not a provider invoice. Punk runtime infrastructure and GitHub API costs were not priced. This is benchmark avoidance, not realized customer savings.","latencyDisclosure":"Latency covers the planning response boundary. The fresh GitHub read was verified separately and is not included in the planning latency values.","publicationGate":{"all50TreatmentArtifactRouted":true,"all50TreatmentPlansExact":true,"all50FreshReadsSucceeded":true,"treatmentModelRequestsZero":true,"externalWritesAttempted":0,"replayPasses":20,"replayFails":0,"shadowPasses":20,"shadowFails":0,"observationPhaseValid":true,"shadowPhaseValid":true,"liveProviderModelConsistent":true,"liveProviderIsNonMock":true,"errors":0,"fallbacks":0,"malformedPlans":0,"exclusions":0}}