{
  "schemaVersion": "1.0",
  "publishedAt": "2026-08-12",
  "benchmarkRunDate": "2026-08-10",
  "scope": "Constrained natural-language routing for the Anyshift Graph API /ask endpoint",
  "methodology": {
    "graphs": 2,
    "graphDescription": "Anonymized live infrastructure graphs",
    "questions": 352,
    "approvedCases": 88,
    "questionVariantsPerCase": 4,
    "repetitionsPerQuestion": 3,
    "models": 6,
    "totalRequests": 6336,
    "reference": "A deterministic live-graph query for every approved question",
    "binaryIntervals": "95% Wilson intervals",
    "latencyIntervals": "10,000-sample paired bootstrap",
    "latencyPath": "One laptop through a tunnel to live graph environments",
    "modelSettings": "OpenAI reasoning effort none; Claude Sonnet 5 and Opus 5 low effort with thinking disabled"
  },
  "reproducibility": {
    "graphApiCommit": "1983b0d1faa4a097d135279b240de29f5d8c5b3d",
    "summaryGeneratorCommit": "36811d01997b9a97a1acc3de6fbce9df51632ed8",
    "systemPromptSha256": "83e2ea013c4e5debfc6ddadc3483805d5889b251fecb4a537c73b5e3f7dbd373",
    "routeSchemaSha256": "1096565e0279f2ac92d79ed843cb762bbb6c19785008e81688d9c42af548689e"
  },
  "units": {
    "accuracy": "percent",
    "latency": "milliseconds",
    "cost": "US dollars per 1,000 requests"
  },
  "models": [
    {
      "model": "Claude Haiku 4.5",
      "provider": "Anthropic",
      "httpSuccessPct": 100,
      "rawRouteAccuracyPct": 42.71,
      "normalizedRouteAccuracyPct": 95.45,
      "finalAnswerAccuracyPct": 100,
      "latencyP50Ms": 1296.5,
      "latencyP95Ms": 2523,
      "observedCostPer1000Usd": 1.210473,
      "cacheReadSharePct": 99.824
    },
    {
      "model": "GPT-5.6 Luna",
      "provider": "OpenAI",
      "httpSuccessPct": 100,
      "rawRouteAccuracyPct": 52.94,
      "normalizedRouteAccuracyPct": 95.45,
      "finalAnswerAccuracyPct": 100,
      "latencyP50Ms": 1469,
      "latencyP95Ms": 3686.5,
      "observedCostPer1000Usd": 0.254527,
      "cacheReadSharePct": 98.118
    },
    {
      "model": "Claude Sonnet 5",
      "provider": "Anthropic",
      "httpSuccessPct": 100,
      "rawRouteAccuracyPct": 44.03,
      "normalizedRouteAccuracyPct": 94.89,
      "finalAnswerAccuracyPct": 99.72,
      "latencyP50Ms": 1982.5,
      "latencyP95Ms": 3167.5,
      "observedCostPer1000Usd": 3.249659
    },
    {
      "model": "GPT-5.6 Terra",
      "provider": "OpenAI",
      "httpSuccessPct": 100,
      "rawRouteAccuracyPct": 50.38,
      "normalizedRouteAccuracyPct": 95.45,
      "finalAnswerAccuracyPct": 100,
      "latencyP50Ms": 1563,
      "latencyP95Ms": 2893,
      "observedCostPer1000Usd": 2.574957
    },
    {
      "model": "Claude Opus 5",
      "provider": "Anthropic",
      "httpSuccessPct": 100,
      "rawRouteAccuracyPct": 44.32,
      "normalizedRouteAccuracyPct": 95.17,
      "finalAnswerAccuracyPct": 99.72,
      "latencyP50Ms": 2269,
      "latencyP95Ms": 4511,
      "observedCostPer1000Usd": 8.194885
    },
    {
      "model": "GPT-5.6 Sol",
      "provider": "OpenAI",
      "httpSuccessPct": 100,
      "rawRouteAccuracyPct": 50,
      "normalizedRouteAccuracyPct": 94.6,
      "finalAnswerAccuracyPct": 99.15,
      "latencyP50Ms": 2373.5,
      "latencyP95Ms": 5141.5,
      "observedCostPer1000Usd": 5.547338
    }
  ],
  "controlledStudies": {
    "lunaCaching": {
      "questions": 352,
      "repetitionsPerConfiguration": 3,
      "requestsPerConfiguration": 1056,
      "automatic": {
        "normalizedRouteAccuracyPct": 95.45,
        "cacheReadSharePct": 97.886,
        "cacheWriteSharePct": 2.083,
        "latencyP50Ms": 924,
        "latencyP95Ms": 2195,
        "observedCostPer1000Usd": 0.260113
      },
      "explicit": {
        "normalizedRouteAccuracyPct": 95.45,
        "cacheReadSharePct": 99.521,
        "cacheWriteSharePct": 0.284,
        "latencyP50Ms": 927,
        "latencyP95Ms": 2208.3,
        "observedCostPer1000Usd": 0.223177
      },
      "observedCostReductionPct": 14.2,
      "cacheWriteReductionPct": 86.4
    },
    "haikuLunaExplicitCaching": {
      "requestsPerModel": 1056,
      "normalizedRouteDisagreements": 0,
      "haiku": {
        "latencyP50Ms": 981,
        "latencyP95Ms": 2041,
        "observedCostPer1000Usd": 1.209606
      },
      "luna": {
        "latencyP50Ms": 948.5,
        "latencyP95Ms": 2379.5,
        "observedCostPer1000Usd": 0.216999
      }
    },
    "lunaFastMode": {
      "latencyP50ReductionPct": 7.3,
      "latencyP95ReductionPct": 11.4,
      "costIncreasePct": 109,
      "routeDisagreements": 0
    },
    "productionLuna": {
      "runDate": "2026-08-11",
      "graphCount": 1,
      "questions": 40,
      "useCases": 10,
      "repetitions": 3,
      "requests": 120,
      "httpSuccesses": 120,
      "correctNormalizedRoutes": 120,
      "correctReferenceAnswers": 120,
      "retries": 0,
      "cacheReadSharePct": 99.804,
      "observedCostPer1000Usd": 0.216185,
      "fullyUncachedEquivalentCostPer1000Usd": 1.935365,
      "timingMs": {
        "endToEnd": { "p50": 3458, "p95": 6777 },
        "clientEdgeResidual": { "p50": 2269, "p95": 5068 },
        "serverRequest": { "p50": 948, "p95": 2730 },
        "modelRouter": { "p50": 869, "p95": 2689 },
        "authentication": { "p50": 29, "p95": 52 },
        "graphExecution": { "p50": 20, "p95": 116 }
      },
      "endToEndCorrelation": {
        "clientEdgeResidual": 0.933,
        "modelRouter": 0.303,
        "authentication": 0.033,
        "graphExecution": 0.093
      }
    }
  },
  "pricing": {
    "capturedAt": "2026-08-10",
    "openAiSource": "https://developers.openai.com/api/docs/models/gpt-5.6-luna",
    "anthropicSource": "https://platform.claude.com/docs/en/about-claude/pricing",
    "note": "Observed costs apply the captured standard token-class rates to measured usage. They exclude negotiated discounts, taxes, and non-model infrastructure."
  },
  "limitations": [
    "This benchmark evaluates constrained natural-language routing for one Graph API endpoint, not general model capability.",
    "The two source graphs were live and could change over time.",
    "Latency was measured through one client and tunnel path and is not a provider service-level claim.",
    "The public aggregate omits customer names, graph identifiers, questions, resource values, and request-level traces."
  ]
}