{
  "schemaVersion": 1,
  "status": "protocol_not_executed",
  "evidenceCutoff": "2026-11-13",
  "pins": {
    "website": "fe6bb4e0105cdff76ff12b071406b38c358879e3",
    "product": "bb65bdf0f1320d0f344b0f1ef68a301e9dfd01a4"
  },
  "coverage": {
    "indexed": {
      "kind": "maintained_public_source_counters",
      "snapshotAt": "2026-11-13T16:04:21.525843Z",
      "recordsMirrored": 104284404,
      "chunksIndexed": 176710718,
      "chunksTotal": 176807937,
      "sourceFamilies": 125,
      "enabledSourceFamilies": 123,
      "provenance": "Collected by Devin in a metadata-only READ ONLY transaction on the production isolated public-source database. Credential-free query receipt attached; no record bodies or customer data read. The offline generator does not connect to production.",
      "receiptStatus": "attached",
      "receipt": {
        "queryPath": "scripts/ai-search-study-coverage.sql",
        "querySha256": "0a1595950b8427854b4ce53a597136f536177229365a3bc957cc98e28f09c1bd",
        "schemaCommit": "bd551407d87ea11124bb5ed789ea164980694334",
        "counterSemanticsPath": "packages/db/src/dao/public-source-mirror-counts.ts",
        "collectionMode": "isolated-required; metadata-only READ ONLY transaction; no full recount or index-availability audit",
        "joinCardinality": "Each source belongs to one namespace; pn.id is its primary key, so the join does not multiply source rows",
        "result": {
          "snapshotAt": "2026-11-13T16:04:21.525843Z",
          "readOnly": "on",
          "sourceFamilies": 125,
          "enabledFamilies": 123,
          "mirroredRecordRows": 104284404,
          "indexedChunks": 176710718,
          "totalChunks": 176807937
        }
      },
      "scope": "public_sources joined to nondeleted public_source_namespaces; nondeleted sources; first snapshot only; 8s statement timeout and 1s lock timeout",
      "limits": "Maintained counters, not a full row recount or unique works. Families can overlap. No live retrieval, freshness, quality, legal completeness or accuracy guarantee. No upstream corpus totals added."
    },
    "directory": {
      "kind": "website_directory",
      "path": "lib/data-sources.ts",
      "entries": 227,
      "uniqueNames": 225,
      "distinctDomainStrings": 211,
      "categories": 14,
      "provenance": "Direct census of dataSourceCategories.groups[].sources at the pinned website revision; exact string deduplication only.",
      "limits": "Editorial inventory. Entries, domains and categories are different units. No live availability or indexed-document inference."
    },
    "searchableUpstream": {
      "kind": "provider_reach",
      "records": null,
      "status": "not_censused",
      "limits": "Configured routes do not establish accessible holdings. Overlapping provider collections must never be added."
    }
  },
  "metrics": [
    {
      "key": "accuracy",
      "label": "Answer accuracy",
      "unit": "Claims checked against sources",
      "arca": null,
      "standardWeb": null,
      "status": "not_measured"
    },
    {
      "key": "completeness",
      "label": "Evidence completeness",
      "unit": "Required evidence included",
      "arca": null,
      "standardWeb": null,
      "status": "not_measured"
    },
    {
      "key": "speed",
      "label": "End-to-end speed",
      "unit": "Time to a finished answer",
      "arca": null,
      "standardWeb": null,
      "status": "not_measured"
    },
    {
      "key": "cost",
      "label": "Total serving cost",
      "unit": "Dollars per attempt, including failures",
      "arca": null,
      "standardWeb": null,
      "status": "not_measured"
    }
  ],
  "protocol": {
    "status": "proposed_unapproved",
    "domains": [
      "scientific_literature",
      "medical_literature",
      "regulatory_guidance",
      "legal_research"
    ],
    "uniqueQuestions": 20,
    "questionsPerDomain": 5,
    "arms": 2,
    "repeats": 3,
    "plannedAttempts": 120,
    "plannedAttemptsPerArm": 60,
    "completedAttempts": null,
    "rootModel": "openai/gpt-6.1-sol",
    "reasoningEffort": "medium",
    "providerRoute": null,
    "modelRevision": null,
    "rootStepsMax": 12,
    "deadlineSeconds": 600,
    "readVisibleCharactersMax": 20000,
    "evidenceVisibleCharactersMax": 120000,
    "rootInputTokensPerRequestMax": 64000,
    "rootInputTokensTotalMax": 500000,
    "rootOutputReasoningTokensTotalMax": 32000,
    "finalVisibleTokensMax": 4000,
    "bootstrap": {
      "resamples": 10000,
      "seed": 20261113,
      "unit": "question_cluster_stratified_by_domain",
      "interval": "percentile_95"
    },
    "spendingAuthorized": false,
    "budgetEnforced": false,
    "planningOnly": {
      "apiUsdRange": [
        100,
        220
      ],
      "laborUsd": 6800,
      "infrastructureUsdRange": [
        50,
        150
      ],
      "fullStudyUsdRange": [
        6950,
        7170
      ],
      "limits": "Conditional, unquoted estimates. Unknown provider/license commitments and nested usage can invalidate the estimate. No spending authorization or implemented hard cap. Human evaluation labor is separate from runtime serving cost."
    }
  },
  "historical": {
    "kind": "historical_regulatory_deliverable_outcomes_not_search_accuracy",
    "path": "content/data/lifesci-diamond.json",
    "sha256": "96899eb0972f98254dda841d03875d258ac07505a76491c0eee4143aecef9c1d",
    "version": "2026-11-03-gdp-v6",
    "releaseDate": "2026-11-04",
    "sourceFingerprint": "ea23abfc202177b563475b30e0b9a996ae160a317ed4654b7a4704181784b0f7",
    "requestedModelPairs": [
      "openai/gpt-6-astra",
      "anthropic/claude-opus-5",
      "anthropic/claude-opus-5.5"
    ],
    "cohort": "55 tasks per arm per requested-model pair; one archived answer per task/configuration; all three matched requested-model pairs in this release",
    "judges": [
      "openai/gpt-6.1-sol"
    ],
    "judgingEffort": "medium",
    "answerSettings": "Requested configurations only. Complete generation-time effort, temperature, top-p, seed and output limits were not recorded; gateway cache auto. One Arca Opus 5 strict pass used other models.",
    "control": "Plain API with task documents and six Exa results, at most 2500 characters each; no full external page/PDF reading. Not a fully matched ordinary-web baseline.",
    "limits": "Mixed-date answers, 102 refreshed cells, asymmetric readers, incomplete routing history, selected tasks, provisional/uneven key review and no complete total-cost or latency ledger.",
    "sourceNotes": [
      "https://github.com/arcainc/lifesci-diamond/blob/main/RUN_NOTES.md",
      "evals/reg_affairs_bench/publish/regrades/2026-11-03/README.md",
      "evals/reg_affairs_bench/publish/public/DATASHEET.md"
    ]
  },
  "publicationGates": [
    "Licensed independently authored questions and fully source-validated keys",
    "Frozen same-model/provider configuration, full readers and all nested spending bounds",
    "Approved budget and rights before any paid execution",
    "All paired repeats with complete failure, timing and cost accounting",
    "Blinded domain-expert scoring, uncertainty and public-safe release review"
  ],
  "coverageQuerySql": "-- Run only with authorized access to the isolated public-source database.\n-- This reads maintained metadata counters, not source or customer content.\n-- Each source joins at most one namespace through its primary key.\n-- A future execution yields a new snapshot, not the frozen 13 Nov receipt.\nBEGIN READ ONLY;\nSET LOCAL statement_timeout = '8s';\nSET LOCAL lock_timeout = '1s';\n\nSELECT json_build_object(\n  'snapshotAt', current_timestamp,\n  'readOnly', current_setting('transaction_read_only'),\n  'sourceFamilies', count(*),\n  'enabledFamilies', count(*) FILTER (WHERE ps.enabled),\n  'mirroredRecordRows', sum(ps.records_mirrored_count),\n  'indexedChunks', sum(ps.chunks_indexed_count),\n  'totalChunks', sum(ps.chunks_total_count)\n)\nFROM public_sources ps\nJOIN public_source_namespaces pn\n  ON pn.id = ps.namespace_id\n AND pn.deleted_at IS NULL\nWHERE ps.deleted_at IS NULL;\n\nROLLBACK;\n",
  "historicalResults": [
    {
      "model": "openai/gpt-6-astra",
      "name": "GPT-6 Astra",
      "tasks": 55,
      "arca": {
        "passes": 16,
        "passPct": 29.09090909090909,
        "meanScore": 87.11454545454546
      },
      "plain": {
        "passes": 11,
        "passPct": 20,
        "meanScore": 83.78909090909092
      },
      "strictDeltaPp": 9.090909090909092,
      "strictDeltaCi95": {
        "lo": 0,
        "hi": 18.181818181818183
      },
      "meanDelta": 3.3254545454545457,
      "meanDeltaCi95": {
        "lo": -1.6163636363636362,
        "hi": 9.592727272727272
      }
    },
    {
      "model": "anthropic/claude-opus-5",
      "name": "Claude Opus 5",
      "tasks": 55,
      "arca": {
        "passes": 5,
        "passPct": 9.090909090909092,
        "meanScore": 77.5
      },
      "plain": {
        "passes": 1,
        "passPct": 1.8181818181818181,
        "meanScore": 56.20000000000001
      },
      "strictDeltaPp": 7.2727272727272725,
      "strictDeltaCi95": {
        "lo": 1.8181818181818181,
        "hi": 14.545454545454545
      },
      "meanDelta": 21.3,
      "meanDeltaCi95": {
        "lo": 12.690909090909091,
        "hi": 30.023636363636363
      }
    },
    {
      "model": "anthropic/claude-opus-5.5",
      "name": "Claude Opus 5.5",
      "tasks": 55,
      "arca": {
        "passes": 7,
        "passPct": 12.727272727272727,
        "meanScore": 79.4
      },
      "plain": {
        "passes": 2,
        "passPct": 3.6363636363636362,
        "meanScore": 65.31272727272727
      },
      "strictDeltaPp": 9.090909090909092,
      "strictDeltaCi95": {
        "lo": 0,
        "hi": 18.181818181818183
      },
      "meanDelta": 14.087272727272726,
      "meanDeltaCi95": {
        "lo": 6.358181818181819,
        "hi": 22.560000000000002
      }
    }
  ]
}