{
  "schema_version": "article-evidence-audit-1",
  "audit_date": "2026-09-24",
  "runtime": {
    "python": "3.11.13",
    "dependencies": {
      "numpy": "1.26.4",
      "scipy": "1.17.1",
      "scikit-learn": "1.9.1"
    }
  },
  "pilot_runner_sha256": "5c41ffd01ea787711cf00a8bc493aafc70e28b797b3a273da319b4954026c596",
  "source_hash_checks": [
    {
      "id": "flip2-discrepancies-prepared",
      "sha256": "df996aee84bd5ed3e6381f9f6db864a377ab0b4809a9a48ff146d06a3dc069df",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-flip2-composition-predictions",
      "sha256": "72ea3c506b8645ae6020207f904fbd5c0b09549cd125fcdcabfb487e68681b9c",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-flip2-composition-report",
      "sha256": "abd08d3f493428d06f43740c94e268575d755560e72120c73b294127acb395e6",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/flip2-composition/report.json",
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-flip2-composition-source",
      "sha256": "7157313eab75ff8f93b934ac186852c8c82f8bb885617cab70c89427c631c1e4",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/flip2-composition/source.json",
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-flip2-train-mean-predictions",
      "sha256": "599dca4e6fa643a7cdfc96ac339033f61e5846ce225af17ac2ac3917ecc14fac",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-flip2-train-mean-report",
      "sha256": "eb89260f6e138bb9ae2fdf3b882a87fdf8aeabd54a0ac6f17a73f5671c96b4d8",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/flip2-train-mean/report.json",
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-flip2-train-mean-source",
      "sha256": "8bd712289f136edbfde3fa885166ef85e4f6fe3c4c437810c24dadb87107cc3f",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/flip2-train-mean/source.json",
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-recipe_code",
      "sha256": "b3407940ca4c8a1cdcf3a3bb3849c3e26d7f85272b2ee63087307378ffaa79dc",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/packages/rewirebench/src/rewirebench/adapters/sequence.py",
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-table",
      "sha256": "8562d4891270a73026fc7852c05f41206ad52e8fcae1d706d83ec57572c7c949",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-receipt",
      "sha256": "68631e653cd5aa9047b3f4e88cbb3fec94630847c1b73fad8de1018423d81fac",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-preparation_code",
      "sha256": "8a1d691de8fdd9fcf8242fe6fa2cfe2703a1c3281b880fe2a226cb8bd946fd1b",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "flip2-discrepancies-environment",
      "sha256": "748257abb3664595db1ebf892799e91c6f1a4ca79b0e65eccb3958cf5644a671",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-cohort",
      "sha256": "389702ff4c647d7ce10a90092a6fa811ae777d15997baf39ce9aae0346247bd0",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-outcomes",
      "sha256": "a637ca0e307e66ff48811ec7efa22b9ce453bc7883b04f0cacb867f7283132d8",
      "uri": "https://raw.githubusercontent.com/KosuriLab/MFASS/master/processed_data/snv/snv_data_clean.txt",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-annotations",
      "sha256": "71a857fe647c4e68acbb41ca61e959c47e1176de89b1442bd6ca1772aa60d5a1",
      "uri": "https://raw.githubusercontent.com/KosuriLab/MFASS/master/processed_data/snv/snv_func_annot.txt",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-split",
      "sha256": "999ebcb7e63a5c5eaa8780fa468e59ac1f934260ad50102814174c396317f052",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/splits/split-v2.tsv",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-baseline-kmer-position-v2-scores",
      "sha256": "ed0ebb3d74183deb1fb475d0e706c0b1211fd6ab9d20449c2c194e57f912d862",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/baseline-kmer-position-v2.scores.npy",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-baseline-kmer-position-v2-predictions",
      "sha256": "2f3117c225a8da9ea737abfa2d0f7e1dec97696ae4bacd263864bec9d7f923eb",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/baseline-kmer-position-v2.predictions.tsv",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-baseline-kmer-position-v2-report",
      "sha256": "9a0b78674cc714177fec6e8c487588d6186d4fe93d6d7dbcb48bed6858885e15",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/baseline-kmer-position-v2.json",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-dnabert2-117m-frozen-pair-logreg-scores",
      "sha256": "e6b019babf0b3fb9ec05d4261a9383c0c896183a76c3b31634564c7a63d01105",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/dnabert2-117m-frozen-pair-logreg.scores.npy",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-dnabert2-117m-frozen-pair-logreg-predictions",
      "sha256": "3abded2932366e6e2c8d6e0da5836c4240239372758b3c68cd5f19635b8cee11",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/dnabert2-117m-frozen-pair-logreg.predictions.tsv",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-dnabert2-117m-frozen-pair-logreg-report",
      "sha256": "60b28541853de349f878d6d1ccbcbe7dfd21b69db976e9c01f60e88bdba0f2a1",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/dnabert2-117m-frozen-pair-logreg.json",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-spliceai-1.3.1-predictions",
      "sha256": "b39df773067e4ecd862980f177da69d6117c75c93c754f4cbdd7a77bd419851f",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/spliceai-1.3.1.predictions.tsv",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-spliceai-1.3.1-report",
      "sha256": "6d5c59eb0fd60d95064d97e331c9cdb12b2a34db7dc509ff73dfb9502ddf02dd",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/spliceai-1.3.1.json",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-pangolin-maskFalse-predictions",
      "sha256": "faa7d4cb3728111f1c5a8e8be6cd2c5958c4cced7c7a366ec96171d03b855ad6",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/pangolin-maskFalse.predictions.tsv",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-pangolin-maskFalse-report",
      "sha256": "bb0bb6732808699e54938233df1835dfc1f775f33ba7d6acd916e53d588a3c44",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/benchmarks/mfass/results/pangolin-maskFalse.json",
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-table",
      "sha256": "c57f9cfe91d02ab54f6bc20934f5b8f678b37c3b13d8d20aa805fa7d73e62c4f",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-receipt",
      "sha256": "a9fa54bcf99457f9b23424ad656a952eb262b376845327e267dc035cf85a012e",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-preparation_code",
      "sha256": "8a1d691de8fdd9fcf8242fe6fa2cfe2703a1c3281b880fe2a226cb8bd946fd1b",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mfass-v2-discrepancies-environment",
      "sha256": "748257abb3664595db1ebf892799e91c6f1a4ca79b0e65eccb3958cf5644a671",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-prepared",
      "sha256": "e5d6fb9e5fadec1f9e6235ca3ed2054ac05f476ccc8f5dc67ec500febeae8237",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-mrnabench-composition-predictions",
      "sha256": "a8f80cfea7ddd197ef9446e161f5a761741b3f778dfc1459275c3355432e15c5",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-mrnabench-composition-report",
      "sha256": "8b68dfe5725bf5d8c108265039e3787f4b3b55986eab8a430082c4d446a4f960",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/mrnabench-composition/report.json",
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-mrnabench-composition-source",
      "sha256": "ef486f7be79a6f5589afd7494b7acfa13ca698da50f0ffd0e50fd6eba1eb9934",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/mrnabench-composition/source.json",
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-mrnabench-train-mean-predictions",
      "sha256": "fb435ffda2e56792a9c781fba8dfda535b3d6508cf28f484162c5fd9065a87d8",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-mrnabench-train-mean-report",
      "sha256": "c036d9fa75b6ba4d5c65d31cd575ee2bfc357beec3ca7e6c80f07e3128a04e71",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/mrnabench-train-mean/report.json",
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-mrnabench-train-mean-source",
      "sha256": "b8bbc4f3f2ef54d92721015205bdd73a81e4cfcbc596985e8724be3416899d54",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/mrnabench-train-mean/source.json",
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-recipe_code",
      "sha256": "b3407940ca4c8a1cdcf3a3bb3849c3e26d7f85272b2ee63087307378ffaa79dc",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/packages/rewirebench/src/rewirebench/adapters/sequence.py",
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-table",
      "sha256": "d49ff1b4dacdfe0663c35bdad6a9ad91e79f5862fe96f36a627acb8407610bf3",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-receipt",
      "sha256": "5edeb10e42be225f6b85481c95eb8985ab1724d3af353737fdafd29ccdfc4022",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-preparation_code",
      "sha256": "8a1d691de8fdd9fcf8242fe6fa2cfe2703a1c3281b880fe2a226cb8bd946fd1b",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "mrnabench-discrepancies-environment",
      "sha256": "748257abb3664595db1ebf892799e91c6f1a4ca79b0e65eccb3958cf5644a671",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "proteingym-amfr-discrepancies-prepared",
      "sha256": "d9e220a0e2b0675e73f4b79b4ae45b8f807dedbb886bbd4b2f8e886822ddf57f",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "proteingym-amfr-discrepancies-proteingym-esm2-predictions",
      "sha256": "7ab0c8901406d5c1ab759038c8beef235721938394cfdf192c46671e98e0bbca",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "proteingym-amfr-discrepancies-proteingym-esm2-report",
      "sha256": "6a14b3866141d5779c76da8071d85c73b7cce0dc4e101c93122706873e847d26",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/proteingym-esm2/report.json",
      "status": "passed"
    },
    {
      "id": "proteingym-amfr-discrepancies-proteingym-esm2-source",
      "sha256": "90888f1f18625cdc2e9fb0bf7ecc1b1c4c2d6b700e8b4971e4349297926f996f",
      "uri": "https://raw.githubusercontent.com/rewire-bio/rewire-benchmarks/ca73fa47136d182f2d4ddb083d084712198fc0e2/research/local-runs-2026-09-20/proteingym-esm2/retrieval.json",
      "status": "passed"
    },
    {
      "id": "proteingym-amfr-discrepancies-table",
      "sha256": "4fac11aacc0f00a70e385ed7b8d73427aaa0716906a74e93bc3d48df5ff43f9f",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "proteingym-amfr-discrepancies-receipt",
      "sha256": "9f117c65a3c860b83735d4874d06a807ee1a1523464e9afca493e7dcd9e59b56",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "proteingym-amfr-discrepancies-preparation_code",
      "sha256": "8a1d691de8fdd9fcf8242fe6fa2cfe2703a1c3281b880fe2a226cb8bd946fd1b",
      "uri": null,
      "status": "passed"
    },
    {
      "id": "proteingym-amfr-discrepancies-environment",
      "sha256": "748257abb3664595db1ebf892799e91c6f1a4ca79b0e65eccb3958cf5644a671",
      "uri": null,
      "status": "passed"
    }
  ],
  "metric_replay": [
    {
      "case": "flip2-discrepancies",
      "method": "flip2-composition",
      "metric": "ndcg",
      "actual": 0.954819941821582,
      "status": "passed"
    },
    {
      "case": "flip2-discrepancies",
      "method": "flip2-composition",
      "metric": "spearman",
      "actual": 0.41818225155801514,
      "status": "passed"
    },
    {
      "case": "flip2-discrepancies",
      "method": "flip2-train-mean",
      "metric": "ndcg",
      "actual": 0.9206667522227658,
      "status": "passed"
    },
    {
      "case": "flip2-discrepancies",
      "method": "flip2-train-mean",
      "metric": "spearman",
      "actual": null,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "baseline-kmer-position-v2",
      "metric": "auroc",
      "actual": 0.7779498064677238,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "baseline-kmer-position-v2",
      "metric": "average_precision_sklearn",
      "actual": 0.2864167459589237,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "baseline-kmer-position-v2",
      "metric": "precision_at_capacity",
      "actual": 0.61,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "baseline-kmer-position-v2",
      "metric": "recall_at_capacity",
      "actual": 0.19365079365079366,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "dnabert2-117m-frozen-pair-logreg",
      "metric": "auroc",
      "actual": 0.5500324040216661,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "dnabert2-117m-frozen-pair-logreg",
      "metric": "average_precision_sklearn",
      "actual": 0.04508654312652131,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "dnabert2-117m-frozen-pair-logreg",
      "metric": "precision_at_capacity",
      "actual": 0.03,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "dnabert2-117m-frozen-pair-logreg",
      "metric": "recall_at_capacity",
      "actual": 0.009523809523809525,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "pangolin-maskFalse",
      "metric": "auroc",
      "actual": 0.8756851300560864,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "pangolin-maskFalse",
      "metric": "average_precision_sklearn",
      "actual": 0.3887617543064248,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "pangolin-maskFalse",
      "metric": "precision_at_capacity",
      "actual": 0.65,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "pangolin-maskFalse",
      "metric": "recall_at_capacity",
      "actual": 0.2070063694267516,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "spliceai-1.3.1",
      "metric": "auroc",
      "actual": 0.8055241740253153,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "spliceai-1.3.1",
      "metric": "average_precision_sklearn",
      "actual": 0.2986855472760137,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "spliceai-1.3.1",
      "metric": "precision_at_capacity",
      "actual": 0.64,
      "status": "passed"
    },
    {
      "case": "mfass-v2-discrepancies",
      "method": "spliceai-1.3.1",
      "metric": "recall_at_capacity",
      "actual": 0.2077922077922078,
      "status": "passed"
    },
    {
      "case": "mrnabench-discrepancies",
      "method": "mrnabench-composition",
      "metric": "mse",
      "actual": 1.914439715839851,
      "status": "passed"
    },
    {
      "case": "mrnabench-discrepancies",
      "method": "mrnabench-composition",
      "metric": "pearson",
      "actual": 0.436786917017099,
      "status": "passed"
    },
    {
      "case": "mrnabench-discrepancies",
      "method": "mrnabench-composition",
      "metric": "spearman",
      "actual": 0.49477537290902324,
      "status": "passed"
    },
    {
      "case": "mrnabench-discrepancies",
      "method": "mrnabench-train-mean",
      "metric": "mse",
      "actual": 2.3655294722554605,
      "status": "passed"
    },
    {
      "case": "mrnabench-discrepancies",
      "method": "mrnabench-train-mean",
      "metric": "pearson",
      "actual": null,
      "status": "passed"
    },
    {
      "case": "mrnabench-discrepancies",
      "method": "mrnabench-train-mean",
      "metric": "spearman",
      "actual": null,
      "status": "passed"
    },
    {
      "case": "proteingym-amfr-discrepancies",
      "method": "proteingym-esm2",
      "metric": "AUC",
      "actual": 0.394,
      "status": "passed"
    },
    {
      "case": "proteingym-amfr-discrepancies",
      "method": "proteingym-esm2",
      "metric": "MCC",
      "actual": -0.139,
      "status": "passed"
    },
    {
      "case": "proteingym-amfr-discrepancies",
      "method": "proteingym-esm2",
      "metric": "NDCG",
      "actual": 0.44,
      "status": "passed"
    },
    {
      "case": "proteingym-amfr-discrepancies",
      "method": "proteingym-esm2",
      "metric": "Spearman",
      "actual": -0.209,
      "status": "passed"
    },
    {
      "case": "proteingym-amfr-discrepancies",
      "method": "proteingym-esm2",
      "metric": "Top_recall",
      "actual": 0.057,
      "status": "passed"
    }
  ],
  "metric_replay_checks_passed": 31,
  "boundary_metric_checks_passed": 64,
  "mfass": {
    "test_n": 8324,
    "test_positives": 315,
    "common_n": 8194,
    "methods": {
      "pangolin-maskFalse": {
        "separate": {
          "precision_at_capacity": 0.65,
          "recall_at_capacity": 0.2070063694267516,
          "average_precision_sklearn": 0.3887617543064248,
          "auroc": 0.8756851300560864,
          "n": 8301,
          "positives": 314,
          "capacity": 100,
          "hits": 65,
          "cutoff_tie": false,
          "cutoff_score": 0.55,
          "next_score": 0.54
        },
        "common": {
          "precision_at_capacity": 0.65,
          "recall_at_capacity": 0.21103896103896103,
          "average_precision_sklearn": 0.3908176114823432,
          "auroc": 0.8753433670058066,
          "n": 8194,
          "positives": 308,
          "capacity": 100,
          "hits": 65,
          "cutoff_tie": false,
          "cutoff_score": 0.55,
          "next_score": 0.54
        },
        "top100_membership_unchanged": true,
        "removed_rows": 107,
        "removed_positives": 6
      },
      "spliceai-1.3.1": {
        "separate": {
          "precision_at_capacity": 0.64,
          "recall_at_capacity": 0.2077922077922078,
          "average_precision_sklearn": 0.2986855472760137,
          "auroc": 0.8055241740253153,
          "n": 8194,
          "positives": 308,
          "capacity": 100,
          "hits": 64,
          "cutoff_tie": false,
          "cutoff_score": 0.67,
          "next_score": 0.66
        },
        "common": {
          "precision_at_capacity": 0.64,
          "recall_at_capacity": 0.2077922077922078,
          "average_precision_sklearn": 0.2986855472760137,
          "auroc": 0.8055241740253153,
          "n": 8194,
          "positives": 308,
          "capacity": 100,
          "hits": 64,
          "cutoff_tie": false,
          "cutoff_score": 0.67,
          "next_score": 0.66
        },
        "top100_membership_unchanged": true,
        "removed_rows": 0,
        "removed_positives": 0
      }
    },
    "boundary_subgroups": [
      {
        "label": "0-2",
        "n": 448,
        "positives": 42,
        "outcome_mean": 0.09375,
        "methods": {
          "baseline-kmer-position-v2": {
            "precision_at_capacity": 0.29,
            "recall_at_capacity": 0.6904761904761905,
            "average_precision_sklearn": 0.5183134824574966,
            "auroc": 0.8377902885292048,
            "n": 448,
            "positives": 42,
            "capacity": 100,
            "hits": 29,
            "cutoff_tie": false,
            "cutoff_score": 0.07880887890260946,
            "next_score": 0.0785705838642733
          },
          "dnabert2-117m-frozen-pair-logreg": {
            "precision_at_capacity": 0.16,
            "recall_at_capacity": 0.38095238095238093,
            "average_precision_sklearn": 0.13520653880519168,
            "auroc": 0.6020408163265306,
            "n": 448,
            "positives": 42,
            "capacity": 100,
            "hits": 16,
            "cutoff_tie": false,
            "cutoff_score": 0.48181229519142627,
            "next_score": 0.4691866301824855
          },
          "pangolin-maskFalse": {
            "precision_at_capacity": 0.34,
            "recall_at_capacity": 0.8095238095238095,
            "average_precision_sklearn": 0.6453656386173421,
            "auroc": 0.9238212526389866,
            "n": 448,
            "positives": 42,
            "capacity": 100,
            "hits": 34,
            "cutoff_tie": true,
            "cutoff_score": 0.05,
            "next_score": 0.05
          },
          "spliceai-1.3.1": {
            "precision_at_capacity": 0.31,
            "recall_at_capacity": 0.7560975609756098,
            "average_precision_sklearn": 0.5857307167402802,
            "auroc": 0.9017716296565951,
            "n": 443,
            "positives": 41,
            "capacity": 100,
            "hits": 31,
            "cutoff_tie": true,
            "cutoff_score": 0.06,
            "next_score": 0.06
          }
        }
      },
      {
        "label": "11-30",
        "n": 4239,
        "positives": 160,
        "outcome_mean": 0.0377447511205473,
        "methods": {
          "baseline-kmer-position-v2": {
            "precision_at_capacity": 0.38,
            "recall_at_capacity": 0.2375,
            "average_precision_sklearn": 0.26347426873322705,
            "auroc": 0.7642881220887472,
            "n": 4239,
            "positives": 160,
            "capacity": 100,
            "hits": 38,
            "cutoff_tie": false,
            "cutoff_score": 0.15427048146830472,
            "next_score": 0.1540765633391661
          },
          "dnabert2-117m-frozen-pair-logreg": {
            "precision_at_capacity": 0.04,
            "recall_at_capacity": 0.025,
            "average_precision_sklearn": 0.04278432946451933,
            "auroc": 0.5168101863201765,
            "n": 4239,
            "positives": 160,
            "capacity": 100,
            "hits": 4,
            "cutoff_tie": false,
            "cutoff_score": 0.9966659108516078,
            "next_score": 0.9965495607161495
          },
          "pangolin-maskFalse": {
            "precision_at_capacity": 0.48,
            "recall_at_capacity": 0.3018867924528302,
            "average_precision_sklearn": 0.36263307502559783,
            "auroc": 0.868495158918307,
            "n": 4226,
            "positives": 159,
            "capacity": 100,
            "hits": 48,
            "cutoff_tie": true,
            "cutoff_score": 0.19,
            "next_score": 0.19
          },
          "spliceai-1.3.1": {
            "precision_at_capacity": 0.42,
            "recall_at_capacity": 0.2641509433962264,
            "average_precision_sklearn": 0.28420482071335973,
            "auroc": 0.7848152525533147,
            "n": 4166,
            "positives": 159,
            "capacity": 100,
            "hits": 42,
            "cutoff_tie": true,
            "cutoff_score": 0.24,
            "next_score": 0.24
          }
        }
      },
      {
        "label": "3-10",
        "n": 1690,
        "positives": 81,
        "outcome_mean": 0.04792899408284024,
        "methods": {
          "baseline-kmer-position-v2": {
            "precision_at_capacity": 0.3,
            "recall_at_capacity": 0.37037037037037035,
            "average_precision_sklearn": 0.2818487669278446,
            "auroc": 0.7368659316038642,
            "n": 1690,
            "positives": 81,
            "capacity": 100,
            "hits": 30,
            "cutoff_tie": false,
            "cutoff_score": 0.14425589567909336,
            "next_score": 0.1433132089650353
          },
          "dnabert2-117m-frozen-pair-logreg": {
            "precision_at_capacity": 0.05,
            "recall_at_capacity": 0.06172839506172839,
            "average_precision_sklearn": 0.0628302214735115,
            "auroc": 0.591840649433357,
            "n": 1690,
            "positives": 81,
            "capacity": 100,
            "hits": 5,
            "cutoff_tie": false,
            "cutoff_score": 0.9800823326841788,
            "next_score": 0.9797558628772424
          },
          "pangolin-maskFalse": {
            "precision_at_capacity": 0.43,
            "recall_at_capacity": 0.5308641975308642,
            "average_precision_sklearn": 0.384449295653607,
            "auroc": 0.8594054074843276,
            "n": 1686,
            "positives": 81,
            "capacity": 100,
            "hits": 43,
            "cutoff_tie": true,
            "cutoff_score": 0.14,
            "next_score": 0.14
          },
          "spliceai-1.3.1": {
            "precision_at_capacity": 0.32,
            "recall_at_capacity": 0.4155844155844156,
            "average_precision_sklearn": 0.27226583608733973,
            "auroc": 0.7977561961250794,
            "n": 1671,
            "positives": 77,
            "capacity": 100,
            "hits": 32,
            "cutoff_tie": true,
            "cutoff_score": 0.16,
            "next_score": 0.16
          }
        }
      },
      {
        "label": ">30",
        "n": 1947,
        "positives": 32,
        "outcome_mean": 0.016435541859270673,
        "methods": {
          "baseline-kmer-position-v2": {
            "precision_at_capacity": 0.04,
            "recall_at_capacity": 0.125,
            "average_precision_sklearn": 0.043619720237491,
            "auroc": 0.7594484334203656,
            "n": 1947,
            "positives": 32,
            "capacity": 100,
            "hits": 4,
            "cutoff_tie": false,
            "cutoff_score": 0.04052875937977392,
            "next_score": 0.040224939435734824
          },
          "dnabert2-117m-frozen-pair-logreg": {
            "precision_at_capacity": 0.02,
            "recall_at_capacity": 0.0625,
            "average_precision_sklearn": 0.02098274499062513,
            "auroc": 0.5490535248041775,
            "n": 1947,
            "positives": 32,
            "capacity": 100,
            "hits": 2,
            "cutoff_tie": false,
            "cutoff_score": 0.9850092907674982,
            "next_score": 0.9844730897114375
          },
          "pangolin-maskFalse": {
            "precision_at_capacity": 0.15,
            "recall_at_capacity": 0.46875,
            "average_precision_sklearn": 0.1486754959430206,
            "auroc": 0.8376849790466212,
            "n": 1941,
            "positives": 32,
            "capacity": 100,
            "hits": 15,
            "cutoff_tie": true,
            "cutoff_score": 0.05,
            "next_score": 0.05
          },
          "spliceai-1.3.1": {
            "precision_at_capacity": 0.09,
            "recall_at_capacity": 0.2903225806451613,
            "average_precision_sklearn": 0.07335603629452685,
            "auroc": 0.7437513919106435,
            "n": 1914,
            "positives": 31,
            "capacity": 100,
            "hits": 9,
            "cutoff_tie": true,
            "cutoff_score": 0.07,
            "next_score": 0.07
          }
        }
      }
    ]
  },
  "flip2": {
    "n": 184,
    "training_n": 584,
    "training_mean": 539.7328767123288,
    "composition_mse": 1042.666651290419,
    "train_mean_mse": 1056.3941660479575,
    "relative_mse_reduction": 0.012994690049164221,
    "composition_spearman": 0.41818225155801514,
    "constant_spearman": null,
    "source_provenance": {
      "data_verification": "pinned_source_bytes",
      "dataset_version": "zenodo-18433203-v3",
      "evaluator_url": "https://github.com/J-SNACKKB/FLIP/blob/62cace8735f5610e2743cf06ce0f944b37fffaa6/baselines/aggregate.py#L131-L135",
      "source_csv_sha256": "8e78f6a16cd5298131dca83130d4b94cc0ab4ae9c699460880441c19a65f656f",
      "source_sha256": "7c6d2f02cb89310378ac9897c321fbbd909fb0757ac88803dcce84f5f6b05ce3",
      "source_url": "https://zenodo.org/api/records/18433203/files/rhomax/by_wild_type.csv.gz/content",
      "split_origin": "Archived assignments: validation=True held out of training",
      "upstream_revision": "62cace8735f5610e2743cf06ce0f944b37fffaa6"
    },
    "constant_ndcg": 0.9206667522227658,
    "composition_ndcg": 0.954819941821582,
    "ndcg_difference": 0.03415318959881619,
    "constant_expected_dcg": 3219.780347860353,
    "ideal_dcg": 3497.22670020052,
    "gain_minimum_shift": 436.0,
    "all_predictions_equal": true
  },
  "mrnabench": {
    "n": 15003,
    "training_n": 70011,
    "training_mean": 5.7475703435024,
    "composition_mse": 1.914439715839851,
    "train_mean_mse": 2.3655294722554605,
    "relative_mse_reduction": 0.1906929343752835,
    "composition_spearman": 0.49477537290902324,
    "constant_spearman": null,
    "source_provenance": {
      "data_verification": "pinned_source_bytes",
      "dataset_revision": "ef67f7cf8a999bb1c412ad6551aa7d9f901cbb95",
      "source_license": "unreported",
      "source_sha256": "8b8c57581d472e89a21f8b468b3b0599f6c235c01423264561e6c302d78927d4",
      "source_url": "https://huggingface.co/datasets/morrislab/mrl-sample/resolve/ef67f7cf8a999bb1c412ad6551aa7d9f901cbb95/mrl-sample-designed.parquet",
      "split_sha256": "00e287b18ba4e90cb682af6e3fcdfc97e07f2b4b25d13265cd0eca37996536a2",
      "upstream_code_license": "AGPL-3.0",
      "upstream_revision": "74f96b8e6ae9f41cc3cccff089d826a62d5604b8"
    }
  },
  "proteingym": {
    "n": 2972,
    "assay": "AMFR_HUMAN_Tsuboyama_2023_4G3O",
    "true_top_count": 298,
    "true_90th_percentile": 0.04206218069865769,
    "constant_selected_count": 2972,
    "constant_true_top_hits": 298,
    "constant_top_recall": 1.0,
    "source_provenance": {
      "assay_sha256": {
        "AMFR_HUMAN_Tsuboyama_2023_4G3O": "dd911d925eca79329a496eb6cb7b181ab448e15db244daf1c1f9c01e8a704c2a"
      },
      "data_verification": "local_bytes_hashed_not_independently_source_verified",
      "expected_hashes_checked": [
        "AMFR_HUMAN_Tsuboyama_2023_4G3O"
      ],
      "reference_sha256": "a8f498011532a74aa9fe556a50555a75e928c5837d19c06a87592ae04049b308",
      "source_url": "https://github.com/OATML-Markslab/ProteinGym/tree/144fe22b07dfaeec2b366f2346203a9838a55b4c",
      "upstream_revision": "144fe22b07dfaeec2b366f2346203a9838a55b4c"
    }
  }
}