{
  "schema_version": 1,
  "generated_at": "2026-11-03T23:38:23Z",
  "source": {
    "repository": "https://github.com/AVERI-org/BenchRisk",
    "ref": "main",
    "commit": "f520c3cb6d11f61acb5b6bde938f8fc546b6ff98",
    "path": "data/scores",
    "site": "https://benchrisk.ai"
  },
  "cloudflare": {
    "namespace": "benchrisk",
    "instance": "benchmarks",
    "endpoints": {
      "mcp": "https://ns-f1fc49e3-9bca-4aa2-b5a1-ef5dbc18588f.search.ai.cloudflare.com/mcp",
      "search": "https://ns-f1fc49e3-9bca-4aa2-b5a1-ef5dbc18588f.search.ai.cloudflare.com/search",
      "chat_completions": "https://ns-f1fc49e3-9bca-4aa2-b5a1-ef5dbc18588f.search.ai.cloudflare.com/chat/completions"
    },
    "metadata_fields": [
      "benchmark",
      "benchmark_name",
      "doc_type",
      "source_url",
      "benchrisk_url"
    ],
    "doc_types": {
      "paper": "The paper BenchRisk cites as the benchmark's definitive reference (arXiv PDF, or the arXiv HTML rendering when the PDF exceeds Cloudflare's 4 MiB limit).",
      "website": "The project website BenchRisk cites as the benchmark's definitive reference.",
      "repository": "README of the benchmark's official code repository, when the BenchRisk entry names one."
    }
  },
  "benchmarks": [
    {
      "tag": "ailuminate-0.5",
      "name": "AILuminate05",
      "title": "Introducing v0.5 of the AI Safety Benchmark from MLCommons",
      "description": "Benchmark indicates the propensity of AI systems to respond in a hazardous manner to prompts from malicious or vulnerable users that might result in harm to themselves or others.",
      "reference": "https://arxiv.org/abs/2404.12241",
      "reference_note": null,
      "summary": "This paper introduces v0.5 of the AI Safety Benchmark, which has been created by the MLCommons AI Safety Working Group. The AI Safety Benchmark has been designed to assess the safety risks of AI systems that use chat-tuned language models. We introduce a principled approach to specifying and constructing the benchmark, which for v0.5 covers only a single use case (an adult chatting to a general-purpose assistant in English), and a limited set of personas (i.e., typical users, malicious users, and vulnerable users). We created a new taxonomy of 13 hazard categories, of which 7 have tests in the v0.5 benchmark. We plan to release version 1.0 of the AI Safety Benchmark by the end of 2024. The v1.0 benchmark will provide meaningful insights into the safety of AI systems. However, the v0.5 benchmark should not be used to assess the safety of AI systems. We have sought to fully document the limitations, flaws, and challenges of v0.5. This release of v0.5 of the AI Safety Benchmark includes (1) a principled approach to specifying and constructing the benchmark, which comprises use cases, types of systems under test (SUTs), language and context, personas, tests, and test items; (2) a taxonomy of 13 hazard categories with definitions and subcategories; (3) tests for seven of the hazard categories, each comprising a unique set of test items, i.e., prompts. There are 43,090 test items in total, which we created with templates; (4) a grading system for AI systems against the benchmark; (5) an openly available platform, and downloadable tool, called ModelBench that can be used to evaluate the safety of AI systems on the benchmark; (6) an example evaluation report which benchmarks the performance of over a dozen openly available chat-tuned language models; (7) a test specification for the benchmark.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/AILuminate05.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/AILuminate05.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          106,
          64,
          89,
          140,
          101,
          141,
          102,
          10,
          12,
          13,
          14,
          19,
          21,
          92,
          27,
          53,
          33,
          36,
          160,
          38,
          88,
          82,
          83,
          57,
          42,
          104,
          105,
          103,
          107,
          120,
          128,
          129,
          130,
          131,
          132,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          5,
          6,
          8,
          7,
          9,
          62,
          11,
          164,
          69,
          142,
          165,
          178,
          84,
          86,
          183,
          184,
          85,
          87,
          90,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          180,
          179,
          147,
          166,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          91,
          149,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          48,
          153,
          182,
          135,
          137,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          98,
          156,
          55,
          99,
          58,
          138,
          60,
          157,
          100,
          59,
          139,
          61,
          158,
          34,
          51,
          159,
          200,
          35,
          56,
          37,
          75,
          197,
          122,
          123,
          66,
          127,
          76,
          39,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          121,
          124,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2404.12241",
          "role": "definitive-reference",
          "key": "ailuminate-0.5/paper-arxiv-2404.12241.pdf",
          "content_url": "https://arxiv.org/pdf/2404.12241",
          "format": "pdf",
          "sha256": "92a06f300abae8d2979be732fc4cdb155434a2cfe3cb687ed06b5292944d2b9e",
          "size_bytes": 806881,
          "fetched_at": "2026-11-03T23:21:13Z",
          "uploaded_sha256": "92a06f300abae8d2979be732fc4cdb155434a2cfe3cb687ed06b5292944d2b9e"
        }
      ]
    },
    {
      "tag": "ailuminate-1.0",
      "name": "AILuminate10",
      "title": "AILuminate: Introducing v1.0 of the AI Risk and Reliability Benchmark from MLCommons",
      "description": "Benchmark indicates the propensity of AI systems to respond in a hazardous manner to prompts from malicious or vulnerable users that might result in harm to themselves or others",
      "reference": "https://arxiv.org/abs/2503.05731",
      "reference_note": null,
      "summary": "The rapid advancement and deployment of AI systems have created an urgent need for standard safety-evaluation frameworks. This paper introduces AILuminate v1.0, the first comprehensive industry-standard benchmark for assessing AI-product risk and reliability. Its development employed an open process that included participants from multiple fields. The benchmark evaluates an AI system's resistance to prompts designed to elicit dangerous, illegal, or undesirable behavior in 12 hazard categories, including violent crimes, nonviolent crimes, sex-related crimes, child sexual exploitation, indiscriminate weapons, suicide and self-harm, intellectual property, privacy, defamation, hate, sexual content, and specialized advice (election, financial, health, legal). Our method incorporates a complete assessment standard, extensive prompt datasets, a novel evaluation framework, a grading and reporting system, and the technical as well as organizational infrastructure for long-term support and evolution. In particular, the benchmark employs an understandable five-tier grading scale (Poor to Excellent) and incorporates an innovative entropy-based system-response evaluation.\nIn addition to unveiling the benchmark, this report also identifies limitations of our method and of building safety benchmarks generally, including evaluator uncertainty and the constraints of single-turn interactions. This work represents a crucial step toward establishing global standards for AI risk and reliability evaluation while acknowledging the need for continued development in areas such as multiturn interactions, multimodal understanding, coverage of additional languages, and emerging hazard categories. Our findings provide valuable insights for model developers, system integrators, and policymakers working to promote safer AI deployment.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/AILuminate10.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/AILuminate10.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          106,
          3,
          70,
          5,
          8,
          7,
          9,
          10,
          11,
          69,
          84,
          86,
          85,
          87,
          14,
          71,
          80,
          79,
          19,
          21,
          72,
          27,
          53,
          73,
          28,
          81,
          30,
          31,
          32,
          54,
          55,
          138,
          139,
          33,
          34,
          35,
          36,
          160,
          75,
          122,
          123,
          38,
          88,
          82,
          76,
          78,
          44,
          74,
          43,
          161,
          52,
          67,
          104,
          105,
          103,
          107,
          112,
          114,
          115,
          120,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136
        ],
        "absent_mitigations": [
          186,
          198,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          163,
          4,
          199,
          89,
          140,
          6,
          101,
          141,
          102,
          62,
          164,
          142,
          178,
          12,
          165,
          183,
          184,
          13,
          90,
          65,
          144,
          17,
          193,
          143,
          201,
          145,
          18,
          146,
          180,
          179,
          147,
          166,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          26,
          91,
          149,
          150,
          151,
          93,
          46,
          196,
          47,
          152,
          29,
          94,
          135,
          137,
          48,
          182,
          153,
          49,
          154,
          95,
          96,
          50,
          97,
          155,
          202,
          156,
          98,
          58,
          99,
          60,
          157,
          59,
          100,
          61,
          158,
          51,
          159,
          200,
          56,
          37,
          197,
          66,
          83,
          127,
          57,
          39,
          42,
          77,
          45,
          108,
          109,
          113,
          118,
          110,
          119,
          116,
          117,
          111,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2503.05731",
          "role": "definitive-reference",
          "key": "ailuminate-1.0/paper-arxiv-2503.05731.pdf",
          "content_url": "https://arxiv.org/pdf/2503.05731",
          "format": "pdf",
          "sha256": "c49f18e164aa2172fdc378c751b188fd7c047f29e58f0a722c2e3bc5d71ce63c",
          "size_bytes": 2312478,
          "fetched_at": "2026-11-03T23:21:16Z",
          "uploaded_sha256": "c49f18e164aa2172fdc378c751b188fd7c047f29e58f0a722c2e3bc5d71ce63c"
        }
      ]
    },
    {
      "tag": "air-bench-2024",
      "name": "AIRBench",
      "title": "AIR-Bench 2024: A Safety Benchmark Based on Risk Categories from Regulations and Policies",
      "description": "Benchmark evaluates LLM safety by measuring responses to prompts that are mapped to granular risk categories derived from government regulations and AI policies.",
      "reference": "https://arxiv.org/abs/2407.17436v2",
      "reference_note": null,
      "summary": "Foundation models (FMs) provide societal benefits but also amplify risks. Governments, companies, and researchers have proposed regulatory frameworks, acceptable use policies, and safety benchmarks in response. However, existing public benchmarks often define safety categories based on previous literature, intuitions, or common sense, leading to disjointed sets of categories for risks specified in recent regulations and policies, which makes it challenging to evaluate and compare FMs across these benchmarks. To bridge this gap, we introduce AIR-Bench 2024, the first AI safety benchmark aligned with emerging government regulations and company policies, following the regulation-based safety categories grounded in our AI risks study, AIR 2024. AIR 2024 decomposes 8 government regulations and 16 company policies into a four-tiered safety taxonomy with 314 granular risk categories in the lowest tier. AIR-Bench 2024 contains 5,694 diverse prompts spanning these categories, with manual curation and human auditing to ensure quality. We evaluate leading language models on AIR-Bench 2024, uncovering insights into their alignment with specified safety concerns. By bridging the gap between public benchmarks and practical AI risks, AIR-Bench 2024 provides a foundation for assessing model safety across jurisdictions, fostering the development of safer and more responsible AI systems.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/AIRBench.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/AIRBench.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          68,
          3,
          5,
          101,
          102,
          10,
          12,
          13,
          14,
          92,
          94,
          135,
          55,
          51,
          38,
          88,
          104,
          105,
          103,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          4,
          163,
          70,
          199,
          89,
          140,
          6,
          8,
          7,
          9,
          141,
          62,
          11,
          164,
          69,
          142,
          165,
          178,
          84,
          86,
          183,
          184,
          85,
          87,
          90,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          196,
          47,
          81,
          152,
          29,
          137,
          48,
          182,
          153,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          156,
          98,
          99,
          58,
          138,
          60,
          157,
          100,
          59,
          139,
          61,
          158,
          33,
          34,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          122,
          123,
          66,
          83,
          82,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          107,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2407.17436v2",
          "role": "definitive-reference",
          "key": "air-bench-2024/paper-arxiv-2407.17436v2.html",
          "content_url": "https://arxiv.org/html/2407.17436v2",
          "format": "html",
          "sha256": "6040eb7db306b75d0f8e3dd7a7e133b9b53ee7df5bf220623f8a627afa787018",
          "size_bytes": 302385,
          "fetched_at": "2026-11-03T23:21:23Z",
          "uploaded_sha256": "6040eb7db306b75d0f8e3dd7a7e133b9b53ee7df5bf220623f8a627afa787018",
          "uploaded_at": "2026-11-03T23:22:32Z"
        }
      ]
    },
    {
      "tag": "anthropic-red-team",
      "name": "AnthropicRedTeam",
      "title": "Red Teaming Language Models to Reduce Harms: Methods, Scaling Behaviors, and Lessons Learned",
      "description": "Benchmark indicates whether an LLM will refuse to generate malicious code when prompted, signaling its resistance to abuse and helping users evaluate the model’s safety and robustness.",
      "reference": "https://arxiv.org/pdf/2209.07858",
      "reference_note": null,
      "summary": "We describe our early efforts to red team language models in order to simultaneously discover, measure, and attempt to reduce their potentially harmful outputs. We make three main contributions. First, we investigate scaling behaviors for red teaming across 3 model sizes (2.7B, 13B, and 52B parameters) and 4 model types: a plain language model (LM); an LM prompted to be helpful, honest, and harmless; an LM with rejection sampling; and a model trained to be helpful and harmless using reinforcement learning from human feedback (RLHF). We find that the RLHF models are increasingly difficult to red team as they scale, and we find a flat trend with scale for the other model types. Second, we release our dataset of 38,961 red team attacks for others to analyze and learn from. We provide our own analysis of the data and find a variety of harmful outputs, which range from offensive language to more subtly harmful non-violent unethical outputs. Third, we exhaustively describe our instructions, processes, statistical methodologies, and uncertainty about red teaming. We hope that this transparency accelerates our ability to work together as a community in order to develop shared norms, practices, and technical standards for how to red team language models.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/AnthropicRedTeam.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/AnthropicRedTeam.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          198,
          70,
          199,
          140,
          14,
          65,
          71,
          143,
          80,
          19,
          138,
          139,
          33,
          34,
          200,
          35,
          36,
          122,
          123,
          38,
          104,
          128,
          129,
          130,
          131,
          124,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          1,
          186,
          106,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          89,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          10,
          62,
          11,
          164,
          69,
          142,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          13,
          87,
          90,
          144,
          17,
          193,
          201,
          145,
          18,
          79,
          146,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          48,
          153,
          182,
          135,
          137,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          98,
          156,
          55,
          58,
          99,
          60,
          157,
          59,
          100,
          61,
          158,
          51,
          159,
          56,
          160,
          197,
          75,
          37,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          105,
          103,
          107,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          132,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/pdf/2209.07858",
          "role": "definitive-reference",
          "key": "anthropic-red-team/paper-arxiv-2209.07858.pdf",
          "content_url": "https://arxiv.org/pdf/2209.07858",
          "format": "pdf",
          "sha256": "4396213de970d6fcdf9f6659056ad0d8cde5ee88cc522713ae9918105496b794",
          "size_bytes": 3266545,
          "fetched_at": "2026-11-03T23:21:26Z",
          "uploaded_sha256": "4396213de970d6fcdf9f6659056ad0d8cde5ee88cc522713ae9918105496b794"
        }
      ]
    },
    {
      "tag": "arc-agi-private",
      "name": "ARCAGIPrivate",
      "title": "On the Measure of Intelligence",
      "description": "Benchmark indicates how well a system can perform abstract reasoning tasks that require generalization from minimal examples, reflecting human-like fluid intelligence.",
      "reference": "https://arxiv.org/abs/1911.01547",
      "reference_note": null,
      "summary": "To make deliberate progress towards more intelligent and more human-like artificial systems, we need to be following an appropriate feedback signal: we need to be able to define and evaluate intelligence in a way that enables comparisons between two systems, as well as comparisons with humans. Over the past hundred years, there has been an abundance of attempts to define and measure intelligence, across both the fields of psychology and AI. We summarize and critically assess these definitions and evaluation approaches, while making apparent the two historical conceptions of intelligence that have implicitly guided them. We note that in practice, the contemporary AI community still gravitates towards benchmarking intelligence by comparing the skill exhibited by AIs and humans at specific tasks such as board games and video games. We argue that solely measuring skill at any given task falls short of measuring intelligence, because skill is heavily modulated by prior knowledge and experience: unlimited priors or unlimited training data allow experimenters to \"buy\" arbitrary levels of skills for a system, in a way that masks the system's own generalization power. We then articulate a new formal definition of intelligence based on Algorithmic Information Theory, describing intelligence as skill-acquisition efficiency and highlighting the concepts of scope, generalization difficulty, priors, and experience. Using this definition, we propose a set of guidelines for what a general AI benchmark should look like. Finally, we present a benchmark closely following these guidelines, the Abstraction and Reasoning Corpus (ARC), built upon an explicit set of priors designed to be as close as possible to innate human priors. We argue that ARC can be used to measure a human-like form of general fluid intelligence and that it enables fair general intelligence comparisons between AI systems and humans.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/ARCAGIPrivate.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/ARCAGIPrivate.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          68,
          89,
          140,
          5,
          101,
          141,
          102,
          142,
          13,
          90,
          145,
          146,
          147,
          134,
          148,
          92,
          24,
          151,
          28,
          152,
          94,
          153,
          154,
          32,
          96,
          155,
          156,
          157,
          158,
          159,
          160,
          122,
          123,
          38,
          88,
          82,
          43,
          161,
          67,
          104,
          105,
          103,
          107,
          108,
          112,
          115,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136,
          93
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          6,
          8,
          7,
          9,
          10,
          62,
          11,
          164,
          69,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          201,
          18,
          79,
          19,
          179,
          180,
          21,
          166,
          194,
          22,
          195,
          181,
          23,
          185,
          63,
          72,
          25,
          26,
          27,
          91,
          149,
          53,
          150,
          73,
          46,
          81,
          196,
          29,
          47,
          135,
          137,
          48,
          182,
          30,
          49,
          31,
          95,
          50,
          97,
          54,
          202,
          98,
          55,
          138,
          99,
          58,
          60,
          59,
          139,
          100,
          61,
          33,
          34,
          51,
          200,
          35,
          56,
          36,
          37,
          75,
          197,
          66,
          83,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          45,
          52,
          109,
          113,
          114,
          118,
          110,
          119,
          116,
          117,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/1911.01547",
          "role": "definitive-reference",
          "key": "arc-agi-private/paper-arxiv-1911.01547.pdf",
          "content_url": "https://arxiv.org/pdf/1911.01547",
          "format": "pdf",
          "sha256": "773b9ecec3df33e664ab5a784f9e48818e7fd0d8e211b3243f4f4e0fdecb2fee",
          "size_bytes": 1280262,
          "fetched_at": "2026-11-03T23:21:29Z",
          "uploaded_sha256": "773b9ecec3df33e664ab5a784f9e48818e7fd0d8e211b3243f4f4e0fdecb2fee"
        }
      ]
    },
    {
      "tag": "arc-agi-public",
      "name": "ARCAGIPublic",
      "title": "On the Measure of Intelligence",
      "description": "Benchmark indicates how well a system can perform abstract reasoning tasks that require generalization from minimal examples, reflecting human-like fluid intelligence.",
      "reference": "https://arxiv.org/abs/1911.01547",
      "reference_note": null,
      "summary": "To make deliberate progress towards more intelligent and more human-like artificial systems, we need to be following an appropriate feedback signal: we need to be able to define and evaluate intelligence in a way that enables comparisons between two systems, as well as comparisons with humans. Over the past hundred years, there has been an abundance of attempts to define and measure intelligence, across both the fields of psychology and AI. We summarize and critically assess these definitions and evaluation approaches, while making apparent the two historical conceptions of intelligence that have implicitly guided them. We note that in practice, the contemporary AI community still gravitates towards benchmarking intelligence by comparing the skill exhibited by AIs and humans at specific tasks such as board games and video games. We argue that solely measuring skill at any given task falls short of measuring intelligence, because skill is heavily modulated by prior knowledge and experience: unlimited priors or unlimited training data allow experimenters to \"buy\" arbitrary levels of skills for a system, in a way that masks the system's own generalization power. We then articulate a new formal definition of intelligence based on Algorithmic Information Theory, describing intelligence as skill-acquisition efficiency and highlighting the concepts of scope, generalization difficulty, priors, and experience. Using this definition, we propose a set of guidelines for what a general AI benchmark should look like. Finally, we present a benchmark closely following these guidelines, the Abstraction and Reasoning Corpus (ARC), built upon an explicit set of priors designed to be as close as possible to innate human priors. We argue that ARC can be used to measure a human-like form of general fluid intelligence and that it enables fair general intelligence comparisons between AI systems and humans.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/ARCAGIPublic.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/ARCAGIPublic.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          68,
          89,
          140,
          5,
          101,
          141,
          102,
          142,
          13,
          90,
          145,
          146,
          147,
          134,
          148,
          92,
          152,
          94,
          153,
          154,
          96,
          155,
          156,
          157,
          158,
          159,
          160,
          122,
          123,
          38,
          88,
          82,
          104,
          105,
          103,
          107,
          108,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136,
          93
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          6,
          8,
          7,
          9,
          10,
          62,
          11,
          164,
          69,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          201,
          18,
          79,
          19,
          179,
          180,
          21,
          166,
          194,
          22,
          195,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          46,
          81,
          196,
          29,
          47,
          135,
          137,
          48,
          182,
          30,
          49,
          31,
          95,
          32,
          50,
          97,
          54,
          202,
          98,
          55,
          138,
          99,
          58,
          60,
          59,
          139,
          100,
          61,
          33,
          34,
          51,
          200,
          35,
          56,
          36,
          37,
          75,
          197,
          66,
          83,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/1911.01547",
          "role": "definitive-reference",
          "key": "arc-agi-public/paper-arxiv-1911.01547.pdf",
          "content_url": "https://arxiv.org/pdf/1911.01547",
          "format": "pdf",
          "sha256": "773b9ecec3df33e664ab5a784f9e48818e7fd0d8e211b3243f4f4e0fdecb2fee",
          "size_bytes": 1280262,
          "fetched_at": "2026-11-03T23:21:32Z",
          "uploaded_sha256": "773b9ecec3df33e664ab5a784f9e48818e7fd0d8e211b3243f4f4e0fdecb2fee"
        }
      ]
    },
    {
      "tag": "bbq",
      "name": "BBQ",
      "title": "BBQ: A Hand-Built Bias Benchmark for Question Answering",
      "description": "Benchmark measures social biases against people belonging to protected classes along nine social dimensions relevant for U.S. English-speaking contexts.",
      "reference": "https://arxiv.org/abs/2110.08193",
      "reference_note": null,
      "summary": "It is well documented that NLP models learn social biases, but little work has been done on how these biases manifest in model outputs for applied tasks like question answering (QA). We introduce the Bias Benchmark for QA (BBQ), a dataset of question sets constructed by the authors that highlight attested social biases against people belonging to protected classes along nine social dimensions relevant for U.S. English-speaking contexts. Our task evaluates model responses at two levels: (i) given an under-informative context, we test how strongly responses reflect social biases, and (ii) given an adequately informative context, we test whether the model's biases override a correct answer choice. We find that models often rely on stereotypes when the context is under-informative, meaning the model's outputs consistently reproduce harmful biases in this setting. Though models are more accurate when the context provides an informative answer, they still rely on stereotypes and average up to 3.4 percentage points higher accuracy when the correct answer aligns with a social bias than when it conflicts, with this difference widening to over 5 points on examples targeting gender for most models tested.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/bbq.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/bbq.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          68,
          89,
          5,
          101,
          102,
          10,
          62,
          13,
          90,
          19,
          21,
          22,
          92,
          93,
          94,
          30,
          95,
          96,
          97,
          98,
          99,
          100,
          33,
          34,
          122,
          123,
          38,
          104,
          105,
          103,
          109,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          140,
          6,
          8,
          7,
          9,
          141,
          11,
          164,
          69,
          142,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          180,
          179,
          147,
          166,
          194,
          134,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          46,
          196,
          47,
          81,
          152,
          29,
          135,
          137,
          48,
          182,
          153,
          49,
          154,
          31,
          32,
          50,
          155,
          54,
          202,
          156,
          55,
          138,
          58,
          157,
          60,
          59,
          139,
          61,
          158,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          107,
          108,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2110.08193",
          "role": "definitive-reference",
          "key": "bbq/paper-arxiv-2110.08193.pdf",
          "content_url": "https://arxiv.org/pdf/2110.08193",
          "format": "pdf",
          "sha256": "7aa0d9642bced4a5e08c16187ca9c8303b860267801122aa2a43c4757fe21c6d",
          "size_bytes": 1204913,
          "fetched_at": "2026-11-03T23:21:35Z",
          "uploaded_sha256": "7aa0d9642bced4a5e08c16187ca9c8303b860267801122aa2a43c4757fe21c6d"
        }
      ]
    },
    {
      "tag": "big-bench",
      "name": "BigBench",
      "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
      "description": "Benchmark indicates how well a system can perform across a diverse set of 204 challenging tasks spanning linguistics, mathematics, commonsense reasoning, and more, reflecting its generalization and reasoning capabilities.",
      "reference": "https://arxiv.org/abs/2206.04615",
      "reference_note": null,
      "summary": "Language models demonstrate both quantitative improvement and new qualitative capabilities with increasing scale. Despite their potentially transformative impact, these new capabilities are as yet poorly characterized. In order to inform future research, prepare for disruptive new model capabilities, and ameliorate socially harmful effects, it is vital that we understand the present and near-future capabilities and limitations of language models. To address this challenge, we introduce the Beyond the Imitation Game benchmark (BIG-bench). BIG-bench currently consists of 204 tasks, contributed by 450 authors across 132 institutions. Task topics are diverse, drawing problems from linguistics, childhood development, math, common-sense reasoning, biology, physics, social bias, software development, and beyond. BIG-bench focuses on tasks that are believed to be beyond the capabilities of current language models. We evaluate the behavior of OpenAI's GPT models, Google-internal dense transformer architectures, and Switch-style sparse transformers on BIG-bench, across model sizes spanning millions to hundreds of billions of parameters. In addition, a team of human expert raters performed all tasks in order to provide a strong baseline. Findings include: model performance and calibration both improve with scale, but are poor in absolute terms (and when compared with rater performance); performance is remarkably similar across model classes, though with benefits from sparsity; tasks that improve gradually and predictably commonly involve a large knowledge or memorization component, whereas tasks that exhibit \"breakthrough\" behavior at a critical scale often involve multiple steps or components, or brittle metrics; social bias typically increases with scale in settings with ambiguous context, but this can be improved with prompting.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/BigBench.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/BigBench.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          199,
          10,
          90,
          143,
          80,
          146,
          91,
          53,
          95,
          154,
          97,
          155,
          98,
          156,
          99,
          157,
          100,
          158,
          122,
          123,
          38,
          44,
          104,
          105,
          103,
          107,
          108,
          109,
          118,
          119,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136
        ],
        "absent_mitigations": [
          186,
          1,
          106,
          198,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          89,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          62,
          11,
          164,
          69,
          142,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          13,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          201,
          145,
          18,
          79,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          149,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          48,
          153,
          182,
          135,
          137,
          30,
          49,
          154,
          31,
          32,
          50,
          96,
          54,
          202,
          55,
          138,
          58,
          60,
          59,
          139,
          61,
          33,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          74,
          43,
          45,
          161,
          52,
          67,
          112,
          113,
          114,
          110,
          115,
          116,
          117,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2206.04615",
          "role": "definitive-reference",
          "key": "big-bench/paper-arxiv-2206.04615.pdf",
          "content_url": "https://arxiv.org/pdf/2206.04615",
          "format": "pdf",
          "sha256": "1a7f7766060a3d152e18d9a2c2844466e877004c97f2c89220bb89bf35b0fa35",
          "size_bytes": 3874852,
          "fetched_at": "2026-11-03T23:21:38Z",
          "uploaded_sha256": "1a7f7766060a3d152e18d9a2c2844466e877004c97f2c89220bb89bf35b0fa35"
        }
      ]
    },
    {
      "tag": "big-bench-extra-hard",
      "name": "BigBenchExtraHard",
      "title": "BIG-Bench Extra Hard",
      "description": "Benchmark indicates how well a system can perform 23 novel tasks that probe advanced reasoning capabilities, each designed to be significantly more challenging than its counterpart in the original BIG-Bench (BBH) suite.",
      "reference": "https://arxiv.org/abs/2502.19187",
      "reference_note": null,
      "summary": "Large language models (LLMs) are increasingly deployed in everyday applications, demanding robust general reasoning capabilities and diverse reasoning skillset. However, current LLM reasoning benchmarks predominantly focus on mathematical and coding abilities, leaving a gap in evaluating broader reasoning proficiencies. One particular exception is the BIG-Bench dataset, which has served as a crucial benchmark for evaluating the general reasoning capabilities of LLMs, thanks to its diverse set of challenging tasks that allowed for a comprehensive assessment of general reasoning across various skills within a unified framework. However, recent advances in LLMs have led to saturation on BIG-Bench, and its harder version BIG-Bench Hard (BBH). State-of-the-art models achieve near-perfect scores on many tasks in BBH, thus diminishing its utility. To address this limitation, we introduce BIG-Bench Extra Hard (BBEH), a new benchmark designed to push the boundaries of LLM reasoning evaluation. BBEH replaces each task in BBH with a novel task that probes a similar reasoning capability but exhibits significantly increased difficulty. We evaluate various models on BBEH and observe a (harmonic) average accuracy of 9.8\\% for the best general-purpose model and 44.8\\% for the best reasoning-specialized model, indicating substantial room for improvement and highlighting the ongoing challenge of achieving robust general reasoning in LLMs. We release BBEH publicly at: this [https URL](https://github.com/google-deepmind/bbeh).",
      "related_links": [
        "https://github.com/google-deepmind/bbeh"
      ],
      "benchrisk": {
        "source_file": "data/scores/BigBenchExtraHard.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/BigBenchExtraHard.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          10,
          142,
          90,
          143,
          80,
          146,
          91,
          53,
          93,
          153,
          95,
          154,
          97,
          155,
          98,
          156,
          99,
          157,
          100,
          158,
          122,
          123,
          38,
          44,
          161,
          104,
          105,
          103,
          107,
          108,
          109,
          118,
          119,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136
        ],
        "absent_mitigations": [
          186,
          1,
          106,
          198,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          89,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          62,
          11,
          164,
          69,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          13,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          201,
          145,
          18,
          79,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          149,
          150,
          151,
          73,
          28,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          135,
          137,
          48,
          182,
          30,
          49,
          154,
          31,
          32,
          50,
          96,
          54,
          202,
          55,
          138,
          58,
          60,
          59,
          139,
          61,
          33,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          74,
          43,
          45,
          52,
          67,
          112,
          113,
          114,
          110,
          115,
          116,
          117,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2502.19187",
          "role": "definitive-reference",
          "key": "big-bench-extra-hard/paper-arxiv-2502.19187.pdf",
          "content_url": "https://arxiv.org/pdf/2502.19187",
          "format": "pdf",
          "sha256": "9738f4c10d4648d1e0c53191efcaa671950815fe410b11fcc89342c17ac1c3da",
          "size_bytes": 789875,
          "fetched_at": "2026-11-03T23:21:41Z",
          "uploaded_sha256": "9738f4c10d4648d1e0c53191efcaa671950815fe410b11fcc89342c17ac1c3da"
        },
        {
          "doc_type": "repository",
          "url": "https://github.com/google-deepmind/bbeh",
          "role": "related",
          "key": "big-bench-extra-hard/repository-github-google-deepmind-bbeh-readme.md",
          "content_url": "https://api.github.com/repos/google-deepmind/bbeh/readme",
          "format": "md",
          "sha256": "86d16ad213c64a4777a5368ae1a76664d1152b9316c04dede597b8e8cbedf56c",
          "size_bytes": 7541,
          "fetched_at": "2026-11-03T23:21:41Z",
          "uploaded_sha256": "86d16ad213c64a4777a5368ae1a76664d1152b9316c04dede597b8e8cbedf56c"
        }
      ]
    },
    {
      "tag": "big-bench-hard",
      "name": "BigBenchHard",
      "title": "Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them",
      "description": "Benchmark indicates how well a system can perform 23 challenging tasks from the BIG-Bench Hard (BBH) suite, assessing its ability to solve complex reasoning problems",
      "reference": "https://arxiv.org/abs/2210.09261",
      "reference_note": null,
      "summary": "BIG-Bench (Srivastava et al., 2022) is a diverse evaluation suite that focuses on tasks believed to be beyond the capabilities of current language models. Language models have already made good progress on this benchmark, with the best model in the BIG-Bench paper outperforming average reported human-rater results on 65% of the BIG-Bench tasks via few-shot prompting. But on what tasks do language models fall short of average human-rater performance, and are those tasks actually unsolvable by current language models?\nIn this work, we focus on a suite of 23 challenging BIG-Bench tasks which we call BIG-Bench Hard (BBH). These are the task for which prior language model evaluations did not outperform the average human-rater. We find that applying chain-of-thought (CoT) prompting to BBH tasks enables PaLM to surpass the average human-rater performance on 10 of the 23 tasks, and Codex (code-davinci-002) to surpass the average human-rater performance on 17 of the 23 tasks. Since many tasks in BBH require multi-step reasoning, few-shot prompting without CoT, as done in the BIG-Bench evaluations (Srivastava et al., 2022), substantially underestimates the best performance and capabilities of language models, which is better captured via CoT prompting. As further analysis, we explore the interaction between CoT and model scale on BBH, finding that CoT enables emergent task performance on several BBH tasks with otherwise flat scaling curves.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/BigBenchHard.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/BigBenchHard.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          199,
          10,
          90,
          143,
          80,
          146,
          91,
          53,
          93,
          153,
          95,
          154,
          97,
          155,
          98,
          156,
          99,
          157,
          100,
          158,
          51,
          122,
          123,
          38,
          44,
          161,
          104,
          105,
          103,
          107,
          108,
          109,
          118,
          119,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136
        ],
        "absent_mitigations": [
          186,
          1,
          106,
          198,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          89,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          62,
          11,
          164,
          69,
          142,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          13,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          201,
          145,
          18,
          79,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          149,
          150,
          151,
          73,
          28,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          135,
          137,
          48,
          182,
          30,
          49,
          154,
          31,
          32,
          50,
          96,
          54,
          202,
          55,
          138,
          58,
          60,
          59,
          139,
          61,
          33,
          34,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          74,
          43,
          45,
          52,
          67,
          112,
          113,
          114,
          110,
          115,
          116,
          117,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2210.09261",
          "role": "definitive-reference",
          "key": "big-bench-hard/paper-arxiv-2210.09261.html",
          "content_url": "https://arxiv.org/html/2210.09261",
          "format": "html",
          "sha256": "38e08609cad1ddc75188855406c557d8e75638c9f13a03d1540adca7557d159a",
          "size_bytes": 1327362,
          "fetched_at": "2026-11-03T23:21:47Z",
          "uploaded_sha256": "38e08609cad1ddc75188855406c557d8e75638c9f13a03d1540adca7557d159a"
        }
      ]
    },
    {
      "tag": "bold",
      "name": "BOLDBias",
      "title": "BOLD: Dataset and Metrics for Measuring Biases in Open-Ended Language Generation",
      "description": "Benchmark indicates whether an open-ended text generation model maintains fairness and avoids social biases across domains such as profession, gender, race, religion, and ideology when prompted with real-world English contexts, helping users evaluate the model’s equity and ethical behavior.",
      "reference": "https://arxiv.org/abs/2101.11718",
      "reference_note": null,
      "summary": "Recent advances in deep learning techniques have enabled machines to generate cohesive open-ended text when prompted with a sequence of words as context. While these models now empower many downstream applications from conversation bots to automatic storytelling, they have been shown to generate texts that exhibit social biases. To systematically study and benchmark social biases in open-ended language generation, we introduce the Bias in Open-Ended Language Generation Dataset (BOLD), a large-scale dataset that consists of 23,679 English text generation prompts for bias benchmarking across five domains: profession, gender, race, religion, and political ideology. We also propose new automated metrics for toxicity, psycholinguistic norms, and text gender polarity to measure social biases in open-ended text generation from multiple angles. An examination of text generated from three popular language models reveals that the majority of these models exhibit a larger social bias than human-written Wikipedia text across all domains. With these results we highlight the need to benchmark biases in open-ended language generation and caution users of language generation models on downstream tasks to be cognizant of these embedded prejudices.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/BOLDBias.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/BOLDBias.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          106,
          64,
          89,
          70,
          6,
          10,
          62,
          69,
          84,
          86,
          85,
          87,
          145,
          18,
          22,
          48,
          137,
          30,
          135,
          51,
          159,
          122,
          123,
          82,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126
        ],
        "absent_mitigations": [
          186,
          198,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          199,
          140,
          5,
          8,
          101,
          7,
          9,
          141,
          102,
          11,
          164,
          142,
          178,
          12,
          165,
          183,
          184,
          13,
          90,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          201,
          79,
          146,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          153,
          182,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          98,
          156,
          55,
          99,
          58,
          138,
          60,
          157,
          100,
          59,
          139,
          61,
          158,
          33,
          34,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          38,
          88,
          66,
          83,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          104,
          105,
          103,
          107,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          133,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2101.11718",
          "role": "definitive-reference",
          "key": "bold/paper-arxiv-2101.11718.pdf",
          "content_url": "https://arxiv.org/pdf/2101.11718",
          "format": "pdf",
          "sha256": "6c9b6fe5ac56368101503726afc560dd6ef1f91a1ece4fb618b72fcd859b4d4b",
          "size_bytes": 1623576,
          "fetched_at": "2026-11-03T23:21:50Z",
          "uploaded_sha256": "6c9b6fe5ac56368101503726afc560dd6ef1f91a1ece4fb618b72fcd859b4d4b"
        }
      ]
    },
    {
      "tag": "decodingtrust-privacy",
      "name": "DecodingTrustPrivacy",
      "title": "DecodingTrustPrivacy",
      "description": "Benchmark evaluates language models (specifically GPT models) privacy by looking into if they leas sesitive training data, reveal private user inputs and understand non pii related private information in conversations.",
      "reference": "https://decodingtrust.github.io",
      "reference_note": null,
      "summary": "DecodingTrust aims at providing a thorough assessment of trustworthiness in GPT models.\nThis research endeavor is designed to help researchers and practitioners better understand the capabilities, limitations, and potential risks involved in deploying these state-of-the-art Large Language Models (LLMs).\nThis project is organized around the following eight primary perspectives of trustworthiness, including:\n* Toxicity\n* Stereotype and bias\n* Adversarial robustness\n* Out-of-Distribution Robustness\n* Privacy\n* Robustness to Adversarial Demonstrations\n* Machine Ethics\n* Fairness",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/DecodingTrustPrivacy.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/DecodingTrustPrivacy.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": true,
        "adopted_mitigations": [
          1,
          64,
          89,
          10,
          69,
          12,
          13,
          14,
          92,
          53,
          94,
          153,
          154,
          157,
          158,
          123,
          104,
          103,
          128,
          129,
          130,
          132,
          124,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          62,
          11,
          164,
          142,
          165,
          178,
          84,
          86,
          183,
          184,
          85,
          87,
          90,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          150,
          151,
          73,
          28,
          93,
          46,
          196,
          47,
          81,
          152,
          29,
          135,
          137,
          48,
          182,
          30,
          49,
          31,
          95,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          98,
          156,
          55,
          138,
          99,
          58,
          60,
          59,
          139,
          100,
          61,
          33,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          122,
          38,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          105,
          107,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          131,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "website",
          "url": "https://decodingtrust.github.io",
          "role": "definitive-reference",
          "key": "decodingtrust-privacy/website-decodingtrust-github-io.html",
          "content_url": "https://decodingtrust.github.io",
          "format": "html",
          "sha256": "ab3758cd0fd211cfd47ecd5beca43fd01b42084bea613f639cd717902fa8bc10",
          "size_bytes": 12500,
          "fetched_at": "2026-11-03T23:21:51Z",
          "uploaded_sha256": "ab3758cd0fd211cfd47ecd5beca43fd01b42084bea613f639cd717902fa8bc10"
        }
      ]
    },
    {
      "tag": "decodingtrust-toxicity",
      "name": "DecodingTrustToxicity",
      "title": "DecodingTrustToxicity",
      "description": "Benchmark evaluates the toxicity of sentence completions.",
      "reference": "https://decodingtrust.github.io",
      "reference_note": null,
      "summary": "DecodingTrust aims at providing a thorough assessment of trustworthiness in GPT models.\nThis research endeavor is designed to help researchers and practitioners better understand the capabilities, limitations, and potential risks involved in deploying these state-of-the-art Large Language Models (LLMs).\nThis project is organized around the following eight primary perspectives of trustworthiness, including:\n* Toxicity\n* Stereotype and bias\n* Adversarial robustness\n* Out-of-Distribution Robustness\n* Privacy\n* Robustness to Adversarial Demonstrations\n* Machine Ethics\n* Fairness",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/DecodingTrustToxicity.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/DecodingTrustToxicity.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          106,
          68,
          3,
          10,
          69,
          142,
          12,
          86,
          13,
          87,
          90,
          71,
          80,
          146,
          185,
          94,
          182,
          202,
          138,
          139,
          33,
          122,
          123,
          38,
          127,
          104,
          105,
          103,
          107,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          64,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          4,
          163,
          70,
          199,
          89,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          62,
          11,
          164,
          178,
          84,
          165,
          183,
          184,
          85,
          14,
          65,
          144,
          17,
          193,
          143,
          201,
          145,
          18,
          79,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          196,
          47,
          81,
          152,
          29,
          135,
          137,
          48,
          153,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          98,
          156,
          55,
          58,
          99,
          60,
          157,
          59,
          100,
          61,
          158,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          88,
          66,
          83,
          82,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "website",
          "url": "https://decodingtrust.github.io",
          "role": "definitive-reference",
          "key": "decodingtrust-toxicity/website-decodingtrust-github-io.html",
          "content_url": "https://decodingtrust.github.io",
          "format": "html",
          "sha256": "ab3758cd0fd211cfd47ecd5beca43fd01b42084bea613f639cd717902fa8bc10",
          "size_bytes": 12500,
          "fetched_at": "2026-11-03T23:21:51Z",
          "uploaded_sha256": "ab3758cd0fd211cfd47ecd5beca43fd01b42084bea613f639cd717902fa8bc10"
        }
      ]
    },
    {
      "tag": "ethics",
      "name": "Ethics",
      "title": "Aligning AI With Shared Human Values",
      "description": "Benchmark measures model capacity for reflecting human notions of justice, well-being, duties, virtues, and commonsense morality",
      "reference": "https://arxiv.org/abs/2008.02275",
      "reference_note": null,
      "summary": "We show how to assess a language model's knowledge of basic concepts of morality. We introduce the ETHICS dataset, a new benchmark that spans concepts in justice, well-being, duties, virtues, and commonsense morality. Models predict widespread moral judgments about diverse text scenarios. This requires connecting physical and social world knowledge to value judgements, a capability that may enable us to steer chatbot outputs or eventually regularize open-ended reinforcement learning agents. With the ETHICS dataset, we find that current language models have a promising but incomplete ability to predict basic human ethical judgements. Our work shows that progress can be made on machine ethics today, and it provides a steppingstone toward AI that is aligned with human values.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/Ethics.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/Ethics.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          64,
          10,
          90,
          143,
          21,
          22,
          148,
          93,
          81,
          153,
          154,
          96,
          155,
          156,
          157,
          158,
          160,
          88,
          104,
          105,
          103,
          109,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          89,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          62,
          11,
          164,
          69,
          142,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          13,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          80,
          145,
          18,
          201,
          79,
          146,
          19,
          180,
          179,
          147,
          166,
          194,
          134,
          195,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          46,
          196,
          47,
          152,
          29,
          94,
          135,
          137,
          48,
          182,
          30,
          49,
          31,
          95,
          32,
          50,
          97,
          54,
          202,
          98,
          55,
          138,
          99,
          58,
          60,
          59,
          139,
          100,
          61,
          33,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          37,
          75,
          197,
          122,
          123,
          38,
          66,
          83,
          82,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          107,
          108,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          126,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2008.02275",
          "role": "definitive-reference",
          "key": "ethics/paper-arxiv-2008.02275.pdf",
          "content_url": "https://arxiv.org/pdf/2008.02275",
          "format": "pdf",
          "sha256": "93a7fdb244b18fc0592268dc1055133a1d67bc85b8cfac347ae8707390c23936",
          "size_bytes": 2856411,
          "fetched_at": "2026-11-03T23:21:54Z",
          "uploaded_sha256": "93a7fdb244b18fc0592268dc1055133a1d67bc85b8cfac347ae8707390c23936"
        }
      ]
    },
    {
      "tag": "gpqa",
      "name": "GPQA",
      "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark",
      "description": "Benchmark indicates how well a system can answer graduate-level multiple-choice questions in biology, chemistry, and physics, with performance comparable to skilled non-experts even when provided unrestricted internet access.",
      "reference": "https://arxiv.org/abs/2311.12022",
      "reference_note": null,
      "summary": "We present GPQA, a challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. We ensure that the questions are high-quality and extremely difficult: experts who have or are pursuing PhDs in the corresponding domains reach 65% accuracy (74% when discounting clear mistakes the experts identified in retrospect), while highly skilled non-expert validators only reach 34% accuracy, despite spending on average over 30 minutes with unrestricted access to the web (i.e., the questions are \"Google-proof\"). The questions are also difficult for state-of-the-art AI systems, with our strongest GPT-4 based baseline achieving 39% accuracy. If we are to use future AI systems to help us answer very hard questions, for example, when developing new scientific knowledge, we need to develop scalable oversight methods that enable humans to supervise their outputs, which may be difficult even if the supervisors are themselves skilled and knowledgeable. The difficulty of GPQA both for skilled non-experts and frontier AI systems should enable realistic scalable oversight experiments, which we hope can help devise ways for human experts to reliably get truthful information from AI systems that surpass human capabilities.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/GPQA.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/GPQA.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          106,
          68,
          89,
          140,
          101,
          141,
          102,
          142,
          12,
          13,
          90,
          143,
          146,
          148,
          26,
          152,
          153,
          154,
          96,
          97,
          155,
          157,
          158,
          159,
          160,
          122,
          38,
          88,
          82,
          52,
          104,
          108,
          128,
          129,
          130,
          131,
          132,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          64,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          5,
          6,
          8,
          7,
          9,
          10,
          62,
          11,
          164,
          69,
          165,
          178,
          84,
          86,
          183,
          184,
          85,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          80,
          145,
          18,
          201,
          79,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          196,
          47,
          81,
          29,
          94,
          135,
          137,
          48,
          182,
          30,
          49,
          31,
          95,
          32,
          50,
          54,
          202,
          98,
          156,
          55,
          138,
          99,
          58,
          60,
          59,
          139,
          100,
          61,
          33,
          34,
          51,
          200,
          35,
          56,
          36,
          37,
          75,
          197,
          123,
          66,
          83,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          67,
          105,
          103,
          107,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          124,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2311.12022",
          "role": "definitive-reference",
          "key": "gpqa/paper-arxiv-2311.12022.pdf",
          "content_url": "https://arxiv.org/pdf/2311.12022",
          "format": "pdf",
          "sha256": "58a85a21b653a01d5794f84952936f58e6a5218d781588f44ae183bb749bf023",
          "size_bytes": 617128,
          "fetched_at": "2026-11-03T23:21:57Z",
          "uploaded_sha256": "58a85a21b653a01d5794f84952936f58e6a5218d781588f44ae183bb749bf023"
        }
      ]
    },
    {
      "tag": "gsm8k",
      "name": "GSM8K",
      "title": "Training Verifiers to Solve Math Word Problems",
      "description": "Benchmark measures model capability in solving diverse grade school math word problems.",
      "reference": "https://arxiv.org/abs/2110.14168",
      "reference_note": null,
      "summary": "State-of-the-art language models can match human performance on many tasks, but they still struggle to robustly perform multi-step mathematical reasoning. To diagnose the failures of current models and support research, we introduce GSM8K, a dataset of 8.5K high quality linguistically diverse grade school math word problems. We find that even the largest transformer models fail to achieve high test performance, despite the conceptual simplicity of this problem distribution. To increase performance, we propose training verifiers to judge the correctness of model completions. At test time, we generate many candidate solutions and select the one ranked highest by the verifier. We demonstrate that verification significantly improves performance on GSM8K, and we provide strong empirical evidence that verification scales more effectively with increased data than a finetuning baseline.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/GSM8K.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/GSM8K.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          89,
          70,
          140,
          5,
          102,
          10,
          90,
          145,
          19,
          92,
          94,
          153,
          97,
          98,
          99,
          157,
          123,
          88,
          82,
          127,
          103,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          199,
          6,
          8,
          101,
          7,
          9,
          141,
          62,
          11,
          164,
          69,
          142,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          13,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          201,
          18,
          79,
          146,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          196,
          47,
          81,
          152,
          29,
          135,
          137,
          48,
          182,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          155,
          54,
          202,
          156,
          55,
          138,
          58,
          60,
          100,
          59,
          139,
          61,
          158,
          33,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          122,
          38,
          66,
          83,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          104,
          105,
          107,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          133,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2110.14168",
          "role": "definitive-reference",
          "key": "gsm8k/paper-arxiv-2110.14168.pdf",
          "content_url": "https://arxiv.org/pdf/2110.14168",
          "format": "pdf",
          "sha256": "a52417d8fcd006de882e3b404850e2cc4246815828897c18c6ce6c7a448985a4",
          "size_bytes": 2935719,
          "fetched_at": "2026-11-03T23:22:00Z",
          "uploaded_sha256": "a52417d8fcd006de882e3b404850e2cc4246815828897c18c6ce6c7a448985a4"
        }
      ]
    },
    {
      "tag": "hellaswag",
      "name": "HellaSwag",
      "title": "HellaSwag: Can a Machine Really Finish Your Sentence?",
      "description": "Benchmark capacity to reason about physical and social concepts by prompting for sentence endings from challenging, adversarially filtered alternatives, providing users with a measure of the scope of model capacity.",
      "reference": "https://arxiv.org/abs/1905.07830",
      "reference_note": null,
      "summary": "Recent work by Zellers et al. (2018) introduced a new task of commonsense natural language inference: given an event description such as \"A woman sits at a piano,\" a machine must select the most likely followup: \"She sets her fingers on the keys.\" With the introduction of BERT, near human-level performance was reached. Does this mean that machines can perform human level commonsense inference?\nIn this paper, we show that commonsense inference still proves difficult for even state-of-the-art models, by presenting HellaSwag, a new challenge dataset. Though its questions are trivial for humans (>95% accuracy), state-of-the-art models struggle (\\<48%). We achieve this via Adversarial Filtering (AF), a data collection paradigm wherein a series of discriminators iteratively select an adversarial set of machine-generated wrong answers. AF proves to be surprisingly robust. The key insight is to scale up the length and complexity of the dataset examples towards a critical 'Goldilocks' zone wherein generated text is ridiculous to humans, yet often misclassified by state-of-the-art models.\nOur construction of HellaSwag, and its resulting difficulty, sheds light on the inner workings of deep pretrained models. More broadly, it suggests a new path forward for NLP research, in which benchmarks co-evolve with the evolving state-of-the-art in an adversarial way, so as to present ever-harder challenges.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/HellaSwag.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/HellaSwag.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          64,
          68,
          89,
          70,
          6,
          62,
          69,
          142,
          84,
          85,
          87,
          14,
          71,
          80,
          48,
          153,
          30,
          135,
          154,
          96,
          97,
          98,
          99,
          100,
          33,
          122,
          123,
          82,
          108,
          109,
          128,
          129,
          130,
          132,
          124,
          125,
          126
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          199,
          140,
          5,
          8,
          101,
          7,
          9,
          141,
          102,
          10,
          11,
          164,
          165,
          12,
          178,
          86,
          183,
          184,
          13,
          90,
          65,
          144,
          17,
          193,
          143,
          201,
          145,
          18,
          79,
          146,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          137,
          182,
          49,
          31,
          95,
          32,
          50,
          155,
          54,
          202,
          156,
          55,
          138,
          58,
          157,
          60,
          59,
          139,
          61,
          158,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          38,
          88,
          66,
          83,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          104,
          105,
          103,
          107,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          131,
          133,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/1905.07830",
          "role": "definitive-reference",
          "key": "hellaswag/paper-arxiv-1905.07830.pdf",
          "content_url": "https://arxiv.org/pdf/1905.07830",
          "format": "pdf",
          "sha256": "9a042fdcc4a22848cffde37e9c5257e423443980172bab7367dffb4e779aac37",
          "size_bytes": 1192712,
          "fetched_at": "2026-11-03T23:22:03Z",
          "uploaded_sha256": "9a042fdcc4a22848cffde37e9c5257e423443980172bab7367dffb4e779aac37"
        }
      ]
    },
    {
      "tag": "humaneval",
      "name": "HumanEval",
      "title": "Evaluating Large Language Models Trained on Code",
      "description": "Benchmark indicates how well the system under test can support a human writing python code by specifying docstrings whose function bodies are completed by the system.",
      "reference": "https://arxiv.org/abs/2107.03374",
      "reference_note": null,
      "summary": "We introduce Codex, a GPT language model fine-tuned on publicly available code from GitHub, and study its Python code-writing capabilities. A distinct production version of Codex powers GitHub Copilot. On HumanEval, a new evaluation set we release to measure functional correctness for synthesizing programs from docstrings, our model solves 28.8% of the problems, while GPT-3 solves 0% and GPT-J solves 11.4%. Furthermore, we find that repeated sampling from the model is a surprisingly effective strategy for producing working solutions to difficult prompts. Using this method, we solve 70.2% of our problems with 100 samples per problem. Careful investigation of our model reveals its limitations, including difficulty with docstrings describing long chains of operations and with binding operations to variables. Finally, we discuss the potential broader impacts of deploying powerful code generation technologies, covering safety, security, and economics.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/HumanEval.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/HumanEval.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          68,
          89,
          70,
          140,
          5,
          101,
          141,
          102,
          10,
          12,
          165,
          13,
          90,
          19,
          92,
          93,
          94,
          153,
          154,
          95,
          96,
          97,
          98,
          138,
          99,
          139,
          100,
          33,
          160,
          88,
          82,
          161,
          104,
          105,
          103,
          107,
          128,
          129,
          131,
          132,
          124,
          125,
          126
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          199,
          6,
          8,
          7,
          9,
          62,
          11,
          164,
          69,
          142,
          178,
          84,
          86,
          183,
          184,
          85,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          46,
          196,
          47,
          81,
          152,
          29,
          135,
          137,
          48,
          182,
          30,
          49,
          31,
          32,
          50,
          155,
          54,
          202,
          156,
          55,
          58,
          60,
          157,
          59,
          61,
          158,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          37,
          75,
          197,
          122,
          123,
          38,
          66,
          83,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          52,
          67,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          130,
          133,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2107.03374",
          "role": "definitive-reference",
          "key": "humaneval/paper-arxiv-2107.03374.pdf",
          "content_url": "https://arxiv.org/pdf/2107.03374",
          "format": "pdf",
          "sha256": "ebae72ea0e8a5eb2ecbccdb985aec6cc1254a7c4d29e6d4de7866db1e66c4855",
          "size_bytes": 1885177,
          "fetched_at": "2026-11-03T23:22:06Z",
          "uploaded_sha256": "ebae72ea0e8a5eb2ecbccdb985aec6cc1254a7c4d29e6d4de7866db1e66c4855"
        }
      ]
    },
    {
      "tag": "humanitys-last-exam",
      "name": "HumanitysLastExam",
      "title": "Humanity's Last Exam",
      "description": "Benchmark indicates whether a model can answer expert-level academic questions across disciplines with precision and understanding, serving as a key signal of the model’s reliability for users who depend on it for rigorous, high-stakes reasoning.",
      "reference": "https://agi.safe.ai/",
      "reference_note": null,
      "summary": "Benchmarks are important tools for tracking the rapid advancements in large language model (LLM) capabilities. However, benchmarks are not keeping pace in difficulty: LLMs now achieve over 90\\% accuracy on popular benchmarks like MMLU, limiting informed measurement of state-of-the-art LLM capabilities. In response, we introduce Humanity's Last Exam (HLE), a multi-modal benchmark at the frontier of human knowledge, designed to be the final closed-ended academic benchmark of its kind with broad subject coverage. HLE consists of 2,500 questions across dozens of subjects, including mathematics, humanities, and the natural sciences. HLE is developed globally by subject-matter experts and consists of multiple-choice and short-answer questions suitable for automated grading. Each question has a known solution that is unambiguous and easily verifiable, but cannot be quickly answered via internet retrieval. State-of-the-art LLMs demonstrate low accuracy and calibration on HLE, highlighting a significant gap between current LLM capabilities and the expert human frontier on closed-ended academic questions. To inform research and policymaking upon a clear understanding of model capabilities, we publicly release HLE at [this https URL.](https://lastexam.ai/)",
      "related_links": [
        "https://lastexam.ai/"
      ],
      "benchrisk": {
        "source_file": "data/scores/HumanitysLastExam.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/HumanitysLastExam.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          162,
          89,
          163,
          140,
          5,
          101,
          102,
          164,
          142,
          165,
          86,
          87,
          90,
          143,
          146,
          166,
          92,
          93,
          33,
          159,
          160,
          122,
          123,
          38,
          88,
          82,
          74,
          104,
          107,
          108,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          70,
          199,
          6,
          8,
          7,
          9,
          141,
          10,
          62,
          11,
          69,
          183,
          178,
          12,
          84,
          184,
          13,
          85,
          14,
          65,
          71,
          144,
          17,
          193,
          80,
          145,
          18,
          201,
          79,
          19,
          180,
          179,
          147,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          48,
          153,
          182,
          135,
          137,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          98,
          156,
          55,
          99,
          58,
          138,
          60,
          157,
          100,
          59,
          139,
          61,
          158,
          34,
          51,
          200,
          35,
          56,
          36,
          37,
          75,
          197,
          66,
          83,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          43,
          45,
          161,
          52,
          67,
          105,
          103,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "website",
          "url": "https://agi.safe.ai/",
          "role": "definitive-reference",
          "key": "humanitys-last-exam/website-agi-safe-ai.html",
          "content_url": "https://agi.safe.ai/",
          "format": "html",
          "sha256": "2a84b6210c1ece5f6cb1f3daa91085fa51165211f08679d2a3f7e3ab0f4cbf18",
          "size_bytes": 469505,
          "fetched_at": "2026-11-03T23:22:06Z",
          "uploaded_sha256": "2a84b6210c1ece5f6cb1f3daa91085fa51165211f08679d2a3f7e3ab0f4cbf18"
        }
      ]
    },
    {
      "tag": "machiavelli",
      "name": "Machiavelli",
      "title": "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark",
      "description": "Benchmark measures the trade-off between reward optimization and ethical behavior in AI agents by quantifying their propensity for harmful actions—such as power-seeking, deception, and causing disutility—within interactive social scenarios.",
      "reference": "https://aypan17.github.io/machiavelli/",
      "reference_note": null,
      "summary": "Artificial agents have traditionally been trained to maximize reward, which may incentivize power-seeking and deception, analogous to how next-token prediction in language models (LMs) may incentivize toxicity. So do agents naturally learn to be Machiavellian? And how do we measure these behaviors in general-purpose models such as GPT-4? Towards answering these questions, we introduce MACHIAVELLI, a benchmark of 134 Choose-Your-Own-Adventure games containing over half a million rich, diverse scenarios that center on social decision-making. Scenario labeling is automated with LMs, which are more performant than human annotators. We mathematize dozens of harmful behaviors and use our annotations to evaluate agents' tendencies to be power-seeking, cause disutility, and commit ethical violations. We observe some tension between maximizing reward and behaving ethically. To improve this trade-off, we investigate LM-based methods to steer agents' towards less harmful behaviors. Our results show that agents can both act competently and morally, so concrete progress can currently be made in machine ethics--designing agents that are Pareto improvements in both safety and capabilities.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/Machiavelli.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/Machiavelli.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": true,
        "adopted_mitigations": [
          1,
          183,
          184,
          90,
          179,
          185,
          135,
          55,
          138,
          139,
          33,
          123,
          104,
          105,
          103,
          107,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          89,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          10,
          62,
          11,
          164,
          69,
          142,
          165,
          12,
          178,
          84,
          86,
          13,
          85,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          19,
          147,
          180,
          21,
          166,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          137,
          48,
          182,
          153,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          156,
          98,
          58,
          99,
          60,
          157,
          59,
          100,
          61,
          158,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          122,
          38,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "website",
          "url": "https://aypan17.github.io/machiavelli/",
          "role": "definitive-reference",
          "key": "machiavelli/website-aypan17-github-io.html",
          "content_url": "https://aypan17.github.io/machiavelli/",
          "format": "html",
          "sha256": "901a31d2ac676bbf127625e2c61988ea242343813b93447198fb1922146ce043",
          "size_bytes": 14414,
          "fetched_at": "2026-11-03T23:22:06Z",
          "uploaded_sha256": "901a31d2ac676bbf127625e2c61988ea242343813b93447198fb1922146ce043"
        }
      ]
    },
    {
      "tag": "mmlu",
      "name": "MMLU",
      "title": "Measuring Massive Multitask Language Understanding",
      "description": "Benchmark evaluates a language model’s multitask academic and professional knowledge by measuring its accuracy on multiple-choice questions across 57 diverse subjects.",
      "reference": "https://arxiv.org/abs/2009.03300",
      "reference_note": null,
      "summary": "We propose a new test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more. To attain high accuracy on this test, models must possess extensive world knowledge and problem solving ability. We find that while most recent models have near random-chance accuracy, the very largest GPT-3 model improves over random chance by almost 20 percentage points on average. However, on every one of the 57 tasks, the best models still need substantial improvements before they can reach expert-level accuracy. Models also have lopsided performance and frequently do not know when they are wrong. Worse, they still have near-random accuracy on some socially important subjects such as morality and law. By comprehensively evaluating the breadth and depth of a model's academic and professional understanding, our test can be used to analyze models across many tasks and to identify important shortcomings.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/MMLU.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/MMLU.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          68,
          89,
          2,
          10,
          11,
          142,
          84,
          85,
          90,
          79,
          146,
          21,
          153,
          154,
          97,
          155,
          54,
          156,
          157,
          59,
          158,
          38,
          52,
          105,
          103,
          108,
          109,
          128,
          129,
          130,
          132,
          124,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          187,
          188,
          189,
          190,
          191,
          192,
          3,
          4,
          163,
          70,
          199,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          62,
          164,
          69,
          165,
          12,
          178,
          86,
          183,
          184,
          13,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          201,
          145,
          18,
          19,
          180,
          179,
          147,
          166,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          135,
          137,
          48,
          182,
          30,
          49,
          31,
          95,
          32,
          50,
          96,
          202,
          98,
          55,
          138,
          99,
          58,
          60,
          139,
          100,
          61,
          33,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          122,
          123,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          67,
          104,
          107,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          131,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2009.03300",
          "role": "definitive-reference",
          "key": "mmlu/paper-arxiv-2009.03300.pdf",
          "content_url": "https://arxiv.org/pdf/2009.03300",
          "format": "pdf",
          "sha256": "d3a7b0f8589f5365ebfd34a3876a9a8b3fde83e0adb9ed19ccf8f0034bb48fc1",
          "size_bytes": 2074939,
          "fetched_at": "2026-11-03T23:22:09Z",
          "uploaded_sha256": "d3a7b0f8589f5365ebfd34a3876a9a8b3fde83e0adb9ed19ccf8f0034bb48fc1"
        }
      ]
    },
    {
      "tag": "realtoxicityprompts",
      "name": "RealToxicityPrompts",
      "title": "RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models",
      "description": "Benchmark measures a language model's propensity for toxic degeneration, quantifying how even seemingly innocuous, naturally-occurring prompts can trigger the generation of toxic text.",
      "reference": "https://toxicdegeneration.allenai.org/",
      "reference_note": "BenchRisk cites https://toxicdegeneration.allenai.org/, which did not respond (connection timeout) when the index was seeded on 2026-11-03. The paper that site presents, RealToxicityPrompts (Gehman et al., 2020), is indexed instead.",
      "summary": "Pretrained neural language models (LMs) are prone to generating racist, sexist, or otherwise toxic language which hinders their safe deployment. We investigate the extent to which pretrained LMs can be prompted to generate toxic language, and the effectiveness of controllable text generation algorithms at preventing such toxic degeneration. We create and release RealToxicityPrompts, a dataset of 100K naturally occurring, sentence-level prompts derived from a large corpus of English web text, paired with toxicity scores from a widely-used toxicity classifier. Using RealToxicityPrompts, we find that pretrained LMs can degenerate into toxic text even from seemingly innocuous prompts. We empirically assess several controllable generation methods, and find that while data- or compute-intensive methods (e.g., adaptive pretraining on non-toxic data) are more effective at steering away from toxicity than simpler solutions (e.g., banning “bad” words), no current method is failsafe against neural toxic degeneration. To pinpoint the potential cause of such persistent toxic degeneration, we analyze two web text corpora used to pretrain several LMs (including GPT-2; Radford et. al, 2019), and find a significant amount of offensive, factually unreliable, and otherwise toxic content. Our work provides a test bed for evaluating toxic generations by LMs and stresses the need for better data selection processes for pretraining.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/RealToxicityPrompts.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/RealToxicityPrompts.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          64,
          3,
          10,
          178,
          13,
          90,
          17,
          180,
          181,
          182,
          123,
          103,
          107,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          4,
          163,
          70,
          199,
          89,
          140,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          102,
          62,
          11,
          164,
          69,
          142,
          165,
          12,
          84,
          86,
          183,
          184,
          85,
          87,
          14,
          65,
          71,
          144,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          19,
          179,
          147,
          21,
          166,
          194,
          134,
          22,
          195,
          148,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          135,
          137,
          48,
          153,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          98,
          156,
          55,
          99,
          58,
          138,
          60,
          157,
          100,
          59,
          139,
          61,
          158,
          33,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          122,
          38,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          104,
          105,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2009.11462",
          "role": "definitive-reference",
          "key": "realtoxicityprompts/paper-arxiv-2009.11462.pdf",
          "content_url": "https://arxiv.org/pdf/2009.11462",
          "format": "pdf",
          "sha256": "5959d04d1cd7a604d3fe59f163c0c38c36439eaa5531127929467829a02cb68a",
          "size_bytes": 1046316,
          "fetched_at": "2026-11-03T23:22:12Z",
          "uploaded_sha256": "5959d04d1cd7a604d3fe59f163c0c38c36439eaa5531127929467829a02cb68a"
        }
      ]
    },
    {
      "tag": "toxigen",
      "name": "Toxigen",
      "title": "ToxiGen: A Large-Scale Machine-Generated Dataset for Adversarial and Implicit Hate Speech Detection",
      "description": "Benchmark indicates whether a system detects implicit hate speech targeting minority groups while avoiding false alarms",
      "reference": "https://arxiv.org/abs/2203.09509",
      "reference_note": null,
      "summary": "Toxic language detection systems often falsely flag text that contains minority group mentions as toxic, as those groups are often the targets of online hate. Such over-reliance on spurious correlations also causes systems to struggle with detecting implicitly toxic language. To help mitigate these issues, we create ToxiGen, a new large-scale and machine-generated dataset of 274k toxic and benign statements about 13 minority groups. We develop a demonstration-based prompting framework and an adversarial classifier-in-the-loop decoding method to generate subtly toxic and benign text with a massive pretrained language model. Controlling machine generation in this way allows ToxiGen to cover implicitly toxic text at a larger scale, and about more demographic groups, than previous resources of human-written text. We conduct a human evaluation on a challenging subset of ToxiGen and find that annotators struggle to distinguish machine-generated text from human-written language. We also find that 94.5% of toxic examples are labeled as hate speech by human annotators. Using three publicly-available datasets, we show that finetuning a toxicity classifier on our data improves its performance on human-written data substantially. We also demonstrate that ToxiGen can be used to fight machine-generated toxicity as finetuning improves the classifier significantly on our evaluation subset. Our code and data can be found at [this https URL.](https://github.com/microsoft/ToxiGen)",
      "related_links": [
        "https://github.com/microsoft/ToxiGen"
      ],
      "benchrisk": {
        "source_file": "data/scores/Toxigen.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/Toxigen.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          106,
          3,
          70,
          140,
          5,
          101,
          141,
          11,
          142,
          12,
          84,
          85,
          13,
          14,
          79,
          19,
          22,
          92,
          94,
          137,
          135,
          55,
          138,
          139,
          33,
          34,
          35,
          122,
          123,
          38,
          88,
          82,
          127,
          104,
          105,
          103,
          107,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136
        ],
        "absent_mitigations": [
          186,
          198,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          163,
          4,
          199,
          89,
          6,
          8,
          7,
          9,
          102,
          10,
          62,
          164,
          69,
          178,
          165,
          86,
          183,
          184,
          87,
          90,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          201,
          145,
          18,
          146,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          196,
          47,
          81,
          152,
          29,
          48,
          153,
          182,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          156,
          98,
          58,
          99,
          60,
          157,
          59,
          100,
          61,
          158,
          51,
          159,
          200,
          56,
          36,
          160,
          197,
          75,
          37,
          66,
          83,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2203.09509",
          "role": "definitive-reference",
          "key": "toxigen/paper-arxiv-2203.09509.pdf",
          "content_url": "https://arxiv.org/pdf/2203.09509",
          "format": "pdf",
          "sha256": "dfcfcc18da305b83db16a33b0c1faed9aee57df06af81ff65c452e90df9378b2",
          "size_bytes": 1510481,
          "fetched_at": "2026-11-03T23:22:15Z",
          "uploaded_sha256": "dfcfcc18da305b83db16a33b0c1faed9aee57df06af81ff65c452e90df9378b2"
        },
        {
          "doc_type": "repository",
          "url": "https://github.com/microsoft/ToxiGen",
          "role": "related",
          "key": "toxigen/repository-github-microsoft-toxigen-readme.md",
          "content_url": "https://api.github.com/repos/microsoft/ToxiGen/readme",
          "format": "md",
          "sha256": "ac4b63d0b74d830750836c2ca755272404dfb3e1704ac2068480ebf8c6a43495",
          "size_bytes": 8957,
          "fetched_at": "2026-11-03T23:22:15Z",
          "uploaded_sha256": "ac4b63d0b74d830750836c2ca755272404dfb3e1704ac2068480ebf8c6a43495"
        }
      ]
    },
    {
      "tag": "truthfulqa",
      "name": "TruthfulQA",
      "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods",
      "description": "Benchmark indicates whether a model can generate factually accurate answers to questions while avoiding common human misconceptions, measuring its susceptibility to imitative falsehoods—critical for users who depend on AI systems for reliable information in high-stakes domains like medicine, law, and science.",
      "reference": "https://arxiv.org/abs/2109.07958",
      "reference_note": null,
      "summary": "We propose a benchmark to measure whether a language model is truthful in generating answers to questions. The benchmark comprises 817 questions that span 38 categories, including health, law, finance and politics. We crafted questions that some humans would answer falsely due to a false belief or misconception. To perform well, models must avoid generating false answers learned from imitating human texts. We tested GPT-3, GPT-Neo/J, GPT-2 and a T5-based model. The best model was truthful on 58% of questions, while human performance was 94%. Models generated many false answers that mimic popular misconceptions and have the potential to deceive humans. The largest models were generally the least truthful. This contrasts with other NLP tasks, where performance improves with model size. However, this result is expected if false answers are learned from the training distribution. We suggest that scaling up models alone is less promising for improving truthfulness than fine-tuning using training objectives other than imitation of text from the web.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/TruthfulQA.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/TruthfulQA.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          5,
          101,
          102,
          10,
          12,
          13,
          90,
          19,
          92,
          94,
          137,
          55,
          138,
          139,
          33,
          88,
          104,
          105,
          103,
          109,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          199,
          89,
          140,
          6,
          8,
          7,
          9,
          141,
          62,
          11,
          164,
          69,
          142,
          165,
          178,
          84,
          86,
          183,
          184,
          85,
          87,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          196,
          47,
          81,
          152,
          29,
          135,
          48,
          182,
          153,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          156,
          98,
          58,
          99,
          60,
          157,
          59,
          100,
          61,
          158,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          122,
          123,
          38,
          66,
          83,
          82,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          107,
          108,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2109.07958",
          "role": "definitive-reference",
          "key": "truthfulqa/paper-arxiv-2109.07958.pdf",
          "content_url": "https://arxiv.org/pdf/2109.07958",
          "format": "pdf",
          "sha256": "6e86175861d24d075bbd8fec987cbdabd0d6332700b001a1ccf335fa72f541b2",
          "size_bytes": 819857,
          "fetched_at": "2026-11-03T23:22:18Z",
          "uploaded_sha256": "6e86175861d24d075bbd8fec987cbdabd0d6332700b001a1ccf335fa72f541b2"
        }
      ]
    },
    {
      "tag": "winogrande",
      "name": "WinoGrande",
      "title": "WinoGrande: An Adversarial Winograd Schema Challenge at Scale",
      "description": "Benchmark indicates how well a system can answer 44k fill-in-a-blank questions with binary options associated with commonsense reasoning",
      "reference": "https://arxiv.org/abs/1907.10641",
      "reference_note": null,
      "summary": "The Winograd Schema Challenge (WSC) (Levesque, Davis, and Morgenstern 2011), a benchmark for commonsense reasoning, is a set of 273 expert-crafted pronoun resolution problems originally designed to be unsolvable for statistical models that rely on selectional preferences or word associations. However, recent advances in neural language models have already reached around 90% accuracy on variants of WSC. This raises an important question whether these models have truly acquired robust commonsense capabilities or whether they rely on spurious biases in the datasets that lead to an overestimation of the true capabilities of machine commonsense. To investigate this question, we introduce WinoGrande, a large-scale dataset of 44k problems, inspired by the original WSC design, but adjusted to improve both the scale and the hardness of the dataset. The key steps of the dataset construction consist of (1) a carefully designed crowdsourcing procedure, followed by (2) systematic bias reduction using a novel AfLite algorithm that generalizes human-detectable word associations to machine-detectable embedding associations. The best state-of-the-art methods on WinoGrande achieve 59.4-79.1%, which are 15-35% below human performance of 94.0%, depending on the amount of the training data allowed. Furthermore, we establish new state-of-the-art results on five related benchmarks - WSC (90.1%), DPR (93.1%), COPA (90.6%), KnowRef (85.6%), and Winogender (97.1%). These results have dual implications: on one hand, they demonstrate the effectiveness of WinoGrande when used as a resource for transfer learning. On the other hand, they raise a concern that we are likely to be overestimating the true capabilities of machine commonsense across all these benchmarks. We emphasize the importance of algorithmic bias reduction in existing and future benchmarks to mitigate such overestimation.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/WinoGrande.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/WinoGrande.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": false,
        "adopted_mitigations": [
          1,
          106,
          199,
          140,
          102,
          10,
          142,
          12,
          13,
          90,
          14,
          19,
          181,
          93,
          94,
          153,
          154,
          95,
          96,
          97,
          155,
          98,
          156,
          138,
          99,
          157,
          139,
          100,
          158,
          33,
          34,
          159,
          35,
          123,
          82,
          104,
          105,
          103,
          107,
          108,
          109,
          128,
          129,
          130,
          131,
          132,
          124,
          125,
          126,
          133,
          136
        ],
        "absent_mitigations": [
          186,
          198,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          70,
          89,
          5,
          6,
          8,
          101,
          7,
          9,
          141,
          62,
          11,
          164,
          69,
          165,
          178,
          84,
          86,
          183,
          184,
          85,
          87,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          145,
          18,
          201,
          79,
          146,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          46,
          196,
          47,
          81,
          152,
          29,
          135,
          137,
          48,
          182,
          30,
          49,
          31,
          32,
          50,
          54,
          202,
          55,
          58,
          60,
          59,
          61,
          51,
          200,
          56,
          36,
          160,
          197,
          75,
          37,
          122,
          38,
          88,
          66,
          83,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/1907.10641",
          "role": "definitive-reference",
          "key": "winogrande/paper-arxiv-1907.10641.pdf",
          "content_url": "https://arxiv.org/pdf/1907.10641",
          "format": "pdf",
          "sha256": "780b26e4c9dd38c61f2f9e7b99a3e3dfec94fd19bb13bc06338770ca3733e67a",
          "size_bytes": 1098857,
          "fetched_at": "2026-11-03T23:22:21Z",
          "uploaded_sha256": "780b26e4c9dd38c61f2f9e7b99a3e3dfec94fd19bb13bc06338770ca3733e67a"
        }
      ]
    },
    {
      "tag": "wordcraft",
      "name": "Wordcraft",
      "title": "WordCraft: An Environment for Benchmarking Commonsense Agents",
      "description": "Benchmark indicates whether an agent can effectively apply commonsense knowledge in a grounded environment by combining entities according to real-world semantics within a simulated “Little Alchemy 2”–style setting, enabling users to evaluate the agent’s practical reasoning abilities.",
      "reference": "https://arxiv.org/abs/2007.09185",
      "reference_note": null,
      "summary": "The ability to quickly solve a wide range of real-world tasks requires a commonsense understanding of the world. Yet, how to best extract such knowledge from natural language corpora and integrate it with reinforcement learning (RL) agents remains an open challenge. This is partly due to the lack of lightweight simulation environments that sufficiently reflect the semantics of the real world and provide knowledge sources grounded with respect to observations in an RL environment. To better enable research on agents making use of commonsense knowledge, we propose WordCraft, an RL environment based on Little Alchemy 2. This lightweight environment is fast to run and built upon entities and relations inspired by real-world semantics. We evaluate several representation learning methods on this new benchmark and propose a new method for integrating knowledge graphs with an RL agent.",
      "related_links": [],
      "benchrisk": {
        "source_file": "data/scores/Wordcraft.mdx",
        "source_url": "https://github.com/AVERI-org/BenchRisk/blob/f520c3cb6d11f61acb5b6bde938f8fc546b6ff98/data/scores/Wordcraft.mdx",
        "score_page": "https://benchrisk.ai/score",
        "date_scored": "2025-04-01",
        "hidden": true,
        "adopted_mitigations": [
          1,
          89,
          70,
          140,
          5,
          101,
          141,
          10,
          142,
          145,
          18,
          122,
          123,
          128,
          129,
          130,
          131,
          132,
          124
        ],
        "absent_mitigations": [
          186,
          198,
          106,
          64,
          162,
          68,
          187,
          188,
          189,
          190,
          191,
          192,
          2,
          3,
          4,
          163,
          199,
          6,
          8,
          7,
          9,
          102,
          62,
          11,
          164,
          69,
          178,
          84,
          165,
          86,
          183,
          12,
          184,
          85,
          13,
          87,
          90,
          14,
          65,
          71,
          144,
          17,
          193,
          143,
          80,
          201,
          79,
          146,
          19,
          179,
          147,
          180,
          166,
          21,
          194,
          134,
          22,
          195,
          148,
          181,
          23,
          185,
          92,
          63,
          24,
          25,
          72,
          26,
          27,
          91,
          149,
          53,
          150,
          151,
          73,
          28,
          93,
          46,
          81,
          196,
          152,
          29,
          94,
          47,
          48,
          153,
          182,
          135,
          137,
          30,
          49,
          154,
          95,
          31,
          32,
          50,
          96,
          97,
          155,
          54,
          202,
          98,
          156,
          55,
          99,
          58,
          138,
          60,
          157,
          100,
          59,
          139,
          61,
          158,
          33,
          34,
          51,
          159,
          200,
          35,
          56,
          36,
          160,
          197,
          75,
          37,
          38,
          66,
          82,
          83,
          88,
          127,
          57,
          76,
          39,
          42,
          77,
          78,
          44,
          74,
          43,
          45,
          161,
          52,
          67,
          104,
          105,
          103,
          107,
          108,
          109,
          112,
          113,
          114,
          118,
          110,
          115,
          116,
          117,
          119,
          111,
          120,
          121,
          125,
          126,
          133,
          136
        ]
      },
      "resources": [
        {
          "doc_type": "paper",
          "url": "https://arxiv.org/abs/2007.09185",
          "role": "definitive-reference",
          "key": "wordcraft/paper-arxiv-2007.09185.pdf",
          "content_url": "https://arxiv.org/pdf/2007.09185",
          "format": "pdf",
          "sha256": "8c0f88b89925d71d9d5e1f2fd23f03e1e5c74da90ce348b72f3db039f2135c4e",
          "size_bytes": 542026,
          "fetched_at": "2026-11-03T23:22:24Z",
          "uploaded_sha256": "8c0f88b89925d71d9d5e1f2fd23f03e1e5c74da90ce348b72f3db039f2135c4e"
        }
      ]
    }
  ]
}