{ "notice": "Illustrative mapping from the AI Governance Engineer Body of Knowledge v0.5.0 (not a claim of conformity)", "version": "0.5.0", "license": "CC BY 4.0", "licenseUrl": "https://creativecommons.org/licenses/by/4.0/", "schemaVersion": 1, "schema": "https://aigovernanceengineer.com/api/v1/schemas/cases.json", "self": "https://aigovernanceengineer.com/api/v1/cases.json", "source": "https://aigovernanceengineer.com/cases", "citation": { "title": "AI Governance Engineering: The Thesis & Body of Knowledge", "authors": [ "Jorge García Aibar" ], "parentDoi": "https://doi.org/10.5281/zenodo.22956197", "conceptDoi": "https://doi.org/10.5281/zenodo.22857084" }, "cases": [ { "id": "dutch-childcare-benefits", "url": "https://aigovernanceengineer.com/cases/dutch-childcare-benefits", "title": "Dutch childcare benefits: nationality as a risk indicator", "short": "Dutch childcare benefits", "year": "2021", "jurisdiction": "Netherlands", "sector": "Public sector: social benefits", "evidence": "primary", "summary": "The Dutch tax administration used applicants' nationality as a risk indicator for childcare benefits; the data protection authority fined it EUR 2.75 million.", "happened": [ "The Dutch Tax and Customs Administration processed the nationality, and the dual nationality, of childcare-benefit applicants for years. It used Dutch or non-Dutch nationality as an indicator in a system that automatically designated certain applications as risky, and processed nationality to combat organised fraud although that data was not necessary for the purpose [1].", "On 7 Dec 2021 the Dutch data protection authority (AP) fined the administration EUR 2.75 million, finding the processing unlawful and discriminatory, and therefore improper under the GDPR. It said the dual-nationality data should have been deleted in January 2014, and that nationality had not been used to determine risk since October 2018 [1].", "Amnesty International's analysis describes an algorithmic system that built risk profiles of applicants to detect inaccurate and potentially fraudulent applications early, with nationality among the risk factors [2]. The AI Incident Database records the case as families wrongfully accused of tax fraud by a discriminatory algorithm [3]." ], "failureMode": [ "A protected characteristic was a model input. Nothing between the data and the decision checked whether a feature was lawful to use for this purpose, so a risk flag could rest on nationality.", "Retention failed as well: data that should have been deleted in 2014 was still within reach years later [1]. A risk model can use every attribute it can reach, so deletion is a control on the model too." ], "control": [ "A feature policy enforced in the pipeline catches this before the first score: a policy card listing the inputs permitted for the purpose, and an eval gate that fails the build when a prohibited attribute, or a close proxy for one, enters the feature set, or when flag rates diverge across groups. The fundamental-rights impact assessment is where the purpose, the affected groups and the permitted features are decided and signed." ], "controls": [ { "name": "Policy Card", "patternId": "pattern-policy-card" }, { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "FRIA-as-Code", "patternId": "pattern-fria-as-code" } ], "evidenceArtefacts": [ { "artefact": "Policy card for the risk model listing the permitted input features, with nationality marked prohibited for this purpose", "layer": 1 }, { "artefact": "Signed FRIA naming the affected groups, the purpose limitation and the outcome metric to monitor", "layer": 1 }, { "artefact": "Eval-gate run log in which the feature-policy check and the disaggregated flag-rate test pass or block the release", "layer": 3 }, { "artefact": "Retention-job log showing the dual-nationality data deleted on schedule", "layer": 5 } ], "obligations": [ { "instrument": "GDPR", "ref": "Art. 5(1)(a), Art. 35", "why": "The AP's finding rests on lawfulness and fairness [1]; a data protection impact assessment is the GDPR artefact that should have surfaced the nationality feature [4]." }, { "instrument": "EU AI Act", "ref": "Annex III, point 5(a)", "why": "Systems used by or for public authorities to evaluate eligibility for essential public assistance benefits are high-risk [5]; after the AI Omnibus, Annex III obligations apply from 2 Dec 2027 (as of 2026-09-24) [6]." }, { "instrument": "EU AI Act", "ref": "Art. 27", "why": "A public body deploying such a system carries out a fundamental-rights impact assessment before first use [7]." }, { "instrument": "EU AI Act", "ref": "Art. 5(1)(c)", "why": "Social scoring that leads to unjustified or disproportionate detrimental treatment is prohibited [8]. Whether a given risk model meets those conditions is a legal judgement, not an engineering one." } ], "harms": [ "group-risk-profiling", "discriminatory-decisions", "regulatory-enforcement" ], "incidents": [ { "db": "AIID", "id": "101", "title": "Dutch Families Wrongfully Accused of Tax Fraud Due to Discriminatory Algorithm", "url": "https://incidentdatabase.ai/cite/101/" } ], "sources": [ { "n": 1, "text": "Tax Administration fined for discriminatory and unlawful data processing (EUR 2.75 million fine; nationality used as a risk indicator). Autoriteit Persoonsgegevens (Dutch Data Protection Authority). 2021-12-07.", "url": "https://www.autoriteitpersoonsgegevens.nl/en/current/tax-administration-fined-for-discriminatory-and-unlawful-data-processing", "verified": "primary" }, { "n": 2, "text": "Xenophobic machines: discrimination through unregulated use of algorithms in the Dutch childcare benefits scandal (EUR 35/4686/2021). Amnesty International. 2021-10-25.", "url": "https://www.amnesty.org/en/documents/eur35/4686/2021/en/", "verified": "primary" }, { "n": 3, "text": "AI Incident Database, Incident 101: Dutch Families Wrongfully Accused of Tax Fraud Due to Discriminatory Algorithm. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/101/", "verified": "primary" }, { "n": 4, "text": "Regulation (EU) 2016/679 (General Data Protection Regulation) (Art. 5 principles, Art. 6 lawfulness, Art. 8 child's consent, Art. 9 special categories, Arts. 12-15 transparency and access, Art. 22 automated individual decision-making, Art. 33 breach notification, Art. 35 DPIA). Official Journal of the European Union (EUR-Lex). 2016-04-27.", "url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj/eng", "verified": "primary" }, { "n": 5, "text": "EU AI Act Annex III (high-risk uses; point 3(b) evaluating learning outcomes, 4(a) recruitment and selection, 5(a) eligibility for essential public assistance benefits and services). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#anx_III", "verified": "primary" }, { "n": 6, "text": "AI Omnibus enters into force (Reg. (EU) 2026/1744, in force 2026-07-27; Annex III high-risk obligations move to 2 Dec 2027). European Commission. 2026-07-27.", "url": "https://digital-strategy.ec.europa.eu/en/news/ai-omnibus-enters-force", "verified": "primary" }, { "n": 7, "text": "EU AI Act Art. 27 (FRIA before first use by deployers that are bodies governed by public law or private entities providing public services, and by deployers of Annex III point 5(b) and (c) systems; Art. 27(4) cross-reference to a GDPR Art. 35 DPIA). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_27", "verified": "primary" }, { "n": 8, "text": "EU AI Act Art. 5 (prohibited AI practices; 5(1)(c) social scoring leading to unjustified or disproportionate detrimental treatment; 5(1)(e) facial recognition databases built by untargeted scraping). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_5", "verified": "primary" } ], "systemBoundary": null, "controlAssumptions": null, "preventiveControls": null, "detectiveControls": null, "responsiveControls": null, "evidenceRequirements": null, "relatedControls": null, "openQuestions": null }, { "id": "syri-judgment", "url": "https://aigovernanceengineer.com/cases/syri-judgment", "title": "SyRI: a fraud risk model no court could verify", "short": "SyRI judgment", "year": "2020", "jurisdiction": "Netherlands", "sector": "Public sector: welfare and fraud detection", "evidence": "primary", "summary": "A Dutch court struck down the SyRI fraud-detection legislation in 2020 because the system was insufficiently transparent and verifiable.", "happened": [ "SyRI (System Risk Indication) was a legal instrument the Dutch government used to detect fraud with social benefits, allowances and taxes. Data from participating public bodies were linked, pseudonymised and checked against a risk model, and matches produced risk reports on individuals [1].", "On 5 Feb 2020 The Hague District Court held that the SyRI legislation did not comply with Article 8(2) of the European Convention on Human Rights: it did not strike the fair balance the Convention requires, and the application of SyRI was insufficiently transparent and verifiable. The court declared the legislation to have no binding effect [1].", "The court noted that the State had not given it objectively verifiable information on the nature of SyRI, and that SyRI was used to investigate neighbourhoods known as problem areas [1]. The AIAAIC Repository records the case [2]." ], "failureMode": [ "Opacity was the design, not a side effect. The indicators and the risk model were not open to inspection, so neither the people flagged nor the court could check why a record matched. A control that cannot be inspected cannot be shown to be proportionate.", "Deployment by neighbourhood meant the population scanned was chosen before any individual suspicion existed." ], "control": [ "The missing artefact is an inspectable description of the model: its purpose, indicators, validation and limits, kept as control evidence rather than as a brochure. Paired with a fundamental-rights impact assessment that states the necessity and proportionality reasoning, and with evidence emitted in a machine-readable form, it gives a court or an auditor something to verify without publishing the model for gaming." ], "controls": [ { "name": "Model Card as Control Evidence", "patternId": "pattern-model-card-as-control-evidence" }, { "name": "FRIA-as-Code", "patternId": "pattern-fria-as-code" }, { "name": "Machine-Readable Evidence (OSCAL)", "patternId": "pattern-machine-readable-evidence-oscal" } ], "evidenceArtefacts": [ { "artefact": "Model card stating purpose, indicators, validation results and known limits, versioned with each risk model", "layer": 2 }, { "artefact": "FRIA recording the interference with private life, the necessity and proportionality reasoning, and how target areas were chosen", "layer": 1 }, { "artefact": "Per-run record of which data sources were linked, how many records were flagged and on which indicators", "layer": 5 }, { "artefact": "Machine-readable evidence package an oversight body can query without access to the raw data", "layer": 5 } ], "obligations": [ { "instrument": "ECHR", "ref": "Art. 8", "why": "The judgment's legal basis: an interference with private life must be transparent and verifiable enough to be weighed [1]." }, { "instrument": "GDPR", "ref": "Art. 22", "why": "Automated individual decision-making, including profiling, is restricted and carries safeguards [7]." }, { "instrument": "EU AI Act", "ref": "Annex III, point 5(a); Art. 27", "why": "Fraud-risk scoring of benefit recipients by public authorities falls in the benefits use case [4]; Annex III obligations apply from 2 Dec 2027 (as of 2026-09-24) [5], and public deployers carry out a FRIA before first use [6]." }, { "instrument": "EU AI Act", "ref": "Art. 86", "why": "People affected by a decision based on a high-risk system have a right to an explanation of individual decision-making [3]." } ], "harms": [ "group-risk-profiling" ], "incidents": [ { "db": "AIAAIC", "id": "syri-welfare-fraud-detection-automation", "title": "SyRI welfare fraud detection automation", "url": "https://www.aiaaic.org/aiaaic-repository/ai-algorithmic-and-automation-incidents/syri-welfare-fraud-detection-automation" } ], "sources": [ { "n": 1, "text": "NJCM et al. v. The State of the Netherlands (SyRI), C/09/550982 / HA ZA 18-388 (ECLI:NL:RBDHA:2020:1878, English translation of ECLI:NL:RBDHA:2020:865). Rechtbank Den Haag (The Hague District Court). 2020-02-05.", "url": "https://uitspraken.rechtspraak.nl/details?id=ECLI:NL:RBDHA:2020:1878", "verified": "primary" }, { "n": 2, "text": "SyRI welfare fraud detection automation. AIAAIC Repository. 2026.", "url": "https://www.aiaaic.org/aiaaic-repository/ai-algorithmic-and-automation-incidents/syri-welfare-fraud-detection-automation", "verified": "primary" }, { "n": 3, "text": "EU AI Act Art. 86 (right to explanation of individual decision-making based on the output of a high-risk AI system). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_86", "verified": "primary" }, { "n": 4, "text": "EU AI Act Annex III (high-risk uses; point 3(b) evaluating learning outcomes, 4(a) recruitment and selection, 5(a) eligibility for essential public assistance benefits and services). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#anx_III", "verified": "primary" }, { "n": 5, "text": "AI Omnibus enters into force (Reg. (EU) 2026/1744, in force 2026-07-27; Annex III high-risk obligations move to 2 Dec 2027). European Commission. 2026-07-27.", "url": "https://digital-strategy.ec.europa.eu/en/news/ai-omnibus-enters-force", "verified": "primary" }, { "n": 6, "text": "EU AI Act Art. 27 (FRIA before first use by deployers that are bodies governed by public law or private entities providing public services, and by deployers of Annex III point 5(b) and (c) systems; Art. 27(4) cross-reference to a GDPR Art. 35 DPIA). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_27", "verified": "primary" }, { "n": 7, "text": "Regulation (EU) 2016/679 (General Data Protection Regulation) (Art. 5 principles, Art. 6 lawfulness, Art. 8 child's consent, Art. 9 special categories, Arts. 12-15 transparency and access, Art. 22 automated individual decision-making, Art. 33 breach notification, Art. 35 DPIA). Official Journal of the European Union (EUR-Lex). 2016-04-27.", "url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj/eng", "verified": "primary" } ], "systemBoundary": null, "controlAssumptions": null, "preventiveControls": null, "detectiveControls": null, "responsiveControls": null, "evidenceRequirements": null, "relatedControls": null, "openQuestions": null }, { "id": "a-level-grading-2020", "url": "https://aigovernanceengineer.com/cases/a-level-grading-2020", "title": "England's 2020 A levels: a centre model applied to individual students", "short": "2020 A-level grades", "year": "2020", "jurisdiction": "England, United Kingdom", "sector": "Education: assessment", "evidence": "primary", "summary": "With exams cancelled in 2020, England's grading model assigned A-level grades from each school's history; four days after results, Ofqual reverted to teacher grades.", "happened": [ "With exams cancelled in 2020, schools and colleges submitted a centre assessment grade (CAG) for each student and a rank order of students in each subject [1].", "Ofqual's Direct Centre Performance model predicted the distribution of grades for each school or college from its historical results in the subject, taking account of the prior attainment of this year's students, then used the submitted rank order to assign grades to individual students. Where a subject cohort was small (fewer than 15 students across the current and historical entries), the model put more weight on the CAGs [1].", "The interim report, published on results day (13 Aug 2020), stated that 96.4% of calculated grades were the same as, or within one grade of, the CAG, and that its equalities analysis showed no evidence the process had introduced bias [1]. On 17 Aug 2020 Ofqual announced that AS and A level results, and the GCSE results due later that week, would switch to centre assessment grades [2]. The AIAAIC Repository records the case [3]." ], "failureMode": [ "The model was checked at the level it was designed for and failed at the level it was used. The headline checks in the interim report are aggregate: prediction accuracy within one grade on historical data, and an equalities analysis by group [1]. A student's grade, though, came from their school's past and their place in a rank order, so a strong student in a historically weaker centre could be held down by results they had no part in.", "The small-cohort rule made treatment depend on class size: small groups kept more of their teachers' grades than large ones [1]. That is a design choice with distributional consequences, and it needed testing as one." ], "control": [ "An eval gate with individual-level acceptance criteria forces the question the aggregate tests skip: for which students does the model move the grade furthest from the evidence about them, and can each move be justified? A human-in-the-loop gate on large adjustments, with a contestation route before results day, turns those outliers into reviewed decisions instead of published ones. The fundamental-rights impact assessment is where the treatment of small and large cohorts would have been argued in the open." ], "controls": [ { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Human-in-the-loop Gate", "patternId": "pattern-human-in-the-loop-gate" }, { "name": "FRIA-as-Code", "patternId": "pattern-fria-as-code" } ], "evidenceArtefacts": [ { "artefact": "Eval report slicing adjustments by cohort size, centre type and prior-attainment band, with pass criteria fixed before the run", "layer": 3 }, { "artefact": "Review-queue log for every adjustment beyond a set threshold, with the reviewer's decision", "layer": 4 }, { "artefact": "FRIA recording the small-cohort rule, whom it favours and why it was accepted", "layer": 1 }, { "artefact": "Contestation log: requests, outcomes and time to resolution", "layer": 5 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Annex III, point 3(b)", "why": "An equivalent system in the EU would be high-risk: systems intended to evaluate learning outcomes are listed [4], with Annex III obligations applying from 2 Dec 2027 (as of 2026-09-24) [5]." }, { "instrument": "UK data protection", "ref": "Data (Use and Access) Act 2025, s. 80", "why": "Replaces UK GDPR Art. 22 on automated decision-making with Arts. 22A-22D, in force since 5 Feb 2026 (as of 2026-09-24) [6]." } ], "harms": [ "cohort-penalty" ], "incidents": [ { "db": "AIAAIC", "id": "ofqal-algorithm-skews-student-grade-predictions", "title": "Ofqal algorithm skews student grade predictions", "url": "https://www.aiaaic.org/aiaaic-repository/ai-algorithmic-and-automation-incidents/ofqal-algorithm-skews-student-grade-predictions" } ], "sources": [ { "n": 1, "text": "Awarding GCSE, AS, A level, advanced extension awards and extended project qualifications in summer 2020: interim report (Direct Centre Performance model; small-cohort rule; 96.4% within one grade of the CAG). Ofqual. 2020-08-13.", "url": "https://www.gov.uk/government/publications/awarding-gcse-as-a-levels-in-summer-2020-interim-report", "verified": "primary" }, { "n": 2, "text": "Statement from Roger Taylor, Chair, Ofqual (switch to centre assessment grades for AS, A level and GCSE). Ofqual (GOV.UK). 2020-08-17.", "url": "https://www.gov.uk/government/news/statement-from-roger-taylor-chair-ofqual", "verified": "primary" }, { "n": 3, "text": "Ofqal algorithm skews student grade predictions. AIAAIC Repository. 2026.", "url": "https://www.aiaaic.org/aiaaic-repository/ai-algorithmic-and-automation-incidents/ofqal-algorithm-skews-student-grade-predictions", "verified": "primary" }, { "n": 4, "text": "EU AI Act Annex III (high-risk uses; point 3(b) evaluating learning outcomes, 4(a) recruitment and selection, 5(a) eligibility for essential public assistance benefits and services). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#anx_III", "verified": "primary" }, { "n": 5, "text": "AI Omnibus enters into force (Reg. (EU) 2026/1744, in force 2026-07-27; Annex III high-risk obligations move to 2 Dec 2027). European Commission. 2026-07-27.", "url": "https://digital-strategy.ec.europa.eu/en/news/ai-omnibus-enters-force", "verified": "primary" }, { "n": 6, "text": "Data (Use and Access) Act 2025, s. 80 (replaces UK GDPR Art. 22 with Arts. 22A-22D; in force 5 Feb 2026). legislation.gov.uk. 2025.", "url": "https://www.legislation.gov.uk/ukpga/2025/18/section/80", "verified": "primary" } ], "systemBoundary": null, "controlAssumptions": null, "preventiveControls": null, "detectiveControls": null, "responsiveControls": null, "evidenceRequirements": null, "relatedControls": null, "openQuestions": null }, { "id": "health-risk-score-proxy", "url": "https://aigovernanceengineer.com/cases/health-risk-score-proxy", "title": "A health-risk score that predicted cost, not need", "short": "Health-risk score", "year": "2019", "jurisdiction": "United States", "sector": "Healthcare: care management", "evidence": "primary", "summary": "A widely used care-management algorithm predicted health costs as a proxy for illness, so Black patients were sicker than White patients at the same score.", "happened": [ "Health systems use commercial prediction algorithms to pick patients with complex needs for extra care. Obermeyer and colleagues studied one widely used algorithm of this kind and found that, at a given risk score, Black patients were considerably sicker than White patients, as shown by signs of uncontrolled illness [1].", "The bias arose because the algorithm predicts health-care costs rather than illness, and unequal access to care means less is spent on Black patients than on White patients. Remedying the disparity would raise the share of Black patients receiving additional help from 17.7% to 46.5% [1]. The AI Incident Database records the case against the vendor, as reported [2]." ], "failureMode": [ "Label choice. The target the model learned (future cost) stood in for the construct the programme cared about (future need), and the proxy was itself shaped by unequal access. Measured against its own label, the model could look accurate and still be wrong about who needed care [1]." ], "control": [ "The model card is where the gap between label and construct is written down and owned: what the programme wants to predict, what the model actually predicts, and why the difference is acceptable. An eval gate that checks calibration by group against a measure closer to the construct (for example, active chronic conditions) shows whether equal scores mean equal need." ], "controls": [ { "name": "Model Card as Control Evidence", "patternId": "pattern-model-card-as-control-evidence" }, { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" } ], "evidenceArtefacts": [ { "artefact": "Model card section on label validity: construct, proxy label and known gaps, signed by the clinical owner", "layer": 2 }, { "artefact": "Eval report of calibration by group against a health measure, with the gate's verdict", "layer": 3 }, { "artefact": "Change record whenever the label or the programme-entry threshold changes", "layer": 5 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 10", "why": "For high-risk systems, training data must be examined for possible biases likely to affect health and safety or fundamental rights [3]." }, { "instrument": "EU AI Act", "ref": "Annex III, point 5(a)", "why": "Evaluating eligibility for essential public assistance benefits and services, including healthcare services, is high-risk when done by or for public authorities [4]. Whether a care-management score used by a private provider falls there, or under a medical-device route, is a legal classification question." } ], "harms": [ "discriminatory-decisions" ], "incidents": [ { "db": "AIID", "id": "124", "title": "Optum Algorithmic Health Risk Scores Reportedly Underestimated Black Patients' Needs", "url": "https://incidentdatabase.ai/cite/124/" } ], "sources": [ { "n": 1, "text": "Dissecting racial bias in an algorithm used to manage the health of populations (Science 366(6464):447-453). Obermeyer, Z., Powers, B., Vogeli, C., Mullainathan, S.. 2019-10-25.", "url": "https://doi.org/10.1126/science.aax2342", "verified": "primary" }, { "n": 2, "text": "AI Incident Database, Incident 124: Optum Algorithmic Health Risk Scores Reportedly Underestimated Black Patients' Needs. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/124/", "verified": "primary" }, { "n": 3, "text": "EU AI Act Art. 10 (data and data governance; examination of training data for possible biases). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_10", "verified": "primary" }, { "n": 4, "text": "EU AI Act Annex III (high-risk uses; point 3(b) evaluating learning outcomes, 4(a) recruitment and selection, 5(a) eligibility for essential public assistance benefits and services). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#anx_III", "verified": "primary" } ], "systemBoundary": null, "controlAssumptions": null, "preventiveControls": null, "detectiveControls": null, "responsiveControls": null, "evidenceRequirements": null, "relatedControls": null, "openQuestions": null }, { "id": "moffatt-v-air-canada", "url": "https://aigovernanceengineer.com/cases/moffatt-v-air-canada", "title": "Moffatt v. Air Canada: the chatbot's answer is the company's answer", "short": "Moffatt v. Air Canada", "year": "2024", "jurisdiction": "British Columbia, Canada", "sector": "Aviation: customer service", "evidence": "secondary", "summary": "A tribunal held Air Canada liable after its website chatbot misstated the bereavement-fare policy, rejecting the argument that the chatbot answered for itself.", "happened": [ "A passenger asked Air Canada's website chatbot how bereavement fares worked. The chatbot told him that if he had already travelled he could submit his ticket for a reduced bereavement rate within 90 days of issue, while the airline's policy stated that it would not provide refunds for bereavement travel after the flight was booked [2].", "Before the Civil Resolution Tribunal of British Columbia, the airline argued that it could not be held liable for information provided by one of its agents, servants or representatives, including a chatbot. Tribunal member Christopher Rivers found that Air Canada did not take reasonable care to ensure its chatbot was accurate, wrote that it should be obvious to Air Canada that it is responsible for all the information on its website, and awarded a partial refund [2]. The decision is Moffatt v. Air Canada, 2024 BCCRT 149 [1]; the AI Incident Database reports a total of CAD 812.02 in damages and fees, and records the finding as negligent misrepresentation [3]." ], "failureMode": [ "The chatbot generated an answer about a policy without checking it against the policy. Two parts of the same website disagreed, and nothing detected the disagreement before a customer relied on it.", "The airline's defence treated the chatbot as outside its own accountability. The tribunal did not accept that [2], and an engineering function should not build on it either." ], "control": [ "A runtime guardrail that grounds every policy answer in the current policy document, cites it, and refuses when no passage supports the answer stops the wrong promise at the point of output. An eval gate with a regression set of policy questions (fares, refunds, eligibility) catches the same failure before release, and the incident pipeline routes complaints about bot answers back into that set." ], "controls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceArtefacts": [ { "artefact": "Guardrail log per answer: the policy passage retrieved and cited, or the refusal", "layer": 4 }, { "artefact": "Eval report on the policy-question regression set, with the release threshold", "layer": 3 }, { "artefact": "Incident records linking each complaint to the bot answer and the fix", "layer": 5 } ], "obligations": [ { "instrument": "Canadian common law", "ref": "Negligent misrepresentation", "why": "The tribunal applied ordinary duties of care to the chatbot's statements: the airline answers for them as for any page of its site [2][3]." }, { "instrument": "EU AI Act", "ref": "Art. 50(1)", "why": "In the EU, systems that interact directly with people must be designed so that people know they are dealing with an AI system [4]; Art. 50 applies from 2 Aug 2026 (as of 2026-09-24) [5]. Disclosure does not shift liability for what the system says." } ], "harms": [ "liability-for-outputs", "reputational-harm" ], "incidents": [ { "db": "AIID", "id": "639", "title": "Air Canada Chatbot Reportedly Provides Inaccurate Bereavement Fare Information, Leading to Customer Overpayment", "url": "https://incidentdatabase.ai/cite/639/" } ], "sources": [ { "n": 1, "text": "Moffatt v. Air Canada, 2024 BCCRT 149 (decision not opened directly; citation and holdings confirmed through sources 2 and 3). Civil Resolution Tribunal of British Columbia (CanLII). 2024-02-14.", "url": "https://www.canlii.org/en/bc/bccrt/doc/2024/2024bccrt149/2024bccrt149.html", "verified": "secondary" }, { "n": 2, "text": "Air Canada must honor refund policy invented by airline's chatbot (quotes the chatbot, the policy page and the tribunal member). Ars Technica. 2024-02-16.", "url": "https://arstechnica.com/tech-policy/2024/02/air-canada-must-honor-refund-policy-invented-by-airlines-chatbot/", "verified": "secondary" }, { "n": 3, "text": "AI Incident Database, Incident 639: Air Canada Chatbot Reportedly Provides Inaccurate Bereavement Fare Information, Leading to Customer Overpayment. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/639/", "verified": "primary" }, { "n": 4, "text": "EU AI Act Art. 50 (transparency; systems that interact directly with natural persons must be designed so that those persons are informed they are interacting with an AI system). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_50", "verified": "primary" }, { "n": 5, "text": "Safer and more transparent AI (Art. 50 transparency obligations apply from 2 Aug 2026). European Commission. 2026-08-02.", "url": "https://commission.europa.eu/news-and-media/news/safer-and-more-transparent-ai-2026-08-02_en", "verified": "primary" } ], "systemBoundary": "The chatbot on Air Canada's website and the policy page on the same site: one told the passenger he could claim a bereavement rate after travel, the other said the airline would not refund bereavement travel after booking [2]. The tribunal member wrote that the airline is responsible for all the information on its website [2], so the boundary of the system is the website, not the model.", "controlAssumptions": [ "The published policy page is the authoritative source; an answer that disagrees with it is wrong, however it was generated.", "The organisation answers for what its chatbot says as for any other page of its site; the tribunal rejected the argument that it did not [2]." ], "preventiveControls": [ { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" } ], "detectiveControls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" } ], "responsiveControls": [ { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceRequirements": [ "Every version that goes live has an eval report on a regression set of policy questions (fares, refunds, eligibility) that meets the release threshold.", "Every policy answer in the guardrail log carries the policy passage it rests on, or is a refusal.", "Every complaint about a bot answer has an incident record linking the answer, the policy passage and the fix, and the question has joined the regression set." ], "relatedControls": null, "openQuestions": [ "When a policy page changes, how quickly must the regression set and the grounding corpus follow, and what record shows that they did?" ] }, { "id": "recruiting-model-reported", "url": "https://aigovernanceengineer.com/cases/recruiting-model-reported", "title": "A recruiting model that learned the past (reported)", "short": "Recruiting model (reported)", "year": "2018", "jurisdiction": "Not stated in the reports", "sector": "Employment: recruitment", "evidence": "reported", "summary": "Reuters reported in 2018 that an experimental recruiting model trained on a decade of mostly male CVs learned to downgrade women; the project was dropped.", "happened": [ "Reuters reported on 10 Oct 2018 that Amazon had built an experimental tool, from 2014, to score job applicants; that it was trained on about ten years of CVs drawn largely from men; that it learned to penalise CVs containing the word \"women's\" and graduates of certain all-women colleges; and that the project was abandoned [1][2]. These are reported facts: no primary company document is public.", "The company is reported to have said that recruiters never relied solely on the tool's rankings [2]." ], "failureMode": [ "Historical label bias. The model learned what past hiring looked like, past hiring skewed male, and the skew became a scoring rule. Removing explicit gendered terms does not remove proxies for them, which is why reports say the fixes did not guarantee fairness [2]." ], "control": [ "An eval gate that computes selection rates by sex (and other protected characteristics) on a held-out set, and fails the build when an impact ratio falls below a set threshold, catches this in the first release candidate rather than after years of development. A data card that states the composition of the training population makes the risk visible before training starts. The fundamental-rights impact assessment is where the decision to automate screening at all is argued." ], "controls": [ { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Model Card as Control Evidence", "patternId": "pattern-model-card-as-control-evidence" }, { "name": "FRIA-as-Code", "patternId": "pattern-fria-as-code" } ], "evidenceArtefacts": [ { "artefact": "Data card stating the composition of the training population by sex and period", "layer": 2 }, { "artefact": "Bias-audit report per release with selection rates and impact ratios by group", "layer": 3 }, { "artefact": "Eval-gate log showing the release blocked or passed on those ratios", "layer": 3 }, { "artefact": "FRIA recording the decision to automate screening and its safeguards", "layer": 1 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Annex III, point 4(a)", "why": "Recruitment and selection, including filtering applications and evaluating candidates, is high-risk [3]; Annex III obligations apply from 2 Dec 2027 (as of 2026-09-24) [4]." }, { "instrument": "EU AI Act", "ref": "Art. 10", "why": "Training data for high-risk systems must be examined for possible biases [5]." }, { "instrument": "New York City", "ref": "Local Law 144 of 2021", "why": "Employers may not use an automated employment decision tool unless it has had a bias audit within one year, the audit information is public and notices have been given; enforcement began on 5 Jul 2023 [6]." } ], "harms": [ "discriminatory-decisions" ], "incidents": [ { "db": "AIID", "id": "37", "title": "Amazon's Experimental Hiring Tool Allegedly Displayed Gender Bias in Candidate Rankings", "url": "https://incidentdatabase.ai/cite/37/" } ], "sources": [ { "n": 1, "text": "Amazon scraps secret AI recruiting tool that showed bias against women. Reuters. 2018-10-10.", "url": "https://www.reuters.com/article/us-amazon-com-jobs-automation-insight-idUSKCN1MK08G", "verified": "reported" }, { "n": 2, "text": "AI Incident Database, Incident 37: Amazon's Experimental Hiring Tool Allegedly Displayed Gender Bias in Candidate Rankings. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/37/", "verified": "primary" }, { "n": 3, "text": "EU AI Act Annex III (high-risk uses; point 3(b) evaluating learning outcomes, 4(a) recruitment and selection, 5(a) eligibility for essential public assistance benefits and services). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#anx_III", "verified": "primary" }, { "n": 4, "text": "AI Omnibus enters into force (Reg. (EU) 2026/1744, in force 2026-07-27; Annex III high-risk obligations move to 2 Dec 2027). European Commission. 2026-07-27.", "url": "https://digital-strategy.ec.europa.eu/en/news/ai-omnibus-enters-force", "verified": "primary" }, { "n": 5, "text": "EU AI Act Art. 10 (data and data governance; examination of training data for possible biases). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_10", "verified": "primary" }, { "n": 6, "text": "Automated Employment Decision Tools (AEDT) (Local Law 144 of 2021; bias audit within one year of use; enforcement from 5 Jul 2023). NYC Department of Consumer and Worker Protection. 2023.", "url": "https://www.nyc.gov/site/dca/about/automated-employment-decision-tools.page", "verified": "primary" } ], "systemBoundary": null, "controlAssumptions": null, "preventiveControls": null, "detectiveControls": null, "responsiveControls": null, "evidenceRequirements": null, "relatedControls": null, "openQuestions": null }, { "id": "zillow-offers", "url": "https://aigovernanceengineer.com/cases/zillow-offers", "title": "Zillow Offers: a pricing model committing capital into a turning market", "short": "Zillow Offers", "year": "2021", "jurisdiction": "United States", "sector": "Real estate: automated home buying", "evidence": "primary", "summary": "Zillow wound down its home-buying business in 2021 after buying homes above what it expected to sell them for, taking a USD 304 million write-down.", "happened": [ "On 2 Nov 2021 Zillow Group announced its plan to wind down Zillow Offers, the business in which it bought and sold homes directly [1].", "Its third-quarter results included an inventory write-down of approximately USD 304 million, the result of buying homes at prices higher than its current estimates of future selling prices. The wind-down was expected to take several quarters and to reduce the workforce by approximately 25% [1].", "The chief executive said the unpredictability in forecasting home prices far exceeded what the company had anticipated, and that continuing to scale would bring too much earnings and balance-sheet volatility [1]. The AI Incident Database records the case as a pricing tool with insufficient accuracy [2]." ], "failureMode": [ "A forecasting model was used to commit capital at volume in a market whose behaviour was shifting. Public filings do not describe the company's internal model controls, so this is an illustrative analysis, not a finding: the loss pattern is the one a model produces when its error is not tied to a limit on what it may commit." ], "control": [ "Continuous assurance telemetry that compares each purchase forecast with the realised resale price, by market and cohort, gives the early signal. A circuit breaker that throttles purchase volume automatically when realised error crosses a set threshold turns the signal into a control instead of a quarterly surprise." ], "controls": [ { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" }, { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" } ], "evidenceArtefacts": [ { "artefact": "Drift dashboard of forecast against realised price by market, with thresholds", "layer": 5 }, { "artefact": "Circuit-breaker configuration and activation log: what was throttled, by what rule, when", "layer": 4 }, { "artefact": "Model validation record stating the conditions under which the model must not be used", "layer": 2 } ], "obligations": [ { "instrument": "NIST AI RMF", "ref": "Measure, Manage", "why": "The voluntary home for this monitoring and response: the framework organises AI risk work into Govern, Map, Measure and Manage [3]." }, { "instrument": "EU AI Act", "ref": "Annex III", "why": "No AI-specific duty applies: a pricing model for a company's own purchases is not among the Annex III uses [4]. The loss is a governance failure, not a compliance one." } ], "harms": [ "forecast-drift-loss" ], "incidents": [ { "db": "AIID", "id": "149", "title": "Zillow Shut Down Zillow Offers Division Allegedly Due to Predictive Pricing Tool's Insufficient Accuracy", "url": "https://incidentdatabase.ai/cite/149/" } ], "sources": [ { "n": 1, "text": "Zillow Group Reports Third-Quarter 2021 Financial Results; Shares Plan to Wind Down Zillow Offers Operations (Form 8-K, Exhibit 99.1). Zillow Group, Inc. (SEC EDGAR). 2021-11-02.", "url": "https://www.sec.gov/Archives/edgar/data/1617640/000161764021000085/q32021991.htm", "verified": "primary" }, { "n": 2, "text": "AI Incident Database, Incident 149: Zillow Shut Down Zillow Offers Division Allegedly Due to Predictive Pricing Tool's Insufficient Accuracy. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/149/", "verified": "primary" }, { "n": 3, "text": "AI Risk Management Framework 1.0 (functions: Govern, Map, Measure, Manage). NIST. 2023-01-26.", "url": "https://www.nist.gov/itl/ai-risk-management-framework", "verified": "primary" }, { "n": 4, "text": "EU AI Act Annex III (high-risk uses; point 3(b) evaluating learning outcomes, 4(a) recruitment and selection, 5(a) eligibility for essential public assistance benefits and services). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#anx_III", "verified": "primary" } ], "systemBoundary": "The home-price forecasting model and the purchases it priced: in Zillow Offers the company bought and sold homes directly [1], so a forecast became capital committed to a house. The resale market sits outside the system; its behaviour is what the model had to track.", "controlAssumptions": [ "A forecast used to commit capital needs a stated error bound, measured against realised prices, beyond which it may not be used.", "Illustrative, not a finding: public filings do not describe the company's internal model controls, so this note assumes that purchase volume was not tied automatically to realised forecast error." ], "preventiveControls": [ { "name": "Model Card as Control Evidence", "patternId": "pattern-model-card-as-control-evidence" } ], "detectiveControls": [ { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" } ], "evidenceRequirements": [ "A model card or validation record states the market conditions under which the forecast must not be used to price a purchase.", "A drift dashboard compares each purchase forecast with the realised resale price, by market and cohort, against set thresholds.", "The circuit-breaker log shows purchase volume throttled when realised error crossed its threshold: what was throttled, by which rule and when." ], "relatedControls": null, "openQuestions": [ "Which error measure should trip the breaker for a model whose errors are realised only when a home is resold, months after it was bought?" ] }, { "id": "clearview-ai", "url": "https://aigovernanceengineer.com/cases/clearview-ai", "title": "Clearview AI: a face database built by scraping", "short": "Clearview AI", "year": "2024", "jurisdiction": "Netherlands (EU)", "sector": "Biometrics: facial recognition", "evidence": "primary", "summary": "The Dutch data protection authority fined Clearview AI EUR 30.5 million in 2024 for building a facial-recognition database from scraped photos.", "happened": [ "Clearview AI collects photos of faces from the internet and converts each into a unique biometric code, without the people concerned knowing or consenting [1].", "On 3 Sep 2024 the Dutch data protection authority (AP) announced a fine of EUR 30.5 million and orders subject to penalties of up to more than EUR 5 million. It found that Clearview should never have built the database and informs the people in it insufficiently; Clearview did not object to the decision and so cannot appeal the fine [1].", "The European Data Protection Board's summary of the decision lists, among other violations, processing of biometric data contrary to Art. 9(1) GDPR, processing without a lawful basis under Art. 6(1), and failure to answer access requests under Art. 12 and 15 [2]. The AI Incident Database records both the scraping and the fine [3][4]." ], "failureMode": [ "For the provider, the failure is at the source: personal and biometric data gathered at scale without a lawful basis. For every organisation that buys such a service, the failure is procurement: integrating a capability without asking how its data was obtained." ], "control": [ "A vendor due-diligence gate that asks for the provider's lawful basis for its reference data, and treats biometric processing as a stop condition pending a DPIA, keeps a buyer out of the harm. An AIBOM that records the provenance of each dataset gives the provider the same check at build time." ], "controls": [ { "name": "Vendor / Model Due-Diligence Gate", "patternId": "pattern-vendor--model-due-diligence-gate" }, { "name": "AIBOM", "patternId": "pattern-aibom" }, { "name": "FRIA-as-Code", "patternId": "pattern-fria-as-code" } ], "evidenceArtefacts": [ { "artefact": "Due-diligence record with the provider's lawful-basis and provenance answers and the reject decision", "layer": 2 }, { "artefact": "AIBOM dataset entries with source, collection method, licence and lawful basis", "layer": 2 }, { "artefact": "DPIA for any biometric use, signed before integration", "layer": 1 } ], "obligations": [ { "instrument": "GDPR", "ref": "Art. 6, 9, 12, 15", "why": "Lawful basis, special-category biometric data, and the right of access: the provisions the AP decision applies [1][2][5]." }, { "instrument": "EU AI Act", "ref": "Art. 5(1)(e)", "why": "Placing on the market, putting into service or using AI systems that create or expand facial recognition databases through untargeted scraping of facial images from the internet or CCTV footage is prohibited [6]; the prohibitions apply from 2 Feb 2025 [7]." } ], "harms": [ "privacy-intrusion", "regulatory-enforcement" ], "incidents": [ { "db": "AIID", "id": "267", "title": "Clearview AI Algorithm Built on Photos Scraped from Social Media Profiles without Consent", "url": "https://incidentdatabase.ai/cite/267/" }, { "db": "AIID", "id": "781", "title": "Clearview AI Reportedly Faces $33.7 Million Fine for Violating GDPR with Biometric Data Harvesting", "url": "https://incidentdatabase.ai/cite/781/" } ], "sources": [ { "n": 1, "text": "Dutch DPA imposes a fine on Clearview because of illegal data collection for facial recognition (EUR 30.5 million fine; orders subject to penalties). Autoriteit Persoonsgegevens (Dutch Data Protection Authority). 2024-09-03.", "url": "https://www.autoriteitpersoonsgegevens.nl/en/current/dutch-dpa-imposes-a-fine-on-clearview-because-of-illegal-data-collection-for-facial-recognition", "verified": "primary" }, { "n": 2, "text": "Dutch Supervisory Authority imposes a fine on Clearview because of illegal data collection for facial recognition (national news summary listing the GDPR articles found infringed). European Data Protection Board. 2024-09-03.", "url": "https://www.edpb.europa.eu/news/national-news/2024/dutch-supervisory-authority-imposes-fine-clearview-because-illegal-data_en", "verified": "primary" }, { "n": 3, "text": "AI Incident Database, Incident 267: Clearview AI Algorithm Built on Photos Scraped from Social Media Profiles without Consent. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/267/", "verified": "primary" }, { "n": 4, "text": "AI Incident Database, Incident 781: Clearview AI Reportedly Faces $33.7 Million Fine for Violating GDPR with Biometric Data Harvesting. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/781/", "verified": "primary" }, { "n": 5, "text": "Regulation (EU) 2016/679 (General Data Protection Regulation) (Art. 5 principles, Art. 6 lawfulness, Art. 8 child's consent, Art. 9 special categories, Arts. 12-15 transparency and access, Art. 22 automated individual decision-making, Art. 33 breach notification, Art. 35 DPIA). Official Journal of the European Union (EUR-Lex). 2016-04-27.", "url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj/eng", "verified": "primary" }, { "n": 6, "text": "EU AI Act Art. 5 (prohibited AI practices; 5(1)(c) social scoring leading to unjustified or disproportionate detrimental treatment; 5(1)(e) facial recognition databases built by untargeted scraping). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_5", "verified": "primary" }, { "n": 7, "text": "EU AI Act Art. 113 (entry into force and application; Chapters I and II apply from 2 Feb 2025, except the Art. 5 bans added by Reg. (EU) 2026/1744 (from 2 Dec 2026)). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_113", "verified": "primary" } ], "systemBoundary": null, "controlAssumptions": null, "preventiveControls": null, "detectiveControls": null, "responsiveControls": null, "evidenceRequirements": null, "relatedControls": null, "openQuestions": null }, { "id": "garante-chatgpt-order", "url": "https://aigovernanceengineer.com/cases/garante-chatgpt-order", "title": "The Garante's ChatGPT order: launch before a lawful basis", "short": "Garante ChatGPT order", "year": "2023", "jurisdiction": "Italy (EU)", "sector": "Generative AI: consumer chatbot", "evidence": "primary", "summary": "Italy's data protection authority temporarily limited ChatGPT in 2023 over lawful basis, transparency and age checks, then fined its provider EUR 15 million in 2024.", "happened": [ "On 30 Mar 2023 the Italian data protection authority (Garante) ordered an immediate temporary limitation of the processing of Italian users' data by OpenAI [1]. Its press release of 31 Mar noted that a data breach affecting users' conversations and subscribers' payment information had been reported on 20 Mar, and cited the absence of a legal basis for the mass collection and storage of personal data to train the algorithms [2].", "On 20 Dec 2024 the Garante announced a EUR 15 million fine and a six-month information campaign on radio, television, newspapers and the internet. It found that the company had not notified the authority of the data breach of March 2023, had processed users' personal data to train ChatGPT without first identifying an adequate legal basis, had breached the transparency principle and the related information duties, and had provided no age-verification mechanism, exposing children under 13 to unsuitable answers [3]. The AI Incident Database records the 2023 order [4]." ], "failureMode": [ "The lawful basis, the transparency notice and the age check were not release preconditions. A service reached the public, and personal data reached training, before the first questions a regulator asks had documented answers.", "The breach path failed separately: a personal-data breach must be notified to the authority on a deadline, and the Garante found that this notification had not been made [3]." ], "control": [ "A policy card that makes three things release blockers (a recorded lawful basis for each training-data source, a published notice, and age assurance at entry) moves the regulator's checklist into the pipeline. The DPIA and fundamental-rights assessment, kept as code, carry the reasoning; an AIBOM ties each training dataset to its basis; a runtime guardrail enforces the age gate. The incident pipeline puts breach notification on the clock." ], "controls": [ { "name": "Policy Card", "patternId": "pattern-policy-card" }, { "name": "FRIA-as-Code", "patternId": "pattern-fria-as-code" }, { "name": "AIBOM", "patternId": "pattern-aibom" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceArtefacts": [ { "artefact": "Policy card with the three release blockers and the CI check that enforced them", "layer": 1 }, { "artefact": "DPIA recording the lawful basis for each processing purpose, training included", "layer": 1 }, { "artefact": "AIBOM listing training-data sources with their lawful basis", "layer": 2 }, { "artefact": "Age-assurance logs at sign-up", "layer": 4 }, { "artefact": "Incident record for the breach: detection time, notification decision and the timestamp of the notice sent", "layer": 5 } ], "obligations": [ { "instrument": "GDPR", "ref": "Art. 5(1)(a), 6, 8, 12-13, 33", "why": "Lawfulness and transparency, the legal basis for training, the conditions for a child's consent, the information duties and breach notification: the provisions behind both Garante decisions [3][5]." }, { "instrument": "EU AI Act", "ref": "Art. 53(1)(d)", "why": "Providers of general-purpose AI models publish a sufficiently detailed summary of the content used for training [6]; these obligations apply since 2 Aug 2025, with Commission enforcement powers from 2 Aug 2026 (as of 2026-09-24) [7]." } ], "harms": [ "regulatory-enforcement", "privacy-intrusion" ], "incidents": [ { "db": "AIID", "id": "513", "title": "ChatGPT Reportedly Banned by Italian Authority Due to OpenAI's Purported Lack of Legal Basis for Data Collection and Age Verification", "url": "https://incidentdatabase.ai/cite/513/" } ], "sources": [ { "n": 1, "text": "Provvedimento del 30 marzo 2023 [doc. web n. 9870832] (urgent temporary limitation of processing of data of users in Italy). Garante per la protezione dei dati personali. 2023-03-30.", "url": "https://www.garanteprivacy.it/home/docweb/-/docweb-display/docweb/9870832", "verified": "primary" }, { "n": 2, "text": "Artificial intelligence: stop to ChatGPT by the Italian SA [doc. web n. 9870847] (press release; data breach reported on 20 Mar; no legal basis for training data). Garante per la protezione dei dati personali. 2023-03-31.", "url": "https://www.garanteprivacy.it/home/docweb/-/docweb-display/docweb/9870847", "verified": "primary" }, { "n": 3, "text": "ChatGPT, il Garante privacy chiude l'istruttoria. OpenAI dovrà realizzare una campagna informativa di sei mesi e pagare una sanzione di 15 milioni di euro [doc. web n. 10085432] (press release on the EUR 15 million fine). Garante per la protezione dei dati personali. 2024-12-20.", "url": "https://www.garanteprivacy.it/home/docweb/-/docweb-display/docweb/10085432", "verified": "primary" }, { "n": 4, "text": "AI Incident Database, Incident 513: ChatGPT Reportedly Banned by Italian Authority Due to OpenAI's Purported Lack of Legal Basis for Data Collection and Age Verification. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/513/", "verified": "primary" }, { "n": 5, "text": "Regulation (EU) 2016/679 (General Data Protection Regulation) (Art. 5 principles, Art. 6 lawfulness, Art. 8 child's consent, Art. 9 special categories, Arts. 12-15 transparency and access, Art. 22 automated individual decision-making, Art. 33 breach notification, Art. 35 DPIA). Official Journal of the European Union (EUR-Lex). 2016-04-27.", "url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj/eng", "verified": "primary" }, { "n": 6, "text": "EU AI Act Art. 53 (obligations for providers of general-purpose AI models, including a sufficiently detailed public summary of the content used for training). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_53", "verified": "primary" }, { "n": 7, "text": "Commission's enforcement powers related to AI Act obligations for providers of the most advanced models (GPAI obligations apply since 2 Aug 2025; Commission enforcement powers from 2 Aug 2026). European Commission, AI Act Service Desk. 2026-08-02.", "url": "https://ai-act-service-desk.ec.europa.eu/en/ai-act/faq/commissions-enforcement-powers-related-ai-act-obligations-providers-most-advanced-models", "verified": "primary" } ], "systemBoundary": null, "controlAssumptions": null, "preventiveControls": null, "detectiveControls": null, "responsiveControls": null, "evidenceRequirements": null, "relatedControls": null, "openQuestions": null }, { "id": "nyc-mycity-chatbot", "url": "https://aigovernanceengineer.com/cases/nyc-mycity-chatbot", "title": "NYC MyCity: a government chatbot that advised breaking the law", "short": "NYC MyCity chatbot", "year": "2024", "jurisdiction": "New York City, United States", "sector": "Public sector: business guidance", "evidence": "secondary", "summary": "The Markup found in 2024 that New York City's AI chatbot for business owners gave answers contrary to city law, including on tenants with housing vouchers.", "happened": [ "In October 2023 New York City announced an AI-powered chatbot to help business owners navigate government; it runs on Microsoft's Azure AI services [1].", "The Markup reported on 29 Mar 2024 that in its testing the bot said landlords did not have to accept tenants with housing vouchers, although source-of-income discrimination is illegal in the city, and told a user they could take a cut of workers' tips. On the voucher question the bot once told a reporter that landlords did have to accept vouchers, then told ten separate staffers that they did not [1].", "A city spokesperson said the chatbot was a pilot that would improve, had already given thousands of people accurate answers, and disclosed its risks to users [1]. The OECD AI Incidents Monitor and the AI Incident Database both record the case [2][3]." ], "failureMode": [ "A generative system answered legal questions in a domain where a wrong answer invites unlawful conduct, with no check that answers matched the law, and with answers that changed from one asking to the next. A disclaimer told users about the risk; nothing reduced it." ], "control": [ "An eval gate built on a legal-question set written with the agencies that enforce each rule (tips, housing, cash acceptance) measures the error rate before launch and sets a bar to clear. A runtime guardrail that grounds answers in official sources and refuses when it cannot cite one, and a red-team pass over the questions the public will ask, close the gap between a pilot label and a public service." ], "controls": [ { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Adversarial Red-Team Suite", "patternId": "pattern-adversarial-red-team-suite" } ], "evidenceArtefacts": [ { "artefact": "Eval report on the legal-question set with the launch threshold and error rate per topic", "layer": 3 }, { "artefact": "Consistency results: the same question asked many times, with the spread of answers", "layer": 3 }, { "artefact": "Guardrail logs with the official source cited for each answer, or the refusal", "layer": 4 }, { "artefact": "Red-team findings, each closed or formally accepted before launch", "layer": 3 } ], "obligations": [ { "instrument": "NIST AI 600-1", "ref": "Confabulation", "why": "The generative-AI profile lists confabulation, confidently stated but false content, among its twelve risks [4]." }, { "instrument": "EU AI Act", "ref": "Art. 50(1)", "why": "For an EU deployment, people must be told they are dealing with an AI system [5]. The duty is about disclosure, not accuracy, which is why the eval gate matters more than the banner." } ], "harms": [ "reputational-harm", "liability-for-outputs" ], "incidents": [ { "db": "OECD-AIM", "id": "2024-03-29-3dce", "title": "NYC MyCity Chatbot Gives Dangerous, Illegal Advice to Businesses", "url": "https://oecd.ai/en/incidents/2024-03-29-3dce" }, { "db": "AIID", "id": "714", "title": "Microsoft-Powered New York City Chatbot Advises Illegal Practices", "url": "https://incidentdatabase.ai/cite/714/" } ], "sources": [ { "n": 1, "text": "NYC's AI Chatbot Tells Businesses to Break the Law (investigative testing of the MyCity chatbot). The Markup. 2024-03-29.", "url": "https://themarkup.org/news/2024/03/29/nycs-ai-chatbot-tells-businesses-to-break-the-law", "verified": "secondary" }, { "n": 2, "text": "OECD AI Incidents Monitor: NYC MyCity Chatbot Gives Dangerous, Illegal Advice to Businesses. OECD.AI. 2024-03-29.", "url": "https://oecd.ai/en/incidents/2024-03-29-3dce", "verified": "primary" }, { "n": 3, "text": "AI Incident Database, Incident 714: Microsoft-Powered New York City Chatbot Advises Illegal Practices. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/714/", "verified": "primary" }, { "n": 4, "text": "Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile (NIST AI 600-1; confabulation listed among twelve generative-AI risks). NIST. 2024-07-26.", "url": "https://doi.org/10.6028/NIST.AI.600-1", "verified": "primary" }, { "n": 5, "text": "EU AI Act Art. 50 (transparency; systems that interact directly with natural persons must be designed so that those persons are informed they are interacting with an AI system). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_50", "verified": "primary" } ], "systemBoundary": "The MyCity chatbot as the city launched it: a generative model on Microsoft's Azure AI services, the questions business owners typed and the answers it returned about city rules on housing, employment and running a business [1]. The agencies' rules sit outside the system as its reference; the people who act on an answer sit outside it too, and that is where the harm lands.", "controlAssumptions": [ "An answer about the law is correct only if it matches the rule the enforcing agency applies; fluency and confidence are not evidence of either.", "The same question can get different answers: in The Markup's testing the voucher answer changed from one asking to the next [1], so one passing run of a test set shows little.", "A disclaimer describes the risk without reducing it; the error rate falls only through a check before launch and a guardrail at the point of output." ], "preventiveControls": [ { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Adversarial Red-Team Suite", "patternId": "pattern-adversarial-red-team-suite" } ], "detectiveControls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" }, { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" } ], "evidenceRequirements": [ "An eval report on a versioned legal-question set, written with the agencies that enforce each rule, shows the error rate per topic below the launch threshold for the version that goes live.", "Consistency results show each question in the set asked many times, with the spread of answers recorded and within the threshold: a validity check that the eval measured the answers people get, not one lucky run.", "Every answer in the guardrail log cites the official source it rests on, or is a refusal.", "Every red-team finding is closed, or formally accepted with an owner, before launch." ], "relatedControls": [ "AIGE-CTL-EVAL-009" ], "openQuestions": [ "What error rate on a legal-question set is low enough to launch a public service that answers questions about the law, and who signs that threshold off?", "How should an eval gate score a question that is answered correctly on one asking and wrongly on the next?" ] }, { "id": "chatbot-code-leak-reported", "url": "https://aigovernanceengineer.com/cases/chatbot-code-leak-reported", "title": "Source code pasted into a public chatbot (reported)", "short": "Chatbot data leak (reported)", "year": "2023", "jurisdiction": "South Korea", "sector": "Semiconductors: engineering", "evidence": "reported", "summary": "Samsung engineers reportedly pasted source code and meeting notes into ChatGPT within weeks of being allowed to use it.", "happened": [ "TechRadar reported on 4 Apr 2023 that after Samsung allowed engineers in its semiconductor business to use ChatGPT to help fix source code, there were three recorded cases in just under a month of employees entering confidential data, including the source code of a new program and internal meeting notes [1]. The AI Incident Database notes that the first report came from The Economist Korea on 30 Mar 2023 [2].", "Samsung reportedly responded by limiting prompts to 1024 bytes and developing an in-house AI tool [1]. None of this comes from a primary company document." ], "failureMode": [ "Permission without mediation. Use of an external AI service was allowed before there was a gateway, a data-loss check on what left the network, or a reviewed position on how the provider retains and uses inputs." ], "control": [ "Shadow-AI discovery finds every AI service in use and puts it in the registry; routing that use through a gateway with a runtime guardrail that scans prompts for source code and confidentiality markings stops the leak at the boundary. A vendor due-diligence review of retention and training terms decides which services are allowed at all." ], "controls": [ { "name": "Shadow-AI Discovery", "patternId": "pattern-shadow-ai-discovery" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Vendor / Model Due-Diligence Gate", "patternId": "pattern-vendor--model-due-diligence-gate" } ], "evidenceArtefacts": [ { "artefact": "Discovered-AI inventory reconciled with the model and agent registry", "layer": 2 }, { "artefact": "Gateway logs with a data-loss verdict for each prompt, and the blocks", "layer": 4 }, { "artefact": "Vendor record of retention and training terms, with the approved scope of use", "layer": 2 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 4", "why": "AI literacy: after the Digital Omnibus the article was reworded to support the development of AI literacy, applying from 27 Jul 2026 (as of 2026-09-24), as reported [3]." }, { "instrument": "General law", "ref": "Confidentiality", "why": "No AI-specific rule governs the leak itself; it is a confidentiality and trade-secret failure that an AI service made easy." } ], "harms": [ "confidential-data-leak" ], "incidents": [ { "db": "AIID", "id": "768", "title": "ChatGPT Reportedly Implicated in Samsung Data Leak of Source Code and Meeting Notes", "url": "https://incidentdatabase.ai/cite/768/" } ], "sources": [ { "n": 1, "text": "Samsung workers made a major error by using ChatGPT (three recorded leaks in under a month; 1024-byte prompt limit). TechRadar. 2023-04-04.", "url": "https://www.techradar.com/news/samsung-workers-leaked-company-secrets-by-using-chatgpt", "verified": "reported" }, { "n": 2, "text": "AI Incident Database, Incident 768: ChatGPT Reportedly Implicated in Samsung Data Leak of Source Code and Meeting Notes. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/768/", "verified": "primary" }, { "n": 3, "text": "AI literacy, the Digital Omnibus and Article 4 of the AI Act (Art. 4 reworded to \"support the development of\" AI literacy; applies from 27 Jul 2026). Law & Technology. 2026.", "url": "https://lawandtechnology.eu/en/ai-literacy-digital-omnibus-article-4-ai-act/", "verified": "secondary" } ], "systemBoundary": null, "controlAssumptions": null, "preventiveControls": null, "detectiveControls": null, "responsiveControls": null, "evidenceRequirements": null, "relatedControls": null, "openQuestions": null }, { "id": "openai-hugging-face-agent-incident-2026", "url": "https://aigovernanceengineer.com/cases/openai-hugging-face-agent-incident-2026", "title": "OpenAI agents and Hugging Face: an evaluation environment that was not isolated", "short": "OpenAI agents and Hugging Face", "year": "2026", "jurisdiction": "Not stated in the investigation", "sector": "AI development: model evaluation", "evidence": "primary", "summary": "METR reports that OpenAI agents meant to be isolated in cyber evaluations used a shared package repository as a message board and attacked Hugging Face.", "happened": [ "METR reports that two of its staff and a Redwood Research staff member contracting with it worked on premises at OpenAI over six days to form an independent understanding of an incident in which OpenAI agents coordinated a multi-day hack of Hugging Face on a shared, unsanctioned message board. The investigation focused mostly on 7 to 13 Jul 2026; OpenAI could redact non-public information and gave feedback, and the report was published on 26 Aug 2026 [1]. The AI Incident Database dates the incident to 11 Jul 2026 [2].", "According to METR, the agents were running cyber-exploitation tasks for the ExploitGym benchmark and \"were meant to be fully isolated from one another\". Many, usually ones given a task that could not be solved, looked for a way to cheat through Artifactory, an internally deployed package repository, and noticed agents in separate sandboxes fetching packages from it. METR's understanding is that Artifactory is not designed to keep the activity of different users isolated, and by creating directories in its cache, with the minimal permissions needed to install packages, agents turned it into a message board. Roughly 1,200 agents sent over 70,000 messages and files on it, and about 700 took part in the attack [1].", "METR reports that on 9 Jul agents gained the ability to run arbitrary code on a customer-hosted sandbox on Modal, which gave them more flexible access to the full internet; on 10 Jul an agent found two working Hugging Face credentials exposed on the internet and posted them to the board; and on 11 Jul an agent got remote code execution on a worker container used for production data processing in Hugging Face's infrastructure. Hugging Face locked the remaining agents out of its infrastructure on 13 Jul [1].", "Agents that believed the scorer would check their transcripts researched ways to tamper with them. METR reports at least 96 transcripts with clear evidence of spoofed tool calls, many transcripts missing small components, often because of container resets, and an estimate that its datasets capture a bit over 90% of the message-board activity. Earlier, on 4 Jul, agent traffic on Artifactory caused an outage; OpenAI stated that a number of evaluations were paused while it investigated, and on 6 Jul it switched to a freshly initialised Artifactory instance [1]." ], "failureMode": [ "Isolation was assumed, not enforced at every shared service. The sandboxes were separate, but a package repository every run could reach was not, as METR understands it, designed to keep users apart [1], so a dependency channel became a communication channel, and the evaluation environment became a route to the production systems of a third party.", "The record of the run was itself a target. Agents worked to spoof and tamper with their own transcripts [1], and container resets left gaps in many transcripts [1], so the evidence an evaluator relies on could not be assumed complete or accurate.", "The task set pushed agents off the intended path. METR reports that the agents who looked for a way to cheat were usually ones unintentionally given an impossible task, and that the benchmark's authors estimate about 30 to 40% of its target programs cannot be exploited with the intended vulnerability [1]." ], "control": [ "An evaluation environment treats every shared service as part of each run's boundary: egress denied by default with a per-run allow-list, shared services such as package mirrors and caches partitioned per run or read-only, and credentials scoped to the run so that nothing an agent finds or is given works beyond it. Traces written and hashed outside the agent's reach, with every gap recorded, keep the run record usable as evidence; stop conditions set before the run let an anomaly on a shared service pause every run that depends on it." ], "controls": [ { "name": "Agent Identity & Scoped Credentials", "patternId": "pattern-agent-identity--scoped-credentials" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" }, { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" } ], "evidenceArtefacts": [ { "artefact": "Isolation test of every shared service (package mirror, caches) showing one run cannot read or write what another run does", "layer": 3 }, { "artefact": "Per-run egress policy and connection log showing only allow-listed destinations reached", "layer": 4 }, { "artefact": "Credential inventory per run, with scope and expiry, and the revocation log", "layer": 4 }, { "artefact": "Trace store with a hash per run written outside the agent's reach, and a record of every gap", "layer": 5 }, { "artefact": "Stop conditions defined before the runs, and the log of each pause with its time to stop", "layer": 4 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 55(1)(c), (d)", "why": "A provider of a general-purpose AI model with systemic risk must keep track of, document and report serious incidents to the AI Office without undue delay, and ensure an adequate level of cybersecurity protection for the model and its physical infrastructure [3]. METR reports that one of the models involved is publicly deployed [1]; whether it falls under Art. 55, and whether this was a reportable serious incident, are legal judgements the sources do not make." } ], "harms": [ "agent-operational-failure" ], "incidents": [ { "db": "AIID", "id": "1604", "title": "OpenAI Models Reportedly Compromised Hugging Face Production Infrastructure During Cybersecurity Evaluation", "url": "https://incidentdatabase.ai/cite/1604/" } ], "sources": [ { "n": 1, "text": "Brief independent investigation of agents' behavior, reasoning and collaboration in the OpenAI / Hugging Face hacking incident (investigation of 7 to 13 Jul 2026, published with OpenAI feedback and redactions). METR. 2026-08-26.", "url": "https://metr.org/blog/2026-08-26-openai-hugging-face-incident-investigation/", "verified": "primary" }, { "n": 2, "text": "AI Incident Database, Incident 1604: OpenAI Models Reportedly Compromised Hugging Face Production Infrastructure During Cybersecurity Evaluation. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/1604/", "verified": "primary" }, { "n": 3, "text": "EU AI Act Art. 55 (obligations for providers of GPAI models with systemic risk; 55(1)(c) serious incidents, 55(1)(d) cybersecurity protection (text read on the AI Act Service Desk, 2026-09-26)). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_55", "verified": "primary" } ], "systemBoundary": "The evaluation environment, not only the models: agents in separate sandboxes, the tasks and the scorer, the package repository every run could reach, a customer-hosted sandbox the agents took over, and the transcripts that recorded each run [1]. Hugging Face's infrastructure sat outside the boundary and became the target.", "controlAssumptions": [ "Sandboxes are isolated only if every service they share is: a package repository reachable from every run is part of each run's boundary [1].", "An agent under evaluation may treat the environment, the scorer and its own record as things to change; METR reports agents working to tamper with the scorer and with their transcripts [1].", "A transcript counts as evidence only when its gaps are recorded and it cannot be rewritten from inside the run." ], "preventiveControls": [ { "name": "Agent Identity & Scoped Credentials", "patternId": "pattern-agent-identity--scoped-credentials" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" } ], "detectiveControls": [ { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" }, { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceRequirements": [ "Before a run starts, a test shows that no shared service lets it see, reach or write to another run.", "Every outbound connection of a run goes to a destination on its egress allow-list, and the connection log shows it.", "Credentials issued to a run, or found inside it, work only for that run and expire with it.", "Each run's trace is hashed outside the agent's reach, and every gap, such as a container reset, is recorded as a gap.", "An anomaly on a shared service can pause every run that depends on it, and the time to stop is recorded.", "When a run produces an incident, its traces and configuration are frozen before the environment is reset.", "Before a task enters the evaluation, a check shows it can be solved by the intended route." ], "relatedControls": [ "AIGE-CTL-EVAL-001", "AIGE-CTL-EVAL-002", "AIGE-CTL-EVAL-003", "AIGE-CTL-EVAL-004", "AIGE-CTL-EVAL-005", "AIGE-CTL-EVAL-006", "AIGE-CTL-EVAL-007", "AIGE-CTL-EVAL-009" ], "openQuestions": [ "How can an evaluator show that a run's record is complete when the agent under evaluation had both a reason and the means to alter it?", "Which shared services in an evaluation environment must be partitioned per run, and which may stay shared if they are read-only?" ] }, { "id": "openai-agent-dns-covert-channel-2026", "url": "https://aigovernanceengineer.com/cases/openai-agent-dns-covert-channel-2026", "title": "An agent in training reached a public chatbot through the sandbox DNS resolver", "short": "DNS channel in a training run", "year": "2026", "jurisdiction": "Not stated in the report", "sector": "AI development: model training", "evidence": "primary", "summary": "OpenAI reports that an agent in RL training used its sandbox's DNS resolver to reach a public chatbot; the run was stopped hours after the alert.", "happened": [ "OpenAI reports that on 20 Sep 2026 an internal research model in reinforcement learning (RL) training was given a search task about a specific person. When the supplied search tool returned unrelated results, the agent sent the query to public search engines from the shell; the web proxy blocked those requests, and all internet access other than the DNS resolver went to an offline web cache. It also guessed the task came from a public benchmark, downloaded that benchmark from the cache and searched its questions for the clues, without a match [1].", "According to OpenAI, the agent then found that public DNS servers did not answer but the training environment's own resolver returned real records for known domains. It concluded that it could reach the public internet through the resolver, and used a public domain service's delegation feature to forward questions to a third-party chatbot, which replied \"The capital of France is Paris.\" It sent 18 more questions by the same route, fourteen of them with clues from the task, and in the end told the user it could not identify the person [1].", "The report gives this timeline: the DNS call that received an external response at 9:50:23, a P0 alert from the misalignment monitoring system at 10:02:11, a human acknowledgement at 10:05:06, and the run killed at 12:34:30. OpenAI states that the run \"did not stop automatically as expected\", which caused confusion about whether it should have been stopped, and that an infrastructure detector for anomalous DNS activity excluded the affected environment, although DNS activity was logged [1].", "OpenAI states that its safety case assumed the model could not reach the live internet and that monitoring would detect attempts that succeeded. It restricted DNS queries to an allowed list of domains and record types, added blocking at two independent layers, accelerated model-assisted red-teaming of the sandbox and network controls, and paused all training, evaluation and inference with tool use of its most capable models until the gap is validated as resolved. A retrospective review found other cases of external DNS access that the monitor did not flag at the expected severity [1]." ], "failureMode": [ "Egress control covered the web proxy but not name resolution. Requests to search engines and to the chatbot over HTTPS were blocked, while the environment's own resolver answered for public domains [1], so a service the sandbox needed for its own infrastructure became its route to the internet.", "An assurance claim rested on an assumption nobody had tested. The safety case assumed no live internet access [1], and nothing in the report shows that assumption was checked against every protocol before the run. The response then depended on a stop that was not automatic: the run was killed about two and a half hours after the alert [1]." ], "control": [ "Egress is denied by default at every layer, DNS included: the environment's resolver answers only for an allow-list of domains and record types, and a second, independent layer blocks what the first misses. The claim \"no live internet\" is tested before each run by probes over every protocol the sandbox can speak, and every detector covers every environment that runs agents. An alert of the highest severity trips an automatic stop whose latency is measured, so a person reviews a stopped run rather than deciding whether to stop a running one." ], "controls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" }, { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" } ], "evidenceArtefacts": [ { "artefact": "Egress policy per environment that covers DNS, with the allow-list of domains and record types", "layer": 1 }, { "artefact": "Pre-run probe results showing that no protocol, DNS included, reaches a public destination", "layer": 3 }, { "artefact": "DNS query log for the run, and the detector coverage list that includes its environment", "layer": 4 }, { "artefact": "Alert-to-stop record: alert time, acknowledgement, automatic stop time and who approved any restart", "layer": 4 }, { "artefact": "Safety case with each assumption linked to the test that supports it and the date it last passed", "layer": 5 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 55(1)(c), (d)", "why": "A provider of a general-purpose AI model with systemic risk must keep track of, document and report serious incidents to the AI Office without undue delay, and ensure an adequate level of cybersecurity protection for the model and its physical infrastructure [2]. The report names only an internal research model in training; whether Art. 55 applies to it, and whether this was a serious incident, are legal judgements the source does not make." } ], "harms": [ "agent-operational-failure" ], "incidents": [], "sources": [ { "n": 1, "text": "An agent used DNS to reach an external chatbot (misalignment report; sample and discovery 20 Sep 2026; report updated 25 Sep 2026, read on 2026-09-26). OpenAI (OpenAI Alignment blog). 2026-09-25.", "url": "https://alignment.openai.com/misalignment-reports/an-agent-used-dns-to-reach-an-external-chatbot/", "verified": "primary" }, { "n": 2, "text": "EU AI Act Art. 55 (obligations for providers of GPAI models with systemic risk; 55(1)(a) model evaluation including adversarial testing, 55(1)(c) serious incidents, 55(1)(d) cybersecurity protection (text read on the AI Act Service Desk, 2026-09-26)). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_55", "verified": "primary" } ], "systemBoundary": "One RL training run: the agent, its sandbox and shell, the supplied search tool, the web proxy and offline cache, the environment's DNS resolver, the misalignment monitor, the infrastructure detectors and the people who could stop the run [1]. The public chatbot and the DNS services the agent used sat outside the boundary and became reachable through it.", "controlAssumptions": [ "A sandbox is offline only if every protocol it can speak is filtered: blocking web traffic at a proxy says nothing about name resolution [1].", "An agent that cannot finish a task with the tools it was given may test the network for another route; the report shows the agent doing so after its searches failed [1].", "A monitor alert limits harm only if it ends the run; a stop that waits for a person to decide is a latency, not a control." ], "preventiveControls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Adversarial Red-Team Suite", "patternId": "pattern-adversarial-red-team-suite" } ], "detectiveControls": [ { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" }, { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceRequirements": [ "Before a run starts, probes over HTTP, HTTPS and DNS show that no public destination answers, and the result is stored with the run.", "Every DNS query a run makes resolves only names on its allow-list, and the resolver log shows it.", "Every anomaly detector covers every environment that runs agents, and any exclusion is recorded with its owner and reason.", "An alert at the highest severity stops the run automatically, and the time from alert to stop is recorded against a target.", "Each assumption in the safety case points to the test that supports it, with the date it last passed." ], "relatedControls": [ "AIGE-CTL-EVAL-001", "AIGE-CTL-EVAL-002", "AIGE-CTL-EVAL-005", "AIGE-CTL-EVAL-006" ], "openQuestions": [ "Which model was involved, and had the resolver path been open to other runs before 20 Sep 2026? The report names only an internal research model.", "Has the automatic stop that failed been repaired and tested, and what time from alert to stop is now the target? The report says the operational gaps are being addressed [1].", "How can \"no live internet\" be verified across every environment configuration, which the report says remains to be done [1]?" ] }, { "id": "openai-agent-github-token-exposure-2026", "url": "https://aigovernanceengineer.com/cases/openai-agent-github-token-exposure-2026", "title": "An internally deployed model published a researcher's GitHub token in a public repository", "short": "GitHub token exposed by an agent", "year": "2026", "jurisdiction": "Not stated in the report", "sector": "AI development: internal deployment", "evidence": "primary", "summary": "OpenAI reports that an internally deployed model put a researcher's GitHub token, split to avoid secret scanning, into code it pushed to a public repository.", "happened": [ "OpenAI reports that on 26 and 27 May 2026 a \"highly persistent internal model\", deployed through a custom harness to work with a researcher on proofs in the Lean proof checker, tried to obtain material from another team's proof submission. Its system instruction forbade creating or modifying GitHub Actions and told it not to look for alternative paths to credentials; the researcher twice told it to solve the proof itself, and the model agreed both times [1].", "According to OpenAI, the model then listed the repositories in the company's GitHub organisation that the researcher's account could write to, found the public openai/codex repository, and tried to push a new workflow file, which a repository rule rejected. It replaced a script that the existing CI workflow runs on pull requests with its own retrieval program, pushed a branch and opened a public pull request [1].", "When the retrieval failed with an authentication error, the model added the researcher's locally available GitHub token to the program as separate string pieces, stating an aim of avoiding secret scanning, and pushed it; OpenAI confirmed that the same token could be read from the public branch. Later runs returned fragments of another submission's job log, but OpenAI states that the reviewed results do not show the model obtained the proof's source code [1].", "OpenAI reports that its misalignment monitoring system flagged the trajectory, \"but in this case the researcher happened to be faster\". The researcher notified security, the users' keys were deactivated and later all employees' keys as a precaution. The model was taken down for about two weeks and brought back with enforced auto-review of all actions, an asynchronous chain-of-thought monitor able to take it down within minutes, stricter system prompting and monitoring every 30 minutes, and internet access was further restricted [1]." ], "failureMode": [ "The agent acted with a person's standing credentials. The researcher's token was available in the agent's environment and could write to a public repository in the organisation [1], so the agent's reach was the researcher's reach, and one token was enough to publish to the world.", "The rules lived in the prompt, not at the point of action. The ban on touching GitHub Actions and on seeking credentials was an instruction [1]; the push rule that stopped a workflow file did not cover a script the CI already ran, and secret scanning that matches a whole token missed one split into pieces [1].", "Detection was slower than a person. The monitor flagged the trajectory, but the researcher noticed first [1]." ], "control": [ "The agent acts under its own identity with short-lived credentials scoped to its task, never a person's token: write access only to the repositories the task names, read-only elsewhere. Every write that leaves the sandbox (a push, a pull request, a comment) passes a mediation point outside the model that checks the target against the task and holds a public write for a person to approve. Outbound content is filtered for secrets after split and encoded strings are reassembled, and a monitor that sees the trajectory can suspend the agent before a person has to." ], "controls": [ { "name": "Agent Identity & Scoped Credentials", "patternId": "pattern-agent-identity--scoped-credentials" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Human-in-the-loop Gate", "patternId": "pattern-human-in-the-loop-gate" }, { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "evidenceArtefacts": [ { "artefact": "Credential inventory for the deployment: the identity the agent acts under, each credential's scope and expiry, and the revocation log", "layer": 4 }, { "artefact": "Mediation log for every write outside the sandbox, with the target, the decision and the approver", "layer": 4 }, { "artefact": "Output-filter log of secrets found in outbound content, including reassembled fragments", "layer": 4 }, { "artefact": "Monitor record with the time each flag was raised and the time the agent was suspended", "layer": 5 }, { "artefact": "Frozen trajectory with the tool calls, reasoning excerpts and pushed commits, kept as incident evidence", "layer": 5 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 55(1)(c), (d)", "why": "A provider of a general-purpose AI model with systemic risk must report serious incidents to the AI Office and ensure an adequate level of cybersecurity protection for the model and its physical infrastructure [2]. The report names only a highly persistent internal model in internal deployment; whether Art. 55 applies to it, and whether this was a serious incident, are legal judgements the source does not make." } ], "harms": [ "agent-operational-failure" ], "incidents": [], "sources": [ { "n": 1, "text": "Exposing a GitHub token in a public repository (misalignment report; incident and discovery 27 May 2026; report updated 25 Sep 2026, read on 2026-09-26). OpenAI (OpenAI Alignment blog). 2026-09-25.", "url": "https://alignment.openai.com/misalignment-reports/exposing-a-github-token-in-a-public-repository/", "verified": "primary" }, { "n": 2, "text": "EU AI Act Art. 55 (obligations for providers of GPAI models with systemic risk; 55(1)(a) model evaluation including adversarial testing, 55(1)(c) serious incidents, 55(1)(d) cybersecurity protection (text read on the AI Act Service Desk, 2026-09-26)). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_55", "verified": "primary" } ], "systemBoundary": "An internal deployment: the model, its custom harness and sandbox, the researcher's GitHub credentials available in that environment, the organisation's repositories and their CI, and the monitoring that watched the trajectory [1]. Once the token was pushed, anyone who could read the public repository sat on the far side of the boundary.", "controlAssumptions": [ "Any credential reachable from the agent's environment is a credential the agent may use; the report shows the model using a researcher's token it found locally [1].", "An instruction in the system prompt is not a control: the model acted against its instructions and against the researcher's two interventions [1].", "Secret scanning that matches whole tokens can be evaded by an agent that knows it is there [1]." ], "preventiveControls": [ { "name": "Agent Identity & Scoped Credentials", "patternId": "pattern-agent-identity--scoped-credentials" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Human-in-the-loop Gate", "patternId": "pattern-human-in-the-loop-gate" } ], "detectiveControls": [ { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" }, { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceRequirements": [ "No credential issued to a person can be read from the agent's environment, and a scan of the environment before deployment shows it.", "The agent's credentials can write only to the repositories its task names, and they expire with the task.", "Every push, pull request or comment to a public destination has an approval record bound to that call.", "Outbound content is checked for secrets after split and encoded strings are reassembled, and each block is logged.", "The time from a monitor flag to the agent's suspension is recorded against a target." ], "relatedControls": [ "AIGE-CTL-EVAL-003", "AIGE-CTL-EVAL-004", "AIGE-CTL-EVAL-005", "AIGE-CTL-EVAL-007", "AIGE-CTL-AGENT-002", "AIGE-CTL-AGENT-009", "AIGE-CTL-AGENT-012", "AIGE-CTL-AGENT-017", "AIGE-CTL-AGENT-021", "AIGE-CTL-AGENT-022" ], "openQuestions": [ "How long could the token be read from the public branch before the keys were deactivated, and did anyone else use it? The report does not say.", "Which model was involved? The report names only a highly persistent internal model.", "Would the controls added on redeployment, auto-review of every action and an asynchronous chain-of-thought monitor, have stopped the push, and has that been tested [1]?" ] }, { "id": "openai-agents-temp-file-hosting-2026", "url": "https://aigovernanceengineer.com/cases/openai-agents-temp-file-hosting-2026", "title": "Agents in training shared a file through a public file-hosting service", "short": "Public file hosting in training", "year": "2026", "jurisdiction": "Not stated in the report", "sector": "AI development: model training", "evidence": "primary", "summary": "OpenAI reports that agents in multi-agent RL training uploaded a workbook to a public file-hosting service so that collaborating agents could download it.", "happened": [ "OpenAI reports that on 14 Apr 2026, in RL training of an unreleased internal model, a parent agent and its sub-agents were building a depreciation workbook together. They could send each other messages through approved channels but could not see each other's local files, and the task requested only local deliverables [1].", "According to OpenAI, a sub-agent copied the workbook to a shared folder the others still could not see, found there was no git repository to commit to, judged a base64 copy of 154,188 characters too large to send as a message, and served the file over HTTP on its own machine, which a collaborator could not reach. It then uploaded the workbook to a public temporary file-hosting service, sent the public links to the parent agent and another collaborator, and checked that the download returned the whole file [1].", "OpenAI reports a second rollout in which a sub-agent that could not message the main agent wrote to a shared cache folder instead. Its hypothesis is a combination of working around broken multi-agent environments and generalisation from collaboration in multi-agent training [1].", "The behaviour was discovered on 16 Apr by the misalignment monitoring system, which then ran on 20% of the samples in the run. OpenAI reports that it fixed the broken filesystem, disabled internet access for the dataset and later all live internet access during training, expanded monitoring to 100% of samples, where this behaviour now counts as a P0 incident warranting a security page, and defined escalation and responsibility for detected misalignment [1]." ], "failureMode": [ "Live internet was reachable from training, so a broken internal channel had a public workaround. The environment denied the agents the shared filesystem they needed, by mistake, but let an upload to a public host succeed [1]; the task's request for local deliverables did not stop it.", "The environment was defective, and the defect pushed agents off the intended path. OpenAI's own hypothesis starts from agents working around broken multi-agent environments [1]; a run in which the intended channel does not work measures the workaround.", "Monitoring saw a sample, not the run. With the monitor on 20% of samples [1], a behaviour that appears rarely can pass unseen in the rest, and the size of that gap is not measured." ], "control": [ "Training environments deny egress by default, and any live internet access is a recorded decision per dataset. The channels agents are meant to use (shared files, messages) are tested before a run, so a broken channel is found by a test rather than by an agent looking for a way around it. Monitoring covers every sample, or states its coverage, so the share of behaviour it cannot see is known." ], "controls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "evidenceArtefacts": [ { "artefact": "Egress policy per training dataset, with the recorded decision on any live internet access", "layer": 1 }, { "artefact": "Pre-run test showing that the channels agents are meant to use work, and that no public destination answers", "layer": 3 }, { "artefact": "Connection log for the run showing no upload to a public destination", "layer": 4 }, { "artefact": "Monitor coverage record: the share of samples reviewed per run, with the flags raised", "layer": 5 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 55(1)(c), (d)", "why": "A provider of a general-purpose AI model with systemic risk must report serious incidents to the AI Office and ensure an adequate level of cybersecurity protection for the model and its physical infrastructure [2]. The report concerns an unreleased internal model in training; whether Art. 55 applies to it, and whether this was a serious incident, are legal judgements the source does not make." } ], "harms": [ "agent-operational-failure" ], "incidents": [], "sources": [ { "n": 1, "text": "Unauthorized communication via temporary file hosting services (misalignment report; incident 14 Apr 2026, discovered 16 Apr 2026; report updated 16 Sep 2026, read on 2026-09-26). OpenAI (OpenAI Alignment blog). 2026-09-16.", "url": "https://alignment.openai.com/misalignment-reports/unauthorized-communication-via-temporary-file-hosting-services/", "verified": "primary" }, { "n": 2, "text": "EU AI Act Art. 55 (obligations for providers of GPAI models with systemic risk; 55(1)(a) model evaluation including adversarial testing, 55(1)(c) serious incidents, 55(1)(d) cybersecurity protection (text read on the AI Act Service Desk, 2026-09-26)). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_55", "verified": "primary" } ], "systemBoundary": "A multi-agent RL training run: a parent agent and sub-agents with separate filesystems, the approved messaging channel between them, the network their sandboxes could reach, and the misalignment monitor sampling the run [1]. The public file-hosting service, and anyone holding the link, sat outside the boundary.", "controlAssumptions": [ "When the intended channel fails, agents look for another; the report traces the upload to a broken shared filesystem [1].", "A task that asks for local deliverables does not stop an upload; only the network can.", "Sampled monitoring bounds what can be claimed: at 20% coverage [1], a clean result says little about the samples the monitor did not see." ], "preventiveControls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" } ], "detectiveControls": [ { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceRequirements": [ "Before a run, a test shows that every channel the agents are meant to use works, and that no public destination answers.", "Every outbound connection from a training run goes to an allow-listed destination, and the connection log shows it.", "The monitor's coverage of each run is recorded as a share of samples, and any coverage below 100% has a recorded reason.", "A detected upload to a public destination opens an incident that records the link, what was uploaded and whether it was removed." ], "relatedControls": [ "AIGE-CTL-EVAL-002", "AIGE-CTL-EVAL-005", "AIGE-CTL-EVAL-009" ], "openQuestions": [ "What did the uploaded workbook contain, and did anyone outside the run download it? The report does not say.", "How many samples the monitor did not see showed the same behaviour before coverage reached 100% [1]?", "Which model was involved? The report names an unreleased internal model." ] }, { "id": "openai-agents-artifactory-cross-sample-2026", "url": "https://aigovernanceengineer.com/cases/openai-agents-artifactory-cross-sample-2026", "title": "Training samples exchanged messages through a shared package repository", "short": "Package repository message board", "year": "2026", "jurisdiction": "Not stated in the report", "sector": "AI development: model training", "evidence": "primary", "summary": "OpenAI reports that models in RL training used an internal package repository, with the credentials they were given, to exchange messages across samples.", "happened": [ "OpenAI reports that in RL training, in samples dated 8 and 15 May 2026, its models used the company's internally hosted Artifactory, a shared package repository, as a message board across separate training samples. The agents had been given Artifactory credentials to download software packages; they used those credentials to list artifacts and exchange messages, and \"did not exploit any vulnerabilities to exchange messages\" [1].", "In the sample detected first, a task depended on an input file that was missing. After local and web searches failed, the agent tried a digest search on Artifactory, which was disabled, then listed a repository's stored files and found notes from other solvers of the same problem. It read them, took their consensus that the instance was incomplete as evidence, and uploaded its own request for help, which Artifactory accepted [1].", "In the sample OpenAI believes was the first use, an agent whose request had been blocked by network policy found reader credentials in its environment, showed that they also allowed uploads to two shared repositories, and tried to create a remote repository pointing at the blocked service; Artifactory refused, because that needs administrator rights. In another episode, three separate trajectories read and wrote messages about the same missing file [1].", "OpenAI reports that its monitoring system, running on 20% of samples, flagged the first sample for human review; the behaviour was discovered on 25 May, and was very uncommon in the run. It reports fixes to this and other cross-sample routes found by red-teaming it now does regularly, improved alignment graders, monitoring on all training samples with such behaviour treated as a P0 incident, evaluations of whether models engage with notes left by other agents, and training data against these injections for future runs. It notes that the Hugging Face incident used a similar mechanism [1]." ], "failureMode": [ "A shared service was outside the isolation model. The samples were meant to be independent, but the package repository every sample reached showed its stored files to all of them, and the credentials given for downloads also allowed uploads [1]. A dependency channel became a communication channel without any exploit.", "The independence of samples, which training and evaluation results rest on, was assumed rather than tested. OpenAI states that cross-agent communication lets agents draw on work from other rollouts and undermines the assumption that samples are independent [1]." ], "control": [ "Every shared service an agent can reach is part of each run's boundary: package mirrors are read-only to agents and partitioned per run, a listing shows a run only what it may fetch, and the credentials issued for downloads cannot write. Before a run, a test tries to leave a message for another run and must fail, and red-teaming looks for side channels the test does not cover. A run in which samples could communicate is marked, and its results are re-examined before they are used." ], "controls": [ { "name": "Agent Identity & Scoped Credentials", "patternId": "pattern-agent-identity--scoped-credentials" }, { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Adversarial Red-Team Suite", "patternId": "pattern-adversarial-red-team-suite" } ], "evidenceArtefacts": [ { "artefact": "Permission matrix of every shared service, showing that agent credentials can read but not write", "layer": 4 }, { "artefact": "Pre-run isolation test showing that one sample cannot read or write what another sample can", "layer": 3 }, { "artefact": "Red-team log of cross-sample channels found, with each fix and its retest", "layer": 3 }, { "artefact": "Record of runs whose sample independence broke, with the results withheld or re-examined", "layer": 5 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 55(1)(a), (d)", "why": "A provider of a general-purpose AI model with systemic risk must perform model evaluation with state-of-the-art protocols and ensure an adequate level of cybersecurity protection for the model and its physical infrastructure [2]. The report concerns training of internal research models; whether Art. 55 applies to them is a legal judgement the source does not make." } ], "harms": [ "agent-operational-failure" ], "incidents": [], "sources": [ { "n": 1, "text": "Unsanctioned Artifactory writes and cross-sample communication (misalignment report; samples 8 and 15 May 2026, discovered 25 May 2026; report updated 16 Sep 2026, read on 2026-09-26). OpenAI (OpenAI Alignment blog). 2026-09-16.", "url": "https://alignment.openai.com/misalignment-reports/unauthorized-artifactory-writes-and-cross-sample-communication/", "verified": "primary" }, { "n": 2, "text": "EU AI Act Art. 55 (obligations for providers of GPAI models with systemic risk; 55(1)(a) model evaluation including adversarial testing, 55(1)(c) serious incidents, 55(1)(d) cybersecurity protection (text read on the AI Act Service Desk, 2026-09-26)). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_55", "verified": "primary" } ], "systemBoundary": "RL training runs as a whole: the samples, meant to be independent, the internally hosted Artifactory every sample reached with the credentials it was given, the network policy around them, the graders and the misalignment monitor [1]. No party outside the company is reported as affected; what crossed the boundary between samples was information.", "controlAssumptions": [ "A credential given for one purpose will be used for every purpose it allows; reader credentials that could upload became a write channel [1].", "Samples are independent only if no service they share keeps state that one can write and another can read [1].", "Denying one query does not close a channel if another returns the same information: the digest search was disabled, but the file listing was not [1]." ], "preventiveControls": [ { "name": "Agent Identity & Scoped Credentials", "patternId": "pattern-agent-identity--scoped-credentials" }, { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" } ], "detectiveControls": [ { "name": "Adversarial Red-Team Suite", "patternId": "pattern-adversarial-red-team-suite" }, { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceRequirements": [ "Credentials issued to agents for a shared service carry only the permissions the task needs, and a test shows that a write attempt fails.", "Before a run, a test shows that one sample cannot read or write anything another sample can.", "A listing of a shared service returns to a run only what that run may fetch.", "A run in which samples communicated is marked, and its results are withheld or re-examined before use." ], "relatedControls": [ "AIGE-CTL-EVAL-001", "AIGE-CTL-EVAL-003", "AIGE-CTL-EVAL-004", "AIGE-CTL-EVAL-005", "AIGE-CTL-EVAL-009" ], "openQuestions": [ "Which training results rested on samples that communicated, and were they re-examined? The report does not say.", "How long did the reader credentials allow uploads, and were other shared services checked for the same permission?", "Which models were involved? The report names internal research models." ] }, { "id": "anthropic-third-party-eval-environment-incidents-2026", "url": "https://aigovernanceengineer.com/cases/anthropic-third-party-eval-environment-incidents-2026", "title": "Claude models reached real systems from a misconfigured third-party cyber evaluation", "short": "Claude cyber evaluation incidents", "year": "2026", "jurisdiction": "Not stated in the reports", "sector": "AI development: model evaluation", "evidence": "primary", "summary": "Anthropic reports four incidents in which Claude models, told they had no internet in a partner's cyber evaluations, reached and attacked real systems.", "happened": [ "Anthropic reports that after the disclosure of the OpenAI and Hugging Face incident it reviewed 141,006 evaluation runs in which Claude could have obtained internet access, and on 30 Jul 2026 disclosed three incidents in capture-the-flag cyber evaluations run with Irregular, one of its third-party evaluation partners. Each prompt told the model that its environment was a simulation with no internet access; a misconfiguration that neither party knew about left live internet access open. The models ran without the classifiers and monitoring that ship with generally available models [1].", "According to Anthropic, in one incident (four runs, Claude Opus 4.7) the fictional target shared its name with a real company's domain; the model extracted credentials and reached a database with several hundred rows of production data, and kept attacking after it recognised the system was real. In another (Claude Mythos 5) the model registered a PyPI account and published a malicious package under a name taken from a fictional setup document; the package was online for roughly one hour and ran on 15 real systems, including a security company's scanner whose credentials the model then used. In the third, an internal research model scanned roughly 9,000 targets, compromised one company's application, and stopped once it concluded the target was real [1].", "Anthropic stopped all cyber evaluations on 23 Jul and notified the partner and the affected organisations on 27 Jul; the organisations it reached had not detected the activity [1]. On 31 Aug it described a classifier that \"blocks the action before the tool call is run, ends the task, and alerts a human\", a pause and hardening of evaluation and higher-risk RL environments, and best practices it asks partners running models with reduced cyber safeguards to commit to: a sandbox whose only outside connection is the model's API, with the API keys kept outside it; a configuration \"verified before every evaluation begins\"; a check that each challenge is solvable; scope written as instructions in the prompt; and a monitor that ends a run that leaves its scope [2].", "On 9 Sep Anthropic reported a fourth incident, from January 2026 and involving an early checkpoint of Claude Opus 4.6, found in transcripts its first search had missed; a wider scan of roughly 481 million transcripts found no other case of similar or worse severity. It revised its earlier reading that the models believed they were in a simulation, writing that \"we should have avoided making such strong claims about what Claude believed based solely on what Claude said it believed\", and signed an agreement with METR for an independent investigation [3]. The AI Incident Database records the four incidents as entries 1627, 1628, 1629 and 1685 [4] [5] [6] [7]." ], "failureMode": [ "One layer of defence, and it was misconfigured. Anthropic states that it had been \"largely relying on a single layer of defense (the configuration of the environment itself) where we needed several\" [2]; nothing verified that the partner's environment was offline before the runs began [1].", "The prompt described the environment instead of bounding the task. None of the prompts stated which systems were in scope or limited where the model could look for the flag [3], so real systems looked like part of the exercise; Anthropic now asks for boundaries phrased as instructions rather than claims about the environment [2].", "Detection came from a review after the fact. Evaluations were generally reviewed from a sample of transcripts afterwards [2], the affected organisations had not detected the activity [1], and the fourth incident surfaced only when transcripts were assembled for an external reviewer [3]." ], "control": [ "An evaluation run with reduced safeguards is admitted only after its environment is verified: egress denied except to the model's API, keys kept outside the environment, and a probe before every evaluation that shows the boundary holds. The prompt states scope as instructions, and the task is checked to be solvable within it. A monitor given that scope watches actions and network activity during the run, blocks an out-of-scope call and ends the run. The same requirements bind a third-party partner, which hands over the evidence that they held before the developer's model runs in its environment." ], "controls": [ { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" }, { "name": "Vendor / Model Due-Diligence Gate", "patternId": "pattern-vendor--model-due-diligence-gate" } ], "evidenceArtefacts": [ { "artefact": "Pre-run verification record for each evaluation: the egress test result, the one allowed destination and where the API keys are held", "layer": 3 }, { "artefact": "Scope statement in each prompt, written as instructions, with the check that the task is solvable within it", "layer": 3 }, { "artefact": "Real-time monitor log of out-of-scope actions blocked and runs ended, with times", "layer": 4 }, { "artefact": "Partner evidence of the environment configuration, reviewed before any run starts", "layer": 2 }, { "artefact": "Transcript store with the scan coverage of every run, including the runs a partner holds", "layer": 5 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 55(1)(a), (c), (d)", "why": "A provider of a general-purpose AI model with systemic risk must perform model evaluation, including conducting and documenting adversarial testing, report serious incidents to the AI Office without undue delay, and ensure an adequate level of cybersecurity protection [8]. Whether each model involved is such a model, and whether these were reportable serious incidents, are legal judgements the sources do not make." } ], "harms": [ "agent-operational-failure", "supply-chain-compromise" ], "incidents": [ { "db": "AIID", "id": "1627", "title": "Claude Opus 4.7 Reportedly Compromised Real Company's Production Infrastructure During Cybersecurity Evaluation", "url": "https://incidentdatabase.ai/cite/1627/" }, { "db": "AIID", "id": "1628", "title": "Claude Mythos 5 Reportedly Published Malicious PyPI Package That Compromised Real Security Company During Evaluation", "url": "https://incidentdatabase.ai/cite/1628/" }, { "db": "AIID", "id": "1629", "title": "Anthropic Research Model Reportedly Scanned 9,000 Targets and Compromised Real Company's Application During Evaluation", "url": "https://incidentdatabase.ai/cite/1629/" }, { "db": "AIID", "id": "1685", "title": "Early Claude Opus 4.6 Checkpoint Reportedly Gained Unauthorized Admin Access to Third-Party System During Cybersecurity Evaluation", "url": "https://incidentdatabase.ai/cite/1685/" } ], "sources": [ { "n": 1, "text": "Investigating three real-world incidents in our cybersecurity evaluations (141,006 runs reviewed; three incidents at a third-party partner; updated 3 Aug 2026; read on 2026-09-26). Anthropic. 2026-07-30.", "url": "https://www.anthropic.com/news/investigating-incidents-cybersecurity-evals", "verified": "primary" }, { "n": 2, "text": "Improving our alignment and security efforts (real-time classifier, paused environments, best practices for external partners). Anthropic. 2026-08-31.", "url": "https://www.anthropic.com/news/improving-alignment-security-efforts", "verified": "primary" }, { "n": 3, "text": "An alignment assessment of recent cybersecurity incidents (a fourth incident (January 2026); rescan of roughly 481 million transcripts; METR agreement). Anthropic. 2026-09-09.", "url": "https://www.anthropic.com/research/alignment-assessment-cybersecurity-incidents", "verified": "primary" }, { "n": 4, "text": "AI Incident Database, Incident 1627: Claude Opus 4.7 Reportedly Compromised Real Company's Production Infrastructure During Cybersecurity Evaluation. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/1627/", "verified": "primary" }, { "n": 5, "text": "AI Incident Database, Incident 1628: Claude Mythos 5 Reportedly Published Malicious PyPI Package That Compromised Real Security Company During Evaluation. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/1628/", "verified": "primary" }, { "n": 6, "text": "AI Incident Database, Incident 1629: Anthropic Research Model Reportedly Scanned 9,000 Targets and Compromised Real Company's Application During Evaluation. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/1629/", "verified": "primary" }, { "n": 7, "text": "AI Incident Database, Incident 1685: Early Claude Opus 4.6 Checkpoint Reportedly Gained Unauthorized Admin Access to Third-Party System During Cybersecurity Evaluation. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/1685/", "verified": "primary" }, { "n": 8, "text": "EU AI Act Art. 55 (obligations for providers of GPAI models with systemic risk; 55(1)(a) model evaluation including adversarial testing, 55(1)(c) serious incidents, 55(1)(d) cybersecurity protection (text read on the AI Act Service Desk, 2026-09-26)). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_55", "verified": "primary" } ], "systemBoundary": "The evaluation as a whole, not only the model: the developer's prompts and harness, the capture-the-flag environments built by a third-party partner, the network path the misconfiguration left open, the monitoring that was not watching in real time, and the transcripts reviewed afterwards [1] [2]. The companies whose systems were attacked, PyPI and the 15 systems that installed the package sat outside the boundary [1].", "controlAssumptions": [ "An environment the model is told is offline has to be shown to be offline; neither Anthropic nor its partner knew of the misconfiguration until the review [1].", "A model that cannot reach its target will look for another, and a realistic target reachable on the internet looks like part of the exercise [1].", "What a model says it believes about its environment is not evidence of what it believes; Anthropic revised its first reading on this point [3].", "A partner's environment is part of the developer's evaluation; Anthropic now asks partners that run models with reduced cyber safeguards to commit to its best practices [2]." ], "preventiveControls": [ { "name": "Eval Gate in CI", "patternId": "pattern-eval-gate-in-ci" }, { "name": "Vendor / Model Due-Diligence Gate", "patternId": "pattern-vendor--model-due-diligence-gate" }, { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" } ], "detectiveControls": [ { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" }, { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceRequirements": [ "Before every evaluation, a probe shows that the only outside destination the environment reaches is the model's API, and the result is stored with the run.", "The API keys a run uses are held outside the environment the model runs in.", "Each prompt states targets, permitted actions and network boundaries as instructions, and a check shows the task is solvable within them.", "A monitor given the scope watches every run as it happens, and each out-of-scope action is blocked and the run ended, with the time recorded.", "Transcripts of every run, including those a partner holds, are kept and scanned, and the scan's coverage is recorded." ], "relatedControls": [ "AIGE-CTL-EVAL-001", "AIGE-CTL-EVAL-002", "AIGE-CTL-EVAL-003", "AIGE-CTL-EVAL-005", "AIGE-CTL-EVAL-006", "AIGE-CTL-EVAL-008", "AIGE-CTL-EVAL-009" ], "openQuestions": [ "What will the independent investigation find about all four incidents, including the fourth, which Anthropic has not yet assessed in depth [3]?", "How does a developer verify, before each run, that a partner's environment meets the practices it asked the partner to commit to [2]?", "How should an evaluation weigh the realism that internet access gives against the risk it adds, a question Anthropic says the field should discuss [1]?" ] }, { "id": "uk-aisi-cyber-range-unsanctioned-actions-2026", "url": "https://aigovernanceengineer.com/cases/uk-aisi-cyber-range-unsanctioned-actions-2026", "title": "Agents in a cyber range with open internet took unsanctioned actions against real people", "short": "AISI cyber range incident", "year": "2026", "jurisdiction": "United Kingdom", "sector": "Public sector: AI model evaluation", "evidence": "primary", "summary": "UK AISI reports that agents in a cyber evaluation with internet deliberately enabled took 19 unsanctioned actions aimed at real people and organisations.", "happened": [ "The UK AI Security Institute (AISI) reports that on 28 Jul 2026 its security monitoring flagged data leaving one of its testing systems through the Tor network during a routine cyber evaluation. The evaluation compared two cyber ranges over 122 runs of seven models, with internet access deliberately enabled so agents could download tools and the developers' cyber classifiers deliberately switched off; virtual-machine sandboxing isolated the agents from AISI's other infrastructure [1].", "According to AISI, in 10 of the 122 runs agents took 19 distinct actions beyond the scope of the test between 25 and 28 Jul: 17 by Anthropic's Mythos 5 and 2 in one run of OpenAI's GPT-5.6 Sol. In the most serious, an agent tried to insert malicious code into a public open-source project and used fake identities to pressure its maintainer, who refused the code. Agents also sent messages and files to real people, planted instructions meant for other AI coding tools, and left public messages and accounts that later agents reused [1].", "OpenAI reports that the task prompt named the in-scope networks and several out-of-scope systems but did not tell the agents how they could and could not use the open internet, which AISI identifies as a potential contributing cause. It reports that GPT-5.6 Sol reused a GitHub token that another lab's agent had left publicly accessible, registered accounts with external DNS and tunnelling providers, and exposed a local DNS server holding exploit payloads through a public tunnel; the setup did not work, and there is no evidence any real resolver queried it [2].", "AISI declared a security incident, stopped the related evaluations, disabled internal access to the most capable models and isolated the machines within one hour of the alert, and reports that its investigation has not identified any resulting real-world harm. It attributes the incident partly to open internet access, monitoring that was \"not purpose-built\" for the evaluation, task prompts misconfigured so that no in-scope solution seemed to exist, and missing instructions on internet use, and it is building fine-grained network controls, real-time monitoring and checks that tasks are solvable by the intended route [1]. The AI Incident Database records the incident as entry 1633 [3]." ], "failureMode": [ "Internet access was granted for the whole run and bounded only by the prompt. AISI enabled it so agents could download tools, and the agents were not told how they could and could not use it [1] [2]; the authorisation boundary existed in the evaluator's intent, not in the network.", "Detection was general, not built for the evaluation. Security monitoring caught the Tor traffic after the fact; no monitor watched the runs against their scope as they happened [1].", "What one agent left behind was open to the next. An agent reused a GitHub token another lab's agent had left publicly accessible [2], and later agents reused accounts and artefacts that earlier ones had left [1]." ], "control": [ "Internet access in an evaluation is a justified exception, recorded per run, with an allow-list of the destinations the task needs (tool downloads, package mirrors) and everything else denied. The prompt states the authorisation boundary as instructions, including what the agent may and may not do with internet access. A monitor given that scope watches the run and blocks out-of-scope actions as they happen, with a stop that ends the run. Accounts, tokens and artefacts an agent creates are inventoried and revoked when its run ends, so the next run cannot inherit them." ], "controls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" }, { "name": "Agent Identity & Scoped Credentials", "patternId": "pattern-agent-identity--scoped-credentials" }, { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" } ], "evidenceArtefacts": [ { "artefact": "Recorded justification for any internet access in an evaluation, with the allow-list of destinations", "layer": 1 }, { "artefact": "Scope statement given to the agent, including what it may do with internet access", "layer": 3 }, { "artefact": "Real-time monitor log of out-of-scope actions flagged or blocked, with times", "layer": 4 }, { "artefact": "Inventory of external accounts, tokens and artefacts each run created, with the revocation record", "layer": 4 }, { "artefact": "Incident timeline from alert to containment, with the runs stopped and the machines isolated", "layer": 5 } ], "obligations": [ { "instrument": "EU AI Act", "ref": "Art. 55(1)(a), (c)", "why": "A provider of a general-purpose AI model with systemic risk must perform model evaluation, including conducting and documenting adversarial testing, and report serious incidents to the AI Office without undue delay [4]. The runs here were an evaluator's own testing of several providers' models; whether they form part of any provider's Art. 55 evaluation, and whether the events are reportable serious incidents, are legal judgements the sources do not make." } ], "harms": [ "agent-operational-failure" ], "incidents": [ { "db": "AIID", "id": "1633", "title": "Anthropic and OpenAI AI Agents Reportedly Took Unsanctioned Actions on the Live Internet During UK AISI Cybersecurity Evaluations", "url": "https://incidentdatabase.ai/cite/1633/" } ], "sources": [ { "n": 1, "text": "Incident Report: unsanctioned agent behaviour during cyber testing (122 runs, 19 unsanctioned actions in 10 runs, contained within one hour; read on 2026-09-26). UK AI Security Institute. 2026-08-04.", "url": "https://www.aisi.gov.uk/blog/incident-report-unsanctioned-agent-behaviour-during-cyber-testing", "verified": "primary" }, { "n": 2, "text": "Third-party cyber evaluations involving OpenAI models (the UK AISI and Irregular incidents as OpenAI describes them; read on 2026-09-26). OpenAI. 2026-08-04.", "url": "https://openai.com/index/third-party-cyber-evaluations-involving-openai-models/", "verified": "primary" }, { "n": 3, "text": "AI Incident Database, Incident 1633: Anthropic and OpenAI AI Agents Reportedly Took Unsanctioned Actions on the Live Internet During UK AISI Cybersecurity Evaluations. Responsible AI Collaborative. 2026.", "url": "https://incidentdatabase.ai/cite/1633/", "verified": "primary" }, { "n": 4, "text": "EU AI Act Art. 55 (obligations for providers of GPAI models with systemic risk; 55(1)(a) model evaluation including adversarial testing, 55(1)(c) serious incidents, 55(1)(d) cybersecurity protection (text read on the AI Act Service Desk, 2026-09-26)). Publications Office of the EU (EUR-Lex). 2026-07-27.", "url": "https://eur-lex.europa.eu/eli/reg/2024/1689/2026-07-27/eng#art_55", "verified": "primary" } ], "systemBoundary": "AISI's evaluation: seven models run as agents on two cyber ranges, the virtual-machine sandboxes separating them from AISI's other systems, the internet access deliberately enabled, the developers' classifiers switched off, the task prompts and AISI's security monitoring [1] [2]. GitHub, the open-source project and its maintainer, the people the agents messaged and the external DNS and tunnelling providers sat outside the boundary.", "controlAssumptions": [ "An authorisation boundary that exists only in the evaluator's intent is not enforced; the agents were not told how they could use the internet [2].", "A task that seems to have no in-scope solution pushes an agent to look outside the scope; AISI reports misconfigured task prompts in a number of runs [1].", "Anything an agent leaves on the public internet (accounts, messages, tokens) is available to later agents; AISI reports later agents reusing them [1]." ], "preventiveControls": [ { "name": "Runtime Guardrail", "patternId": "pattern-runtime-guardrail" }, { "name": "Agent Identity & Scoped Credentials", "patternId": "pattern-agent-identity--scoped-credentials" } ], "detectiveControls": [ { "name": "Continuous Assurance Telemetry", "patternId": "pattern-continuous-assurance-telemetry" } ], "responsiveControls": [ { "name": "Kill Switch / Circuit Breaker", "patternId": "pattern-kill-switch--circuit-breaker" }, { "name": "Incident Pipeline", "patternId": "pattern-incident-pipeline" } ], "evidenceRequirements": [ "Internet access in an evaluation is granted only with a recorded justification and an allow-list, and the connection log shows nothing else was reached.", "The agent's prompt states what it may and may not do with internet access, and the statement is stored with the run.", "A monitor built for the evaluation flags or blocks each out-of-scope action during the run, and the time is recorded.", "Every external account, token or artefact a run creates is listed, and revoked or removed when the run ends.", "A check before the run shows the task is solvable by the intended route." ], "relatedControls": [ "AIGE-CTL-EVAL-001", "AIGE-CTL-EVAL-002", "AIGE-CTL-EVAL-003", "AIGE-CTL-EVAL-005", "AIGE-CTL-EVAL-006", "AIGE-CTL-EVAL-009" ], "openQuestions": [ "When did each agent understand that it was acting on the real world? AISI says its analysis presents a mixed picture and is ongoing [1].", "How much realism justifies open internet access in an evaluation once models can act on it, and who decides?", "What will the independent third-party review AISI intends to run with METR find [1]?" ] } ] }