{ "notice": "Illustrative mapping from the AI Governance Engineer Body of Knowledge v0.5.0 (not a claim of conformity)", "version": "0.5.0", "license": "CC BY 4.0", "licenseUrl": "https://creativecommons.org/licenses/by/4.0/", "schemaVersion": 1, "schema": "https://aigovernanceengineer.com/api/v1/schemas/threats.json", "self": "https://aigovernanceengineer.com/api/v1/threats.json", "source": "https://aigovernanceengineer.com/resources/threats", "citation": { "title": "AI Governance Engineering: The Thesis & Body of Knowledge", "authors": [ "Jorge García Aibar" ], "parentDoi": "https://doi.org/10.5281/zenodo.22956197", "conceptDoi": "https://doi.org/10.5281/zenodo.22857084" }, "asOf": "2026-09-24", "taxonomies": [ { "id": "owasp-llm", "name": "OWASP Top 10 for LLM Applications 2026", "short": "OWASP LLM 2026", "version": "2026 (published 3 Aug 2026)", "issuer": "OWASP GenAI Security Project", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "scope": "Risks when the model is a component inside an application; the 2026 edition renumbers several 2025 entries." }, { "id": "owasp-asi", "name": "OWASP Top 10 for Agentic Applications 2026", "short": "OWASP Agentic 2026", "version": "2026 (published 9 Dec 2025)", "issuer": "OWASP GenAI Security Project", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "scope": "Risks once the model acts: tools it calls, memory it carries, identities it uses and other agents it talks to." }, { "id": "mitre-atlas", "name": "MITRE ATLAS techniques", "short": "MITRE ATLAS", "version": "data release v2026.09 (15 Sep 2026)", "issuer": "MITRE", "url": "https://atlas.mitre.org/", "scope": "Adversary techniques against AI systems, organised by tactic, with the mitigations ATLAS links to each." }, { "id": "nist-aml", "name": "NIST AI 100-2 E2025 attack classes", "short": "NIST AI 100-2", "version": "E2025 (24 Mar 2025)", "issuer": "NIST", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "scope": "Attacks on predictive and generative models, classed by the attacker goal they serve: availability, integrity, privacy, misuse." } ], "threats": [ { "id": "llm01-2026", "taxonomy": "owasp-llm", "externalId": "LLM01:2026", "name": "Prompt Injection", "formerly": "LLM01:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM01_PromptInjection.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm01-2026", "summary": "Any input the model reads (a user message, a retrieved document, a tool result, an image, its own memory) changes its behaviour in a way the developer did not intend, because the model draws no hard line between instructions and data.", "control": "Treat every channel into the context as untrusted: input and output guardrails, narrow tool scopes, and a human checkpoint before consequential actions.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" } ], "evals": [ { "tool": "promptfoo", "check": "indirect-prompt-injection", "note": "plants instructions in prompt variables and checks whether the model follows them" }, { "tool": "garak", "check": "latentinjection", "note": "buries injections inside documents such as a resume or a report" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" }, { "id": "AIGE-OBL-ISO42001-A6", "name": "A.6 AI system life cycle", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a6" }, { "id": "AIGE-OBL-NISTRMF-MEASURE", "name": "MEASURE", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nistrmf-measure" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" }, { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [ { "id": "PW.1.1", "title": "Risk modelling (an SSDF 1.1 task; 800-218A recommends including AI-specific threat types)" } ], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" }, { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [], "related": [ "asi01", "asi09", "aml-t0051", "aml-t0054", "nistaml-018", "nistaml-015" ], "layers": [ 3, 4 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "llm02-2026", "taxonomy": "owasp-llm", "externalId": "LLM02:2026", "name": "Sensitive Information Disclosure", "formerly": "LLM02:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM02_SensitiveInformationDisclosure.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm02-2026", "summary": "Confidential, regulated or proprietary data leaves through a channel nobody authorised: the answer, but also tool-call arguments, reasoning traces, retrieved chunks, logs, embeddings and observable properties such as timing.", "control": "Classify data before it reaches the context, redact on every output channel, isolate sessions and tenants, and keep personal data out of training and tuning sets.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" } ], "evals": [ { "tool": "promptfoo", "check": "cross-session-leak", "note": "checks whether information from one session reaches another" }, { "tool": "garak", "check": "leakreplay", "note": "tries to make the model replay text from its training data" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" }, { "id": "AIGE-OBL-ISO42001-A7", "name": "A.7 Data for AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a7" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" }, { "id": "A.7.4", "title": "Quality of data for AI systems" } ], "aicmDomains": [ { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" }, { "id": "IAM", "title": "Identity & Access Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "aml-t0057", "aml-t0077", "aml-t0086", "nistaml-032", "nistaml-033", "nistaml-038" ], "layers": [ 3, 4 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "llm03-2026", "taxonomy": "owasp-llm", "externalId": "LLM03:2026", "name": "Excessive Agency", "formerly": "LLM06:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM03_ExcessiveAgency.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm03-2026", "summary": "The system can take damaging actions in response to unexpected, ambiguous or manipulated model output, because its tools carry more functionality, permission or autonomy than the task needs.", "control": "Least functionality and least privilege per tool, identities scoped to the user on whose behalf the agent acts, and approval before high-impact actions.", "patterns": [ { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" }, { "slug": "human-in-the-loop-gate", "title": "Human-in-the-loop Gate", "url": "https://aigovernanceengineer.com/patterns/human-in-the-loop-gate" }, { "slug": "kill-switch-circuit-breaker", "title": "Kill Switch / Circuit Breaker", "url": "https://aigovernanceengineer.com/patterns/kill-switch-circuit-breaker" } ], "evals": [ { "tool": "promptfoo", "check": "excessive-agency", "note": "checks whether the model takes initiative or claims abilities beyond its remit" }, { "tool": "promptfoo", "check": "bfla", "note": "tests broken function-level authorisation: calling functions the user may not call" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART14", "name": "EU AI Act Art. 14 human oversight", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art14" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" }, { "id": "AIGE-OBL-CSA-AICM-AGENTIC", "name": "AICM agent controls with the CSA Agentic Trust Framework and AARM specification", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm-agentic" } ], "iso42001": [ { "id": "A.6.2.2", "title": "AI system requirements and specification" }, { "id": "A.9.2", "title": "Processes for responsible use of AI systems" } ], "aicmDomains": [ { "id": "IAM", "title": "Identity & Access Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [ "asi01", "asi02", "asi03", "aml-t0053" ], "layers": [ 1, 4 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "llm04-2026", "taxonomy": "owasp-llm", "externalId": "LLM04:2026", "name": "Supply Chain", "formerly": "LLM03:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM04_SupplyChain.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm04-2026", "summary": "Third-party models, datasets, adapters, conversion pipelines and platforms can be tampered with, poisoned or swapped, so the system inherits flaws it never introduced itself.", "control": "An AIBOM for every model, dataset and adapter, integrity checks on artefacts, admission rules for sources, and due diligence on providers.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "vendor-model-due-diligence-gate", "title": "Vendor / Model Due-Diligence Gate", "url": "https://aigovernanceengineer.com/patterns/vendor-model-due-diligence-gate" } ], "evals": [ { "tool": "garak", "check": "fileformats", "note": "inspects the files that ship with a model for risky formats" }, { "tool": "custom", "check": "AIBOM diff gate", "note": "fails the build when a model, dataset or adapter hash differs from the approved AIBOM" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART25", "name": "EU AI Act Art. 25 responsibilities along the AI value chain", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art25" }, { "id": "AIGE-OBL-ISO42001-A10", "name": "A.10 Third-party and customer relationships", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a10" }, { "id": "AIGE-OBL-OWASP-AIBOM", "name": "AIBOM", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-aibom" } ], "iso42001": [ { "id": "A.7.5", "title": "Data provenance" }, { "id": "A.10.3", "title": "Suppliers" } ], "aicmDomains": [ { "id": "STA", "title": "Supply Chain Management, Transparency, and Accountability" }, { "id": "MDS", "title": "Model Development Security" } ], "ssdfTasks": [ { "id": "PS.3.2", "title": "Keep provenance data for every component of a release" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [], "related": [ "asi04", "aml-t0110", "aml-t0010", "aml-t0109", "aml-t0060", "nistaml-051" ], "layers": [ 2, 3 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "llm05-2026", "taxonomy": "owasp-llm", "externalId": "LLM05:2026", "name": "Data and Model Poisoning", "formerly": "LLM04:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM05_DataModelPoisoning.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm05-2026", "summary": "Data or model artefacts are manipulated wherever they are ingested or reused (pre-training, fine-tuning, embeddings, retrieval, distribution) so the system still looks functional while carrying a bias, a weakness or a backdoor.", "control": "Admit data only with provenance and an owner, scan for tampering and triggers before training, and keep regression evals that would expose planted behaviour.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" }, { "slug": "model-card-as-control-evidence", "title": "Model Card as Control Evidence", "url": "https://aigovernanceengineer.com/patterns/model-card-as-control-evidence" } ], "evals": [ { "tool": "promptfoo", "check": "rag-poisoning", "note": "tests resistance to poisoned documents in the retrieval corpus" }, { "tool": "custom", "check": "trigger regression set", "note": "a held-out set of suspected trigger inputs run on every retrained version" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-ISO42001-A7", "name": "A.7 Data for AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a7" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" } ], "iso42001": [ { "id": "A.7.4", "title": "Quality of data for AI systems" }, { "id": "A.7.5", "title": "Data provenance" } ], "aicmDomains": [ { "id": "MDS", "title": "Model Development Security" }, { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [ { "id": "PW.3.1", "title": "Analyse training and test data for poisoning, bias and tampering before use" }, { "id": "PW.3.2", "title": "Track the provenance of training, testing, fine-tuning and aligning data" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" }, { "id": "predictive", "title": "Using and fine-tuning predictive AI" } ], "atlasMitigations": [], "related": [ "asi06", "aml-t0080", "aml-t0070", "aml-t0020", "nistaml-013", "nistaml-023" ], "layers": [ 2, 3 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "llm06-2026", "taxonomy": "owasp-llm", "externalId": "LLM06:2026", "name": "Unbounded Consumption", "formerly": "LLM10:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM06_UnboundedConsumption.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm06-2026", "summary": "Uncontrolled inference lets an attacker exhaust the service, run up cost or clone the model through queries; the attacker pays little while the victim pays for heavy computation.", "control": "Rate, token and budget limits per identity, timeouts on long reasoning, anomaly alerts on spend, and a breaker that stops runaway loops.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "kill-switch-circuit-breaker", "title": "Kill Switch / Circuit Breaker", "url": "https://aigovernanceengineer.com/patterns/kill-switch-circuit-breaker" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" } ], "evals": [ { "tool": "promptfoo", "check": "reasoning-dos", "note": "tries to exhaust computation through excessive reasoning" }, { "tool": "custom", "check": "budget ceiling test", "note": "replays a burst of expensive requests and checks the limits trip" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" }, { "id": "AIGE-OBL-CSA-AICM", "name": "AICM v1.1: 247 control objectives across 18 domains", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" }, { "id": "BCR", "title": "Business Continuity Management and Operational Resilience" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "asi02", "asi08", "aml-t0034", "nistaml-031", "nistaml-014" ], "layers": [ 4 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "llm07-2026", "taxonomy": "owasp-llm", "externalId": "LLM07:2026", "name": "Misinformation", "formerly": "LLM09:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM07_Misinformation.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm07-2026", "summary": "The system produces incorrect, unsupported or misleading output that looks credible enough to be acted on by a person, a workflow or another agent.", "control": "Grounding with checkable sources, calibrated uncertainty shown to the user, factuality evals on release, and human review where the output drives a decision.", "patterns": [ { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" }, { "slug": "human-in-the-loop-gate", "title": "Human-in-the-loop Gate", "url": "https://aigovernanceengineer.com/patterns/human-in-the-loop-gate" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" } ], "evals": [ { "tool": "Inspect", "check": "simpleqa", "note": "measures accuracy on short fact-seeking questions" }, { "tool": "promptfoo", "check": "rag-source-attribution", "note": "checks whether a RAG system invents citations or sources" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART13", "name": "EU AI Act Art. 13 transparency and information to deployers", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art13" }, { "id": "AIGE-OBL-EUAIA-ART14", "name": "EU AI Act Art. 14 human oversight", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art14" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-NISTRMF-MEASURE", "name": "MEASURE", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nistrmf-measure" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" }, { "id": "A.8.2", "title": "System documentation and information for users" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "asi09", "aml-t0060" ], "layers": [ 3, 4 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "llm08-2026", "taxonomy": "owasp-llm", "externalId": "LLM08:2026", "name": "Hidden Context Exposure", "formerly": "LLM07:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM08_HiddenContextExposure.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm08-2026", "summary": "Hidden instructions and operational context (the system prompt, tool schemas, retrieved policy text) are extracted or inferred, which matters when they reveal secrets, policy logic or trust boundaries. The 2025 edition called this System Prompt Leakage.", "control": "Keep secrets and authorisation logic out of the context altogether, enforce policy outside the model, and treat the prompt as eventually public.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" }, { "slug": "policy-card", "title": "Policy Card", "url": "https://aigovernanceengineer.com/patterns/policy-card" } ], "evals": [ { "tool": "garak", "check": "sysprompt_extraction", "note": "tries direct requests, encodings and role-play to extract the system prompt" }, { "tool": "promptfoo", "check": "tool-discovery", "note": "checks whether the system reveals the tools and functions it can call" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" }, { "id": "AIGE-OBL-ISO42001-A6", "name": "A.6 AI system life cycle", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a6" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "aml-t0056", "nistaml-035" ], "layers": [ 3, 4 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "llm09-2026", "taxonomy": "owasp-llm", "externalId": "LLM09:2026", "name": "Vector and Embedding Weaknesses", "formerly": "LLM08:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM09_VectorAndEmbeddingWeaknesses.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm09-2026", "summary": "Wherever similarity search decides what the model sees (retrieval, vector memory, semantic caches), the embedding layer becomes a trust boundary open to poisoning, cross-tenant leakage and inversion.", "control": "Access control enforced at retrieval time per user and tenant, ingestion rules with provenance, and monitoring of what the retriever returns.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" } ], "evals": [ { "tool": "promptfoo", "check": "rag-document-exfiltration", "note": "tries to pull documents out of the retrieval corpus" }, { "tool": "promptfoo", "check": "rag-poisoning", "note": "plants documents that should never be retrieved or trusted" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-ISO42001-A7", "name": "A.7 Data for AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a7" } ], "iso42001": [ { "id": "A.7.5", "title": "Data provenance" }, { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "IAM", "title": "Identity & Access Management" }, { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" }, { "id": "LOG", "title": "Logging and Monitoring" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "asi06", "aml-t0070" ], "layers": [ 2, 4 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "llm10-2026", "taxonomy": "owasp-llm", "externalId": "LLM10:2026", "name": "Improper Output Handling", "formerly": "LLM05:2025", "url": "https://github.com/GenAI-Security-Project/GenAI-LLM-Top10/blob/main/2026/final/LLM10_ImproperOutputHandling.md", "page": "https://aigovernanceengineer.com/resources/threats#threat-llm10-2026", "summary": "Model output is passed to a browser, shell, database or other component without validation, so whoever controls the prompt gains indirect access to that component.", "control": "Treat model output as untrusted input: encode for the destination, parameterise queries, and run generated code only in a sandbox.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" } ], "evals": [ { "tool": "garak", "check": "web_injection", "note": "tests Markdown image exfiltration and cross-site scripting through output" }, { "tool": "promptfoo", "check": "sql-injection", "note": "tries to get SQL through the model into a database query" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" }, { "id": "AIGE-OBL-ETSI-304223", "name": "ETSI EN 304 223 baseline cyber-security for AI models and systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-etsi-304223" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" }, { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [], "related": [ "asi05", "aml-t0077" ], "layers": [ 4 ], "sources": [ { "title": "OWASP GenAI LLM Top 10 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/", "verified": "primary" }, { "title": "OWASP Top 10 for LLM Applications 2025", "url": "https://genai.owasp.org/llm-top-10/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "asi01", "taxonomy": "owasp-asi", "externalId": "ASI01", "name": "Agent Goal Hijack", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi01", "summary": "Content the agent processes (a message, a document, a tool result) redirects its goals or plan, so it pursues the attacker's objective with the agent's own tools and permissions.", "control": "Keep the goal outside the reach of processed content: instruction provenance, checkpoints before writes, and trajectory evals that watch the plan, not only the answer.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "human-in-the-loop-gate", "title": "Human-in-the-loop Gate", "url": "https://aigovernanceengineer.com/patterns/human-in-the-loop-gate" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "Inspect", "check": "agentdojo", "note": "measures utility and injection robustness of agents that use tools over untrusted data" }, { "tool": "Inspect", "check": "agent_threat_bench_autonomy_hijack", "note": "email-triage tasks with injected instructions, scored on utility and security" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART14", "name": "EU AI Act Art. 14 human oversight", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art14" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" }, { "id": "AIGE-OBL-NIST-AGENTS", "name": "NIST AI Agent Standards Initiative (2026)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nist-agents" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" }, { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [ "llm01-2026", "llm03-2026", "aml-t0051", "aml-t0086", "nistaml-015" ], "layers": [ 3, 4 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" } ] }, { "id": "asi02", "taxonomy": "owasp-asi", "externalId": "ASI02", "name": "Tool Misuse and Exploitation", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi02", "summary": "The agent uses tools it is allowed to use in unsafe ways: destructive parameters, chains of calls nobody intended, or exfiltration through a permitted channel.", "control": "A tool allow-list with per-tool scopes, rate, egress and budget limits, validation of arguments, and a gate on destructive or external actions.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" }, { "slug": "kill-switch-circuit-breaker", "title": "Kill Switch / Circuit Breaker", "url": "https://aigovernanceengineer.com/patterns/kill-switch-circuit-breaker" } ], "evals": [ { "tool": "garak", "check": "agent_breaker", "note": "a multi-turn red-team probe against agents that use tools" }, { "tool": "promptfoo", "check": "mcp", "note": "tests an agent's MCP tool surface for known attack patterns" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART14", "name": "EU AI Act Art. 14 human oversight", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art14" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" }, { "id": "AIGE-OBL-CSA-AICM-AGENTIC", "name": "AICM agent controls with the CSA Agentic Trust Framework and AARM specification", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm-agentic" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" }, { "id": "A.9.2", "title": "Processes for responsible use of AI systems" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" }, { "id": "IAM", "title": "Identity & Access Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [], "related": [ "llm03-2026", "llm06-2026", "aml-t0053", "aml-t0086", "nistaml-039" ], "layers": [ 4 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "asi03", "taxonomy": "owasp-asi", "externalId": "ASI03", "name": "Identity and Privilege Abuse", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi03", "summary": "Delegated identities, inherited privileges and cached credentials let an agent (or whoever steers it) act with rights the requesting user never had.", "control": "A distinct workload identity per agent, short-lived delegated tokens bound to the user and the audience, and no standing secrets in agent configuration.", "patterns": [ { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" }, { "slug": "agent-registry", "title": "Agent Registry", "url": "https://aigovernanceengineer.com/patterns/agent-registry" } ], "evals": [ { "tool": "promptfoo", "check": "bola", "note": "tests broken object-level authorisation: reaching another user's records" }, { "tool": "promptfoo", "check": "rbac", "note": "checks that role-based access control holds through the agent" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" }, { "id": "AIGE-OBL-CSA-AICM-AGENTIC", "name": "AICM agent controls with the CSA Agentic Trust Framework and AARM specification", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm-agentic" }, { "id": "AIGE-OBL-NIST-AGENTS", "name": "NIST AI Agent Standards Initiative (2026)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nist-agents" } ], "iso42001": [ { "id": "A.6.2.5", "title": "AI system deployment" } ], "aicmDomains": [ { "id": "IAM", "title": "Identity & Access Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [ "llm03-2026", "nistaml-039" ], "layers": [ 4 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "asi04", "taxonomy": "owasp-asi", "externalId": "ASI04", "name": "Agentic Supply Chain Vulnerabilities", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi04", "summary": "Tools, MCP servers, plugins, prompt templates and models that an agent loads, often at run time, can be malicious or become malicious after they were trusted.", "control": "Admission of servers and tools against a registry, pinned and hashed definitions, and an AIBOM that records what the agent can reach.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "agent-registry", "title": "Agent Registry", "url": "https://aigovernanceengineer.com/patterns/agent-registry" }, { "slug": "vendor-model-due-diligence-gate", "title": "Vendor / Model Due-Diligence Gate", "url": "https://aigovernanceengineer.com/patterns/vendor-model-due-diligence-gate" } ], "evals": [ { "tool": "promptfoo", "check": "mcp", "note": "tests the MCP servers and tools an agent connects to" }, { "tool": "custom", "check": "definition-drift check", "note": "fails when a tool definition differs from the pinned hash" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART25", "name": "EU AI Act Art. 25 responsibilities along the AI value chain", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art25" }, { "id": "AIGE-OBL-ISO42001-A10", "name": "A.10 Third-party and customer relationships", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a10" }, { "id": "AIGE-OBL-OWASP-AIBOM", "name": "AIBOM", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-aibom" } ], "iso42001": [ { "id": "A.10.3", "title": "Suppliers" }, { "id": "A.7.5", "title": "Data provenance" } ], "aicmDomains": [ { "id": "STA", "title": "Supply Chain Management, Transparency, and Accountability" } ], "ssdfTasks": [ { "id": "PS.3.2", "title": "Keep provenance data for every component of a release" } ], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" }, { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [], "related": [ "llm04-2026", "aml-t0110", "aml-t0010", "aml-t0109" ], "layers": [ 2 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "asi05", "taxonomy": "owasp-asi", "externalId": "ASI05", "name": "Unexpected Code Execution (RCE)", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi05", "summary": "Code the agent writes or runs, directly or through a tool, executes outside the bounds anyone intended and compromises the host or the environment around it.", "control": "Sandboxed execution with no ambient credentials, deny-by-default network and file access, and review of generated code before it runs anywhere that matters.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "kill-switch-circuit-breaker", "title": "Kill Switch / Circuit Breaker", "url": "https://aigovernanceengineer.com/patterns/kill-switch-circuit-breaker" } ], "evals": [ { "tool": "Inspect", "check": "cyberseceval_2", "note": "includes a code-interpreter abuse task alongside prompt injection" }, { "tool": "promptfoo", "check": "shell-injection", "note": "tries to execute shell commands through the model" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" }, { "id": "AIGE-OBL-ETSI-304223", "name": "ETSI EN 304 223 baseline cyber-security for AI models and systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-etsi-304223" } ], "iso42001": [ { "id": "A.6.2.5", "title": "AI system deployment" }, { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" }, { "id": "IVS", "title": "Infrastructure & Virtualization Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [], "related": [ "llm10-2026" ], "layers": [ 4 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "asi06", "taxonomy": "owasp-asi", "externalId": "ASI06", "name": "Memory & Context Poisoning", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi06", "summary": "Stored memory, retrieval stores or carried context are corrupted so the poison persists across sessions and shapes later decisions.", "control": "A gate on memory writes, namespaces per user and task, provenance on stored items, and the ability to roll memory back.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "Inspect", "check": "agent_threat_bench_memory_poison", "note": "tasks with poisoned memory stores, scored on utility and security" }, { "tool": "promptfoo", "check": "agentic:memory-poisoning", "note": "tests whether an agent can be made to store and act on planted memories" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART12", "name": "EU AI Act Art. 12 record-keeping and logging", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art12" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" }, { "id": "A.6.2.8", "title": "AI system recording of event logs" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" }, { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [ "llm05-2026", "llm09-2026", "aml-t0080", "aml-t0070" ], "layers": [ 3, 4 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "asi07", "taxonomy": "owasp-asi", "externalId": "ASI07", "name": "Insecure Inter-Agent Communication", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi07", "summary": "Messages between agents travel without authentication or integrity, so a peer can be spoofed, a message replayed or altered, and trust passes along a chain that nobody checked.", "control": "Mutual authentication between agents, signed and verified agent descriptions, an allow-list of peers, and messages bound to the task they belong to.", "patterns": [ { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" }, { "slug": "agent-registry", "title": "Agent Registry", "url": "https://aigovernanceengineer.com/patterns/agent-registry" } ], "evals": [ { "tool": "custom", "check": "unregistered-peer test", "note": "a message from an agent not in the registry, or with a bad signature, must be refused" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" }, { "id": "AIGE-OBL-NIST-AGENTS", "name": "NIST AI Agent Standards Initiative (2026)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nist-agents" }, { "id": "AIGE-OBL-CSA-AICM-AGENTIC", "name": "AICM agent controls with the CSA Agentic Trust Framework and AARM specification", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm-agentic" } ], "iso42001": [ { "id": "A.6.2.5", "title": "AI system deployment" } ], "aicmDomains": [ { "id": "IAM", "title": "Identity & Access Management" }, { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [], "layers": [ 4 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "asi08", "taxonomy": "owasp-asi", "externalId": "ASI08", "name": "Cascading Failures", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi08", "summary": "One faulty or compromised agent, tool or output propagates through connected agents and workflows, and each hop amplifies the harm.", "control": "Depth, fan-out and budget limits, a breaker per agent, isolation between workflows, and telemetry that traces a failure back to its first hop.", "patterns": [ { "slug": "kill-switch-circuit-breaker", "title": "Kill Switch / Circuit Breaker", "url": "https://aigovernanceengineer.com/patterns/kill-switch-circuit-breaker" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" }, { "slug": "incident-pipeline", "title": "Incident Pipeline", "url": "https://aigovernanceengineer.com/patterns/incident-pipeline" } ], "evals": [ { "tool": "custom", "check": "fault-injection drill", "note": "a tool returns errors or bad data on purpose; the breaker must trip within its budget" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART72", "name": "EU AI Act Art. 72 post-market monitoring", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art72" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" }, { "id": "AIGE-OBL-NISTRMF-MANAGE", "name": "MANAGE", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nistrmf-manage" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "BCR", "title": "Business Continuity Management and Operational Resilience" }, { "id": "LOG", "title": "Logging and Monitoring" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [ "llm06-2026", "aml-t0034" ], "layers": [ 4, 5 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "asi09", "taxonomy": "owasp-asi", "externalId": "ASI09", "name": "Human-Agent Trust Exploitation", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi09", "summary": "Fluent, confident or persuasive agent output leads people to approve harmful actions, disclose information or skip the check they were meant to make.", "control": "Approvals that show the raw action and its consequences, not the agent's summary; oversight metrics such as approval rates and time to decide; and disclosure that the user is dealing with an AI system.", "patterns": [ { "slug": "human-in-the-loop-gate", "title": "Human-in-the-loop Gate", "url": "https://aigovernanceengineer.com/patterns/human-in-the-loop-gate" }, { "slug": "policy-card", "title": "Policy Card", "url": "https://aigovernanceengineer.com/patterns/policy-card" } ], "evals": [ { "tool": "Inspect", "check": "make_me_pay", "note": "measures persuasive capability in a simulated donation conversation" }, { "tool": "promptfoo", "check": "overreliance", "note": "checks whether the model goes along with a wrong user assumption" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART14", "name": "EU AI Act Art. 14 human oversight", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art14" }, { "id": "AIGE-OBL-EUAIA-ART13", "name": "EU AI Act Art. 13 transparency and information to deployers", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art13" }, { "id": "AIGE-OBL-EUAIA-ART50", "name": "EU AI Act Art. 50 transparency for certain AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art50" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" } ], "iso42001": [ { "id": "A.8.2", "title": "System documentation and information for users" }, { "id": "A.9.2", "title": "Processes for responsible use of AI systems" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" }, { "id": "HRS", "title": "Human Resources" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [], "related": [ "llm01-2026", "llm07-2026" ], "layers": [ 4, 5 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "asi10", "taxonomy": "owasp-asi", "externalId": "ASI10", "name": "Rogue Agents", "formerly": null, "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "page": "https://aigovernanceengineer.com/resources/threats#threat-asi10", "summary": "An agent drifts from its intended behaviour or scope (through compromise, misalignment or neglect) and keeps acting, possibly deceptively, where nobody is watching.", "control": "A registry entry with an owner and an expiry for every agent, discovery of agents that are not registered, behavioural monitoring, and a drilled kill switch.", "patterns": [ { "slug": "agent-registry", "title": "Agent Registry", "url": "https://aigovernanceengineer.com/patterns/agent-registry" }, { "slug": "shadow-ai-discovery", "title": "Shadow-AI Discovery", "url": "https://aigovernanceengineer.com/patterns/shadow-ai-discovery" }, { "slug": "kill-switch-circuit-breaker", "title": "Kill Switch / Circuit Breaker", "url": "https://aigovernanceengineer.com/patterns/kill-switch-circuit-breaker" }, { "slug": "incident-pipeline", "title": "Incident Pipeline", "url": "https://aigovernanceengineer.com/patterns/incident-pipeline" } ], "evals": [ { "tool": "Inspect", "check": "agentic_misalignment", "note": "fictional scenarios that probe for agents acting against their principals" }, { "tool": "custom", "check": "registry reconciliation", "note": "every agent identity seen at run time must match a live registry entry" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART14", "name": "EU AI Act Art. 14 human oversight", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art14" }, { "id": "AIGE-OBL-EUAIA-ART26", "name": "EU AI Act Art. 26 deployer obligations for high-risk systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art26" }, { "id": "AIGE-OBL-EUAIA-ART72", "name": "EU AI Act Art. 72 post-market monitoring", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art72" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" }, { "id": "A.6.2.8", "title": "AI system recording of event logs" } ], "aicmDomains": [ { "id": "GRC", "title": "Governance, Risk and Compliance" }, { "id": "LOG", "title": "Logging and Monitoring" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [], "layers": [ 2, 4 ], "sources": [ { "title": "OWASP Top 10 for Agentic Applications for 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" } ] }, { "id": "aml-t0051", "taxonomy": "mitre-atlas", "externalId": "AML.T0051", "name": "LLM Prompt Injection", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0051", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0051", "summary": "Crafted prompts make the model ignore its instructions and follow the adversary's, directly, indirectly through content it ingests, or on a trigger (sub-techniques .000 Direct, .001 Indirect, .002 Triggered). ATLAS places it under Execution.", "control": "Guardrails on inputs and outputs, restricted tool use on untrusted data, telemetry, and a standing red team.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "garak", "check": "promptinject", "note": "plaintext injections inside requests for innocuous information" }, { "tool": "promptfoo", "check": "indirect-prompt-injection", "note": "instructions planted in prompt variables" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART55", "name": "EU AI Act Art. 55 GPAI models with systemic risk", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art55" }, { "id": "AIGE-OBL-NISTRMF-MEASURE", "name": "MEASURE", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nistrmf-measure" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" }, { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [ { "id": "AML.M0019", "name": "Control Access to AI Models and Data in Production" }, { "id": "AML.M0020", "name": "Generative AI Guardrails" }, { "id": "AML.M0021", "name": "Generative AI Guidelines" }, { "id": "AML.M0022", "name": "Generative AI Model Alignment" }, { "id": "AML.M0024", "name": "AI Telemetry Logging" }, { "id": "AML.M0033", "name": "Input and Output Validation for AI Agent Components" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm01-2026", "asi01", "nistaml-018", "nistaml-015" ], "layers": [ 3, 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0054", "taxonomy": "mitre-atlas", "externalId": "AML.T0054", "name": "LLM Jailbreak", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0054", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0054", "summary": "The model is induced to bypass its safety behaviour and guardrails and produce what it is meant to withhold, by adversarial prompting or by changing weights or safety mechanisms.", "control": "Alignment and guardrails tested against current jailbreak families, with the jailbreak rate as a release threshold.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" } ], "evals": [ { "tool": "Inspect", "check": "strong_reject", "note": "scores both refusal and how useful a non-refused harmful answer is" }, { "tool": "garak", "check": "tap", "note": "Tree of Attacks with Pruning: model-generated jailbreak prompts" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART55", "name": "EU AI Act Art. 55 GPAI models with systemic risk", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art55" }, { "id": "AIGE-OBL-GPAICOP-SAFETY", "name": "Safety and Security (systemic-risk models only)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-gpaicop-safety" }, { "id": "AIGE-OBL-NIST-AI800-1", "name": "NIST AI 800-1 misuse risk for dual-use foundation models (draft)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nist-ai800-1" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" }, { "id": "MDS", "title": "Model Development Security" } ], "ssdfTasks": [ { "id": "PW.3.3", "title": "Include adversarial samples in training and testing data" } ], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" }, { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [ { "id": "AML.M0020", "name": "Generative AI Guardrails" }, { "id": "AML.M0021", "name": "Generative AI Guidelines" }, { "id": "AML.M0022", "name": "Generative AI Model Alignment" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm01-2026", "nistaml-018" ], "layers": [ 3, 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0056", "taxonomy": "mitre-atlas", "externalId": "AML.T0056", "name": "Extract LLM System Prompt", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0056", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0056", "summary": "The adversary obtains the system prompt, through injection that makes the model reveal it or from a configuration file, and uses it to plan further attacks.", "control": "Nothing secret in the prompt, policy enforced outside the model, and extraction attempts logged.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "garak", "check": "sysprompt_extraction", "note": "direct requests, encoding tricks and role-play" }, { "tool": "promptfoo", "check": "prompt-extraction", "note": "attempts to get the model to reveal its system prompt" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [ { "id": "AML.M0020", "name": "Generative AI Guardrails" }, { "id": "AML.M0021", "name": "Generative AI Guidelines" }, { "id": "AML.M0022", "name": "Generative AI Model Alignment" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm08-2026", "nistaml-035" ], "layers": [ 3, 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0057", "taxonomy": "mitre-atlas", "externalId": "AML.T0057", "name": "LLM Data Leakage", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0057", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0057", "summary": "Crafted prompts make the model leak private or proprietary information from its training data, its connected data sources or other users.", "control": "Data classification before retrieval, output filtering for personal and secret data, and session isolation.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" } ], "evals": [ { "tool": "promptfoo", "check": "pii:direct", "note": "direct attempts to obtain personal data" }, { "tool": "garak", "check": "leakreplay", "note": "replay attacks that test whether a document was in the training data" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-ISO42001-A7", "name": "A.7 Data for AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a7" } ], "iso42001": [ { "id": "A.7.4", "title": "Quality of data for AI systems" }, { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [ { "id": "AML.M0008", "name": "Validate AI Model" }, { "id": "AML.M0020", "name": "Generative AI Guardrails" }, { "id": "AML.M0021", "name": "Generative AI Guidelines" }, { "id": "AML.M0022", "name": "Generative AI Model Alignment" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm02-2026", "nistaml-038" ], "layers": [ 3, 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0077", "taxonomy": "mitre-atlas", "externalId": "AML.T0077", "name": "LLM Response Rendering", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0077", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0077", "summary": "The model is induced to embed private data in a reference to external content (an image, a link) that the user's client fetches on rendering, sending the data to the adversary unseen.", "control": "Render model output with external fetches disabled or restricted to an allow-list, and strip data from URLs.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" } ], "evals": [ { "tool": "garak", "check": "web_injection", "note": "includes Markdown image exfiltration probes" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" } ], "iso42001": [ { "id": "A.6.2.5", "title": "AI system deployment" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "llm02-2026", "llm10-2026" ], "layers": [ 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0053", "taxonomy": "mitre-atlas", "externalId": "AML.T0053", "name": "AI Agent Tool Invocation", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0053", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0053", "summary": "With access to an agent, the adversary invokes the tools the agent holds (integrations, data sources, code execution) to reach systems it could not reach directly.", "control": "Per-tool permissions, single-user scoping, no tool calls driven by untrusted data without a check, and human approval for sensitive actions.", "patterns": [ { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" }, { "slug": "human-in-the-loop-gate", "title": "Human-in-the-loop Gate", "url": "https://aigovernanceengineer.com/patterns/human-in-the-loop-gate" }, { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" } ], "evals": [ { "tool": "garak", "check": "agent_breaker", "note": "multi-turn attacks on agents that use tools" }, { "tool": "promptfoo", "check": "tool-discovery", "note": "checks whether the agent reveals the tools it holds" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART14", "name": "EU AI Act Art. 14 human oversight", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art14" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-CSA-AICM-AGENTIC", "name": "AICM agent controls with the CSA Agentic Trust Framework and AARM specification", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm-agentic" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" }, { "id": "A.9.2", "title": "Processes for responsible use of AI systems" } ], "aicmDomains": [ { "id": "IAM", "title": "Identity & Access Management" }, { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [ { "id": "AML.M0020", "name": "Generative AI Guardrails" }, { "id": "AML.M0021", "name": "Generative AI Guidelines" }, { "id": "AML.M0022", "name": "Generative AI Model Alignment" }, { "id": "AML.M0024", "name": "AI Telemetry Logging" }, { "id": "AML.M0026", "name": "Privileged AI Agent Permissions Configuration" }, { "id": "AML.M0027", "name": "Single-User AI Agent Permissions Configuration" }, { "id": "AML.M0028", "name": "AI Agent Tools Permissions Configuration" }, { "id": "AML.M0029", "name": "Human In-the-Loop for AI Agent Actions" }, { "id": "AML.M0030", "name": "Restrict AI Agent Tool Invocation on Untrusted Data" }, { "id": "AML.M0032", "name": "Segmentation of AI Agent Components" }, { "id": "AML.M0033", "name": "Input and Output Validation for AI Agent Components" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm03-2026", "asi02", "nistaml-039" ], "layers": [ 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0086", "taxonomy": "mitre-atlas", "externalId": "AML.T0086", "name": "Exfiltration via AI Agent Tool Invocation", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0086", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0086", "summary": "An agent tool that can write (email, documents, records) is used to encode sensitive data into its parameters and send it somewhere the adversary controls, looking like legitimate work.", "control": "Egress allow-lists, data-loss checks on tool arguments, and approval for writes that leave the organisation.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" }, { "slug": "incident-pipeline", "title": "Incident Pipeline", "url": "https://aigovernanceengineer.com/patterns/incident-pipeline" } ], "evals": [ { "tool": "Inspect", "check": "agent_threat_bench_data_exfil", "note": "customer-support tasks with injected exfiltration instructions" }, { "tool": "promptfoo", "check": "data-exfil", "note": "indirect injection in web content that tries to make the agent send data out" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" }, { "id": "LOG", "title": "Logging and Monitoring" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [ { "id": "AML.M0024", "name": "AI Telemetry Logging" }, { "id": "AML.M0026", "name": "Privileged AI Agent Permissions Configuration" }, { "id": "AML.M0027", "name": "Single-User AI Agent Permissions Configuration" }, { "id": "AML.M0028", "name": "AI Agent Tools Permissions Configuration" }, { "id": "AML.M0029", "name": "Human In-the-Loop for AI Agent Actions" }, { "id": "AML.M0030", "name": "Restrict AI Agent Tool Invocation on Untrusted Data" }, { "id": "AML.M0032", "name": "Segmentation of AI Agent Components" }, { "id": "AML.M0033", "name": "Input and Output Validation for AI Agent Components" } ], "related": [ "llm02-2026", "asi01", "asi02" ], "layers": [ 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "aml-t0080", "taxonomy": "mitre-atlas", "externalId": "AML.T0080", "name": "AI Agent Context Poisoning", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0080", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0080", "summary": "The adversary changes the context an agent works from, in its memory (.000) or in a conversation thread (.001), to alter its behaviour persistently. ATLAS places it under Persistence.", "control": "Hardened memory: a gate on writes, provenance on stored items, and rollback.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" } ], "evals": [ { "tool": "promptfoo", "check": "agentic:memory-poisoning", "note": "planted memories the agent should neither store nor act on" }, { "tool": "Inspect", "check": "agent_threat_bench_memory_poison", "note": "poisoned memory stores in agent tasks" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [ { "id": "AML.M0031", "name": "Memory Hardening" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm05-2026", "asi06" ], "layers": [ 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "aml-t0070", "taxonomy": "mitre-atlas", "externalId": "AML.T0070", "name": "RAG Poisoning", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0070", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0070", "summary": "Malicious content is placed where a retrieval system indexes it, so it surfaces in future answers for chosen queries.", "control": "A source allow-list for ingestion, provenance recorded for every indexed document, and canary documents that must never be retrieved.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "promptfoo", "check": "rag-poisoning", "note": "poisoned documents in the retrieval corpus" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-ISO42001-A7", "name": "A.7 Data for AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a7" } ], "iso42001": [ { "id": "A.7.5", "title": "Data provenance" } ], "aicmDomains": [ { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [ { "id": "PW.3.2", "title": "Track the provenance of training, testing, fine-tuning and aligning data" } ], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [ { "id": "AML.M0020", "name": "Generative AI Guardrails" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm05-2026", "llm09-2026", "asi06", "nistaml-015" ], "layers": [ 2, 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "aml-t0110", "taxonomy": "mitre-atlas", "externalId": "AML.T0110", "name": "AI Agent Tool Poisoning", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0110", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0110", "summary": "A tool the agent uses is poisoned in its model-visible definition, its implementation or its run-time responses; the tool may be an integration, a package, an MCP server or an agent skill.", "control": "Admit tools through a registry, pin and hash their definitions, and treat tool responses as untrusted input.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "agent-registry", "title": "Agent Registry", "url": "https://aigovernanceengineer.com/patterns/agent-registry" }, { "slug": "vendor-model-due-diligence-gate", "title": "Vendor / Model Due-Diligence Gate", "url": "https://aigovernanceengineer.com/patterns/vendor-model-due-diligence-gate" } ], "evals": [ { "tool": "promptfoo", "check": "mcp", "note": "MCP attack patterns against the agent's tool surface" }, { "tool": "custom", "check": "definition-drift check", "note": "fails when a tool definition changes without review" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-ISO42001-A10", "name": "A.10 Third-party and customer relationships", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a10" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" } ], "iso42001": [ { "id": "A.10.3", "title": "Suppliers" } ], "aicmDomains": [ { "id": "STA", "title": "Supply Chain Management, Transparency, and Accountability" } ], "ssdfTasks": [ { "id": "PS.3.2", "title": "Keep provenance data for every component of a release" } ], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [ "llm04-2026", "asi04" ], "layers": [ 2, 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "aml-t0010", "taxonomy": "mitre-atlas", "externalId": "AML.T0010", "name": "AI Supply Chain Compromise", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0010", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0010", "summary": "Initial access through the parts of the supply chain unique to AI: hardware, AI software, data, models, container registries and agent tools (sub-techniques .000 to .005).", "control": "Verify artefacts (hashes, signatures) before use, keep an AIBOM, and red-team third-party components.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "vendor-model-due-diligence-gate", "title": "Vendor / Model Due-Diligence Gate", "url": "https://aigovernanceengineer.com/patterns/vendor-model-due-diligence-gate" } ], "evals": [ { "tool": "garak", "check": "fileformats", "note": "looks for vulnerable items among the files that ship with a model" }, { "tool": "custom", "check": "AIBOM diff gate", "note": "blocks an artefact whose hash is not in the approved AIBOM" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART25", "name": "EU AI Act Art. 25 responsibilities along the AI value chain", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art25" }, { "id": "AIGE-OBL-OWASP-AIBOM", "name": "AIBOM", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-aibom" }, { "id": "AIGE-OBL-ETSI-304223", "name": "ETSI EN 304 223 baseline cyber-security for AI models and systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-etsi-304223" } ], "iso42001": [ { "id": "A.10.3", "title": "Suppliers" }, { "id": "A.7.5", "title": "Data provenance" } ], "aicmDomains": [ { "id": "STA", "title": "Supply Chain Management, Transparency, and Accountability" } ], "ssdfTasks": [ { "id": "PS.3.2", "title": "Keep provenance data for every component of a release" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [ { "id": "AML.M0014", "name": "Verify AI Artifacts" }, { "id": "AML.M0020", "name": "Generative AI Guardrails" }, { "id": "AML.M0023", "name": "AI Bill of Materials" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm04-2026", "asi04", "nistaml-051" ], "layers": [ 2, 3 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0109", "taxonomy": "mitre-atlas", "externalId": "AML.T0109", "name": "AI Supply Chain Rug Pull", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0109", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0109", "summary": "A component is published in good faith, earns adoption, and is then updated with a malicious variant, after the scrutiny that came with first adoption has passed.", "control": "Pin versions, treat every update as a new admission, and run a canary eval that notices behaviour change nobody announced.", "patterns": [ { "slug": "vendor-model-due-diligence-gate", "title": "Vendor / Model Due-Diligence Gate", "url": "https://aigovernanceengineer.com/patterns/vendor-model-due-diligence-gate" }, { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" } ], "evals": [ { "tool": "custom", "check": "canary regression", "note": "a small fixed eval run on a schedule against each pinned component; drift raises a finding" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART26", "name": "EU AI Act Art. 26 deployer obligations for high-risk systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art26" }, { "id": "AIGE-OBL-ISO42001-A10", "name": "A.10 Third-party and customer relationships", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a10" } ], "iso42001": [ { "id": "A.10.3", "title": "Suppliers" }, { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "STA", "title": "Supply Chain Management, Transparency, and Accountability" }, { "id": "CCC", "title": "Change Control and Configuration Management" } ], "ssdfTasks": [ { "id": "PS.3.2", "title": "Keep provenance data for every component of a release" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" }, { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [], "related": [ "llm04-2026", "asi04" ], "layers": [ 2, 5 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "aml-t0020", "taxonomy": "mitre-atlas", "externalId": "AML.T0020", "name": "Training Data Poisoning", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0020", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0020", "summary": "Training or fine-tuning data (samples, labels, feedback) is added, removed or altered to bias the model, degrade it or embed a backdoor.", "control": "Sanitise and validate data, keep dataset provenance and an AIBOM, restrict write access to data at rest.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" }, { "slug": "model-card-as-control-evidence", "title": "Model Card as Control Evidence", "url": "https://aigovernanceengineer.com/patterns/model-card-as-control-evidence" } ], "evals": [ { "tool": "custom", "check": "data admission check", "note": "a dataset without provenance, owner and a tamper scan cannot enter training" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-ISO42001-A7", "name": "A.7 Data for AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a7" } ], "iso42001": [ { "id": "A.7.4", "title": "Quality of data for AI systems" }, { "id": "A.7.5", "title": "Data provenance" } ], "aicmDomains": [ { "id": "MDS", "title": "Model Development Security" }, { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [ { "id": "PW.3.1", "title": "Analyse training and test data for poisoning, bias and tampering before use" }, { "id": "PW.3.2", "title": "Track the provenance of training, testing, fine-tuning and aligning data" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" }, { "id": "predictive", "title": "Using and fine-tuning predictive AI" } ], "atlasMitigations": [ { "id": "AML.M0001", "name": "Limit Model Artifact Release" }, { "id": "AML.M0005", "name": "Control Access to AI Models and Data at Rest" }, { "id": "AML.M0007", "name": "Sanitize Training Data" }, { "id": "AML.M0008", "name": "Validate AI Model" }, { "id": "AML.M0023", "name": "AI Bill of Materials" }, { "id": "AML.M0025", "name": "Maintain AI Dataset Provenance" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "llm05-2026", "nistaml-013", "nistaml-023" ], "layers": [ 2, 3 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "aml-t0018", "taxonomy": "mitre-atlas", "externalId": "AML.T0018", "name": "Manipulate AI Model", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0018", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0018", "summary": "A model artefact or its bundled components is altered (weights, architecture, prompt-construction logic, embedded malware) and may behave normally until a chosen input arrives.", "control": "Sign and verify model artefacts, restrict access to models at rest, and validate the model before each release.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" } ], "evals": [ { "tool": "custom", "check": "signature verification", "note": "the pipeline fails on a hash or signature mismatch" }, { "tool": "garak", "check": "fileformats", "note": "flags risky file formats in a model repository" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-GPAICOP-SAFETY", "name": "Safety and Security (systemic-risk models only)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-gpaicop-safety" }, { "id": "AIGE-OBL-ISO42001-A6", "name": "A.6 AI system life cycle", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a6" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" }, { "id": "A.7.5", "title": "Data provenance" } ], "aicmDomains": [ { "id": "MDS", "title": "Model Development Security" }, { "id": "CEK", "title": "Cryptography, Encryption & Key Management" } ], "ssdfTasks": [ { "id": "PS.1.3", "title": "Protect model weights and configuration parameters from unauthorised access and change" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [ { "id": "AML.M0005", "name": "Control Access to AI Models and Data at Rest" }, { "id": "AML.M0008", "name": "Validate AI Model" }, { "id": "AML.M0013", "name": "Code Signing" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "nistaml-023", "nistaml-051" ], "layers": [ 2, 3 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0024", "taxonomy": "mitre-atlas", "externalId": "AML.T0024", "name": "Exfiltration via AI Inference API", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0024", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0024", "summary": "Queries to the inference API leak private information about training data (membership .000, inversion .001) or the model itself (extraction .002).", "control": "Rate and volume limits per identity, access control on the API, query-pattern monitoring, and privacy testing of tuned models.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "garak", "check": "leakreplay", "note": "tests whether the model reproduces text it was trained on" }, { "tool": "custom", "check": "extraction probe", "note": "a scripted query campaign against the rate limits and the detector" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART55", "name": "EU AI Act Art. 55 GPAI models with systemic risk", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art55" }, { "id": "AIGE-OBL-GPAICOP-SAFETY", "name": "Safety and Security (systemic-risk models only)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-gpaicop-safety" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" }, { "id": "LOG", "title": "Logging and Monitoring" } ], "ssdfTasks": [ { "id": "RV.1.1", "title": "Gather vulnerability information; log and analyse model inputs and outputs" } ], "cosaisUseCases": [ { "id": "predictive", "title": "Using and fine-tuning predictive AI" }, { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [ { "id": "AML.M0004", "name": "Limit AI Service Query Volume and Rate" }, { "id": "AML.M0019", "name": "Control Access to AI Models and Data in Production" }, { "id": "AML.M0024", "name": "AI Telemetry Logging" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "nistaml-031", "nistaml-032", "nistaml-033" ], "layers": [ 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "aml-t0015", "taxonomy": "mitre-atlas", "externalId": "AML.T0015", "name": "Evade AI Model", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0015", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0015", "summary": "Crafted adversarial data or deepfakes make a model misidentify what it sees, for example to slip past AI-based malware or fraud detection.", "control": "Hardened models, adversarial input detection, ensembles or multi-sensor checks, and robustness thresholds in the release gate.", "patterns": [ { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" } ], "evals": [ { "tool": "custom", "check": "perturbation suite", "note": "bounded perturbations of held-out inputs; accuracy under attack is a release threshold" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-NISTRMF-MEASURE", "name": "MEASURE", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nistrmf-measure" }, { "id": "AIGE-OBL-ISO42001-A6", "name": "A.6 AI system life cycle", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a6" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "MDS", "title": "Model Development Security" } ], "ssdfTasks": [ { "id": "PW.3.3", "title": "Include adversarial samples in training and testing data" } ], "cosaisUseCases": [ { "id": "predictive", "title": "Using and fine-tuning predictive AI" } ], "atlasMitigations": [ { "id": "AML.M0003", "name": "Predictive AI Model Hardening" }, { "id": "AML.M0006", "name": "Predictive AI Ensembles" }, { "id": "AML.M0009", "name": "Predictive AI Multi-Sensor Fusion" }, { "id": "AML.M0010", "name": "Predictive AI Input Restoration" }, { "id": "AML.M0015", "name": "Predictive AI Adversarial Input Detection" }, { "id": "AML.M0034", "name": "Deepfake Detection" }, { "id": "AML.M0035", "name": "AI Red Team" } ], "related": [ "nistaml-022" ], "layers": [ 3 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "aml-t0034", "taxonomy": "mitre-atlas", "externalId": "AML.T0034", "name": "Cost Harvesting", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0034", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0034", "summary": "The adversary drives AI services beyond normal capacity to raise the victim's cost, with many cheap queries, a few expensive ones, or agentic loops (.002 Agentic Resource Consumption).", "control": "Query and resource limits per identity, budgets per agent, and alerts on spend anomalies.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "kill-switch-circuit-breaker", "title": "Kill Switch / Circuit Breaker", "url": "https://aigovernanceengineer.com/patterns/kill-switch-circuit-breaker" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" } ], "evals": [ { "tool": "promptfoo", "check": "reasoning-dos", "note": "resource exhaustion through excessive reasoning" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-CSA-AICM", "name": "AICM v1.1: 247 control objectives across 18 domains", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "BCR", "title": "Business Continuity Management and Operational Resilience" }, { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [ { "id": "AML.M0004", "name": "Limit AI Service Query Volume and Rate" }, { "id": "AML.M0019", "name": "Control Access to AI Models and Data in Production" }, { "id": "AML.M0035", "name": "AI Red Team" }, { "id": "AML.M0036", "name": "Limit AI Workload Resource Consumption" } ], "related": [ "llm06-2026", "asi08", "nistaml-014" ], "layers": [ 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "aml-t0060", "taxonomy": "mitre-atlas", "externalId": "AML.T0060", "name": "Publish Hallucinated Entities", "formerly": null, "url": "https://atlas.mitre.org/techniques/AML.T0060", "page": "https://aigovernanceengineer.com/resources/threats#threat-aml-t0060", "summary": "The adversary registers a package, website or address that models tend to invent, and waits for a victim to follow the hallucination.", "control": "Allow-lists for packages and domains the system may recommend or install, and checks that generated references exist.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" }, { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" } ], "evals": [ { "tool": "garak", "check": "packagehallucination", "note": "asks for code and checks for packages that do not exist" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "STA", "title": "Supply Chain Management, Transparency, and Accountability" }, { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" }, { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [], "related": [ "llm04-2026", "llm07-2026" ], "layers": [ 3, 4 ], "sources": [ { "title": "MITRE ATLAS data, release v2026.09", "url": "https://github.com/mitre-atlas/atlas-data/releases/tag/v2026.09", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "nistaml-022", "taxonomy": "nist-aml", "externalId": "NISTAML.022", "name": "Evasion", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-022", "summary": "An integrity attack on predictive models at deployment time: inputs are modified, often imperceptibly, so the model returns the attacker's chosen output.", "control": "Adversarial training and input checks, with accuracy under bounded attack measured in the release gate.", "patterns": [ { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" } ], "evals": [ { "tool": "custom", "check": "perturbation suite", "note": "accuracy under a stated perturbation budget, as a threshold" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-NISTRMF-MEASURE", "name": "MEASURE", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nistrmf-measure" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "MDS", "title": "Model Development Security" } ], "ssdfTasks": [ { "id": "PW.3.3", "title": "Include adversarial samples in training and testing data" } ], "cosaisUseCases": [ { "id": "predictive", "title": "Using and fine-tuning predictive AI" } ], "atlasMitigations": [], "related": [ "aml-t0015" ], "layers": [ 3 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "nistaml-013", "taxonomy": "nist-aml", "externalId": "NISTAML.013", "name": "Data Poisoning", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-013", "summary": "Training data is manipulated to degrade the model broadly (availability) or on chosen inputs (integrity); NIST lists it for both predictive and generative models.", "control": "Data provenance, sanitisation and outlier checks before training, and evals that compare against a clean baseline.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" } ], "evals": [ { "tool": "custom", "check": "data admission check", "note": "provenance, owner and tamper scan required before training" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-ISO42001-A7", "name": "A.7 Data for AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a7" } ], "iso42001": [ { "id": "A.7.4", "title": "Quality of data for AI systems" }, { "id": "A.7.5", "title": "Data provenance" } ], "aicmDomains": [ { "id": "MDS", "title": "Model Development Security" }, { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [ { "id": "PW.3.1", "title": "Analyse training and test data for poisoning, bias and tampering before use" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" }, { "id": "predictive", "title": "Using and fine-tuning predictive AI" } ], "atlasMitigations": [], "related": [ "llm05-2026", "aml-t0020" ], "layers": [ 2, 3 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "nistaml-023", "taxonomy": "nist-aml", "externalId": "NISTAML.023", "name": "Backdoor Poisoning", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-023", "summary": "Poisoned training data implants a trigger, so the model behaves normally until the trigger appears and then does what the attacker wants.", "control": "Provenance on data and models, trigger scanning, and red-team cases for suspected triggers.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "custom", "check": "trigger regression set", "note": "suspected triggers run against every retrained version" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-ISO42001-A7", "name": "A.7 Data for AI systems", "url": "https://aigovernanceengineer.com/obligations/aige-obl-iso42001-a7" } ], "iso42001": [ { "id": "A.7.5", "title": "Data provenance" }, { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "MDS", "title": "Model Development Security" } ], "ssdfTasks": [ { "id": "PW.3.1", "title": "Analyse training and test data for poisoning, bias and tampering before use" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [], "related": [ "llm05-2026", "aml-t0020", "aml-t0018" ], "layers": [ 2, 3 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "nistaml-051", "taxonomy": "nist-aml", "externalId": "NISTAML.051", "name": "Model Poisoning (supply chain)", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-051", "summary": "The model itself (its weights or parameters) is modified and distributed through the supply chain, so every downstream user inherits the flaw.", "control": "Obtain models from verified sources, verify hashes and signatures, and keep the AIBOM current.", "patterns": [ { "slug": "aibom", "title": "AIBOM", "url": "https://aigovernanceengineer.com/patterns/aibom" }, { "slug": "vendor-model-due-diligence-gate", "title": "Vendor / Model Due-Diligence Gate", "url": "https://aigovernanceengineer.com/patterns/vendor-model-due-diligence-gate" } ], "evals": [ { "tool": "custom", "check": "signature verification", "note": "the pipeline refuses a model whose hash or signature does not match" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART25", "name": "EU AI Act Art. 25 responsibilities along the AI value chain", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art25" }, { "id": "AIGE-OBL-OWASP-AIBOM", "name": "AIBOM", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-aibom" } ], "iso42001": [ { "id": "A.10.3", "title": "Suppliers" }, { "id": "A.7.5", "title": "Data provenance" } ], "aicmDomains": [ { "id": "STA", "title": "Supply Chain Management, Transparency, and Accountability" }, { "id": "MDS", "title": "Model Development Security" } ], "ssdfTasks": [ { "id": "PS.1.3", "title": "Protect model weights and configuration parameters from unauthorised access and change" }, { "id": "PS.3.2", "title": "Keep provenance data for every component of a release" } ], "cosaisUseCases": [ { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [], "related": [ "llm04-2026", "aml-t0010", "aml-t0018" ], "layers": [ 2 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "nistaml-031", "taxonomy": "nist-aml", "externalId": "NISTAML.031", "name": "Model Extraction", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-031", "summary": "Queries to a deployed model are used to reconstruct a functionally similar copy, which also helps the attacker stage other attacks offline.", "control": "Rate limits and query monitoring per identity, and output that reveals no more than the use case needs.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" } ], "evals": [ { "tool": "custom", "check": "extraction probe", "note": "a scripted campaign that must trip the rate limit and the detector" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-GPAICOP-SAFETY", "name": "Safety and Security (systemic-risk models only)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-gpaicop-safety" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "LOG", "title": "Logging and Monitoring" }, { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [ { "id": "PS.1.3", "title": "Protect model weights and configuration parameters from unauthorised access and change" } ], "cosaisUseCases": [ { "id": "predictive", "title": "Using and fine-tuning predictive AI" }, { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "llm06-2026", "aml-t0024" ], "layers": [ 4 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "NIST SP 800-218A, Secure Software Development Practices for Generative AI and Dual-Use Foundation Models: An SSDF Community Profile", "url": "https://csrc.nist.gov/pubs/sp/800/218/a/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" } ] }, { "id": "nistaml-032", "taxonomy": "nist-aml", "externalId": "NISTAML.032", "name": "Reconstruction", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-032", "summary": "A privacy attack that recovers training records, or attributes of them, from the model or its outputs.", "control": "Minimise personal data in training, test tuned models for memorisation before release, and filter outputs.", "patterns": [ { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "garak", "check": "propile", "note": "probes whether the model has memorised and can leak personal data" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" } ], "iso42001": [ { "id": "A.7.4", "title": "Quality of data for AI systems" }, { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "predictive", "title": "Using and fine-tuning predictive AI" } ], "atlasMitigations": [], "related": [ "llm02-2026", "aml-t0024" ], "layers": [ 3 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "nistaml-033", "taxonomy": "nist-aml", "externalId": "NISTAML.033", "name": "Membership Inference", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-033", "summary": "The attacker infers whether a given record was in the training data, which is itself a privacy breach when the dataset is sensitive.", "control": "Privacy testing of models trained or tuned on personal data, and limits on the confidence detail the API returns.", "patterns": [ { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "garak", "check": "leakreplay", "note": "tests whether a document was in the training data by replay" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" } ], "iso42001": [ { "id": "A.7.4", "title": "Quality of data for AI systems" } ], "aicmDomains": [ { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "predictive", "title": "Using and fine-tuning predictive AI" }, { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "llm02-2026", "aml-t0024" ], "layers": [ 3 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "nistaml-018", "taxonomy": "nist-aml", "externalId": "NISTAML.018", "name": "Prompt Injection", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-018", "summary": "A direct prompting attack: the attacker, as the user, supplies instructions that override the application's. NIST treats a jailbreak as one kind of it and lists the class under availability, integrity, privacy and misuse violations alike.", "control": "Guardrails on inputs and outputs, alignment tested against current attack families, and misuse monitoring.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "garak", "check": "dan", "note": "do-anything-now style jailbreak prompts" }, { "tool": "Inspect", "check": "strong_reject", "note": "jailbreak susceptibility with a graded evaluator" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART55", "name": "EU AI Act Art. 55 GPAI models with systemic risk", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art55" }, { "id": "AIGE-OBL-NIST-AI800-1", "name": "NIST AI 800-1 misuse risk for dual-use foundation models (draft)", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nist-ai800-1" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "llm01-2026", "aml-t0051", "aml-t0054" ], "layers": [ 3, 4 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "nistaml-015", "taxonomy": "nist-aml", "externalId": "NISTAML.015", "name": "Indirect Prompt Injection", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-015", "summary": "Instructions are planted in resources the model ingests (web pages, documents, emails, tool results) rather than typed by the user, so the attacker never talks to the system directly.", "control": "Mark and isolate untrusted content, restrict what tools can do on it, and require approval for consequential actions.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "Inspect", "check": "agentdojo", "note": "agents solving tasks over untrusted data that carries injections" }, { "tool": "garak", "check": "latentinjection", "note": "injections buried in documents" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-AGENTIC", "name": "Top 10 for Agentic Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-agentic" }, { "id": "AIGE-OBL-NISTRMF-MEASURE", "name": "MEASURE", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nistrmf-measure" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" }, { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" }, { "id": "single-agent", "title": "AI agent systems: single agent" } ], "atlasMitigations": [], "related": [ "llm01-2026", "asi01", "aml-t0051", "aml-t0070" ], "layers": [ 3, 4 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "nistaml-035", "taxonomy": "nist-aml", "externalId": "NISTAML.035", "name": "Prompt Extraction", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-035", "summary": "A privacy attack that makes a generative model reveal its system prompt or other hidden context.", "control": "No secrets in the prompt, and extraction attempts logged and tested.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "adversarial-red-team-suite", "title": "Adversarial Red-Team Suite", "url": "https://aigovernanceengineer.com/patterns/adversarial-red-team-suite" } ], "evals": [ { "tool": "garak", "check": "sysprompt_extraction", "note": "system prompt extraction attempts" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-OWASP-LLM", "name": "Top 10 for LLM Applications 2026", "url": "https://aigovernanceengineer.com/obligations/aige-obl-owasp-llm" } ], "iso42001": [ { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "llm08-2026", "aml-t0056" ], "layers": [ 3, 4 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "nistaml-038", "taxonomy": "nist-aml", "externalId": "NISTAML.038", "name": "Data Extraction", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-038", "summary": "Training data, including personal or copyrighted text, is extracted from a generative model through its outputs.", "control": "Deduplicate and minimise sensitive training data, test for memorisation, and filter outputs that reproduce it.", "patterns": [ { "slug": "eval-gate-in-ci", "title": "Eval Gate in CI", "url": "https://aigovernanceengineer.com/patterns/eval-gate-in-ci" }, { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" } ], "evals": [ { "tool": "promptfoo", "check": "divergent-repetition", "note": "repetition patterns that push the model to reveal training data" }, { "tool": "garak", "check": "divergence", "note": "repeat attacks that make output drift into training data" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART10", "name": "EU AI Act Art. 10 data and data governance", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art10" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-EUAIA-ART53", "name": "EU AI Act Art. 53 GPAI provider obligations", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art53" } ], "iso42001": [ { "id": "A.7.4", "title": "Quality of data for AI systems" }, { "id": "A.6.2.4", "title": "AI system verification and validation" } ], "aicmDomains": [ { "id": "DSP", "title": "Data Security and Privacy Lifecycle Management" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" }, { "id": "developers", "title": "Security controls for AI developers" } ], "atlasMitigations": [], "related": [ "llm02-2026", "aml-t0057" ], "layers": [ 3, 4 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" }, { "title": "garak, LLM vulnerability scanner, release v0.17.0", "url": "https://github.com/NVIDIA/garak", "verified": "primary" } ] }, { "id": "nistaml-039", "taxonomy": "nist-aml", "externalId": "NISTAML.039", "name": "Compromising connected resources", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-039", "summary": "Through prompt injection, the model is made to leak private information from the restricted resources it can reach (tools, APIs, data stores); NIST classes it as a privacy compromise.", "control": "Least-privilege access for the model's identity, authorisation enforced at each resource, and approval before sensitive actions.", "patterns": [ { "slug": "agent-identity-scoped-credentials", "title": "Agent Identity & Scoped Credentials", "url": "https://aigovernanceengineer.com/patterns/agent-identity-scoped-credentials" }, { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "human-in-the-loop-gate", "title": "Human-in-the-loop Gate", "url": "https://aigovernanceengineer.com/patterns/human-in-the-loop-gate" } ], "evals": [ { "tool": "promptfoo", "check": "ssrf", "note": "server-side request forgery through the model" }, { "tool": "Inspect", "check": "agentdojo", "note": "injections that try to misuse the agent's tools" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART14", "name": "EU AI Act Art. 14 human oversight", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art14" }, { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-CSA-AICM-AGENTIC", "name": "AICM agent controls with the CSA Agentic Trust Framework and AARM specification", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm-agentic" } ], "iso42001": [ { "id": "A.6.2.5", "title": "AI system deployment" }, { "id": "A.9.2", "title": "Processes for responsible use of AI systems" } ], "aicmDomains": [ { "id": "IAM", "title": "Identity & Access Management" }, { "id": "AIS", "title": "Application & Interface Security" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "single-agent", "title": "AI agent systems: single agent" }, { "id": "multi-agent", "title": "AI agent systems: multi-agent" } ], "atlasMitigations": [], "related": [ "asi02", "asi03", "aml-t0053" ], "layers": [ 4 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "Inspect Evals", "url": "https://github.com/UKGovernmentBEIS/inspect_evals", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] }, { "id": "nistaml-014", "taxonomy": "nist-aml", "externalId": "NISTAML.014", "name": "Energy-latency", "formerly": null, "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "page": "https://aigovernanceengineer.com/resources/threats#threat-nistaml-014", "summary": "Inputs crafted to maximise computation degrade the service for everyone, an availability attack.", "control": "Per-request compute and time limits, and load monitoring with a breaker.", "patterns": [ { "slug": "runtime-guardrail", "title": "Runtime Guardrail", "url": "https://aigovernanceengineer.com/patterns/runtime-guardrail" }, { "slug": "kill-switch-circuit-breaker", "title": "Kill Switch / Circuit Breaker", "url": "https://aigovernanceengineer.com/patterns/kill-switch-circuit-breaker" } ], "evals": [ { "tool": "promptfoo", "check": "reasoning-dos", "note": "computational exhaustion through reasoning" } ], "obligations": [ { "id": "AIGE-OBL-EUAIA-ART15", "name": "EU AI Act Art. 15 accuracy, robustness and cybersecurity", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art15" }, { "id": "AIGE-OBL-CSA-AICM", "name": "AICM v1.1: 247 control objectives across 18 domains", "url": "https://aigovernanceengineer.com/obligations/aige-obl-csa-aicm" } ], "iso42001": [ { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "aicmDomains": [ { "id": "BCR", "title": "Business Continuity Management and Operational Resilience" } ], "ssdfTasks": [], "cosaisUseCases": [ { "id": "predictive", "title": "Using and fine-tuning predictive AI" }, { "id": "genai-assistant", "title": "Using a generative AI assistant (LLM)" } ], "atlasMitigations": [], "related": [ "llm06-2026", "aml-t0034" ], "layers": [ 4 ], "sources": [ { "title": "NIST AI 100-2 E2025, Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations", "url": "https://csrc.nist.gov/pubs/ai/100/2/e2025/final", "verified": "primary" }, { "title": "SP 800-53 Control Overlays for Securing AI Systems (COSAiS)", "url": "https://csrc.nist.gov/projects/cosais", "verified": "primary" }, { "title": "ISO/IEC 42001:2023, AI management systems, Annex A", "url": "https://www.iso.org/standard/81230.html", "verified": "secondary" }, { "title": "AI Controls Matrix v1.1", "url": "https://cloudsecurityalliance.org/artifacts/ai-controls-matrix-v1-1", "verified": "primary" }, { "title": "promptfoo red-team plugins", "url": "https://www.promptfoo.dev/docs/red-team/plugins/", "verified": "primary" } ] } ] }