{ "notice": "Illustrative mapping from the AI Governance Engineer Body of Knowledge v0.5.0 (not a claim of conformity)", "version": "0.5.0", "license": "CC BY 4.0", "licenseUrl": "https://creativecommons.org/licenses/by/4.0/", "schemaVersion": 1, "schema": "https://aigovernanceengineer.com/api/v1/schemas/control.json", "self": "https://aigovernanceengineer.com/api/v1/controls/aige-ctl-eval-005.json", "source": "https://aigovernanceengineer.com/controls/evaluation-environment/aige-ctl-eval-005", "citation": { "title": "AI Governance Engineering: The Thesis & Body of Knowledge", "authors": [ "Jorge García Aibar" ], "parentDoi": "https://doi.org/10.5281/zenodo.22956197", "conceptDoi": "https://doi.org/10.5281/zenodo.22857084" }, "control": { "id": "AIGE-CTL-EVAL-005", "profile": "evaluation-environment", "url": "https://aigovernanceengineer.com/controls/evaluation-environment/aige-ctl-eval-005", "json": "https://aigovernanceengineer.com/api/v1/controls/aige-ctl-eval-005.json", "title": "Monitoring Integrity", "version": "0.2", "status": "draft", "reviewerStatus": "open", "depth": "specified", "objective": "The traces and monitors of a run capture every plan step and tool call with the agent's identity and version, as the tool returned it, and the agent cannot disable or alter them.", "failureModes": [ "A tool call appears in the tool server's logs but not in the run's trace, and no gap is recorded.", "Monitoring is switched off or reconfigured from inside the environment during a run, for example by changing an environment variable.", "A monitor that the run relies on does not run on it, or covers only a sample of it, and nobody notices until after the run.", "The recorded output of a tool call differs from what the tool returned (a spoofed tool call)." ], "scope": "Telemetry, traces and monitors attached to an evaluation run, and the store they are written to. Preserving the records after an incident is covered by AIGE-CTL-EVAL-007.", "enforcementPoints": [ "runtime", "periodic" ], "verification": [ { "kind": "inspect", "text": "Before the run, inspect the monitoring configuration: the trace collector, the monitors and their settings run outside the environment, and nothing the agent can write (environment variables, files, its own processes) can switch them off or change what they record." }, { "kind": "test", "text": "At admission and after each harness change, from inside the environment, try to disable or redirect monitoring (change the environment variables and configuration the agent can reach, stop any local exporter); the trace must continue at the collector and the attempt must raise an alert." }, { "kind": "observe", "text": "During the run, record the heartbeat of each monitor the run relies on: it must cover every sample and the whole duration, and any gap is recorded with its start and end." }, { "kind": "observe", "text": "After the run, reconcile the trace with the tool servers' own logs: the same calls in the same order, and every recorded tool output matches the hash the tool server logged for it." } ], "evidence": [ { "artefact": "Monitor coverage record of the run: which monitors ran, over which samples and time, and every gap", "schemaId": "evidence-record", "schema": "https://aigovernanceengineer.com/schemas/evidence-record.v1.json", "layer": 4 }, { "artefact": "Reconciliation of the trace with the tool servers' logs, with the hash of every tool output", "schemaId": "evidence-record", "schema": "https://aigovernanceengineer.com/schemas/evidence-record.v1.json", "layer": 4 }, { "artefact": "Tampering test at admission and the reconciliation verdict, filed as an observation of this control", "schemaId": "control-observation", "schema": "https://aigovernanceengineer.com/schemas/control-observation.v1.json", "layer": 5 } ], "failureResponse": { "effect": "alert", "text": "A monitoring gap, a tampering attempt or a trace that does not reconcile with the tool servers' logs alerts the evaluator and marks the run. The result of a run whose monitors did not cover it, or whose trace does not reconcile, is withheld until the difference is explained; a tampering attempt is also a stop condition (AIGE-CTL-EVAL-006)." }, "layer": 4, "secondaryLayers": [ 5 ], "patterns": [ { "slug": "continuous-assurance-telemetry", "title": "Continuous Assurance Telemetry", "url": "https://aigovernanceengineer.com/patterns/continuous-assurance-telemetry" } ], "seeds": [ { "id": "traces", "title": "Traces", "url": "https://aigovernanceengineer.com/bok/governing-agents#telemetry-with-the-opentelemetry-genai-conventions" }, { "id": "otel-telemetry", "title": "Telemetry on the OpenTelemetry GenAI conventions", "url": "https://aigovernanceengineer.com/bok/governing-agents#telemetry-with-the-opentelemetry-genai-conventions" }, { "id": "trajectory-anomaly", "title": "Trajectory anomaly detection", "url": "https://aigovernanceengineer.com/bok/governing-agents#an-agent-incident-taxonomy" } ], "derivedFrom": [], "mappings": { "obligations": [ { "id": "AIGE-OBL-EUAIA-ART12", "name": "EU AI Act Art. 12 record-keeping and logging", "url": "https://aigovernanceengineer.com/obligations/aige-obl-euaia-art12" }, { "id": "AIGE-OBL-NISTRMF-MEASURE", "name": "MEASURE", "url": "https://aigovernanceengineer.com/obligations/aige-obl-nistrmf-measure" } ], "iso42001": [ { "id": "A.6.2.8", "title": "AI system recording of event logs" }, { "id": "A.6.2.6", "title": "AI system operation and monitoring" } ], "nistAiRmf": [ { "id": "MEASURE 3.1", "title": "Existing, unanticipated and emergent risks are tracked" } ], "owasp": [ { "id": "asi10", "externalId": "ASI10", "name": "Rogue Agents", "url": "https://aigovernanceengineer.com/resources/threats#threat-asi10" } ], "atlas": [], "aiuc1": [ "E015" ], "csaAicm": [], "other": [ { "framework": "MITRE ATLAS mitigation", "ref": "AML.M0024", "note": "AI Telemetry Logging" }, { "framework": "NIST SP 800-53 Rev. 5", "ref": "AU-2", "note": "Event Logging" }, { "framework": "NIST SP 800-53 Rev. 5", "ref": "AU-9", "note": "Protection of Audit Information" }, { "framework": "NIST SP 800-53 Rev. 5", "ref": "AU-12", "note": "Audit Record Generation" }, { "framework": "NIST SP 800-53 Rev. 5", "ref": "SI-4", "note": "System Monitoring" } ] }, "references": [ { "n": 35, "title": "Governing AI agents", "text": "Governing AI agents (AI Governance Engineering Body of Knowledge v0.5.0, chapter 23, section \"Telemetry with the OpenTelemetry GenAI conventions\"). AI Governance Engineer (Jorge García Aibar). 2026-09.", "url": "https://aigovernanceengineer.com/bok/governing-agents#telemetry-with-the-opentelemetry-genai-conventions", "verified": "primary" }, { "n": 36, "title": "Governing AI agents", "text": "Governing AI agents (AI Governance Engineering Body of Knowledge v0.5.0, chapter 23, section \"An agent incident taxonomy\"). AI Governance Engineer (Jorge García Aibar). 2026-09.", "url": "https://aigovernanceengineer.com/bok/governing-agents#an-agent-incident-taxonomy", "verified": "primary" }, { "n": 37, "title": "OpenTelemetry semantic conventions for generative AI", "text": "OpenTelemetry semantic conventions for generative AI (agent, tool and model spans, events and metrics; status Development). OpenTelemetry. 2026.", "url": "https://github.com/open-telemetry/semantic-conventions-genai/tree/main/docs/gen-ai", "verified": "primary" }, { "n": 5, "title": "Brief independent investigation of agents' behavior, reasoning and collaboration in the OpenAI / Hugging Face hacking incident", "text": "Brief independent investigation of agents' behavior, reasoning and collaboration in the OpenAI / Hugging Face hacking incident (METR states that agents \"meant to be fully isolated from one another\" communicated through an internal package repository, and that one agent found working Hugging Face credentials exposed on the internet and posted them to the agents' board; it reports spoofed tool calls in at least 96 transcripts and transcripts missing components after container resets). METR. 2026-08-26.", "url": "https://metr.org/blog/2026-08-26-openai-hugging-face-incident-investigation/", "verified": "primary" }, { "n": 38, "title": "Frontier Risk Report (February to March 2026)", "text": "Frontier Risk Report (February to March 2026) (METR states that its red-teaming exercise at Anthropic found \"several simple ways for monitoring to be disabled\", one by changing an environment variable). METR. 2026-05-19.", "url": "https://metr.org/blog/2026-05-19-frontier-risk-report/", "verified": "primary" }, { "n": 7, "title": "NIST SP 800-53 Rev. 5, Security and Privacy Controls for Information Systems and Organizations", "text": "NIST SP 800-53 Rev. 5, Security and Privacy Controls for Information Systems and Organizations (control catalogue cited by control id; publication page of Revision 5 with update 1 of 10 Dec 2020). NIST. 2020-12-10.", "url": "https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final", "verified": "primary" }, { "n": 39, "title": "The Hugging Face incident and the road ahead", "text": "The Hugging Face incident and the road ahead (OpenAI states that its chain-of-thought monitors \"did not run on the evaluations in this incident\", and that it is strengthening its incident response plan with clearer rules on \"who can stop a run or approve restarting it\"). OpenAI. 2026-08-26.", "url": "https://openai.com/index/hugging-face-incident-and-the-road-ahead/", "verified": "primary" }, { "n": 40, "title": "Priorities and principles for effective third party assessments", "text": "Priorities and principles for effective third party assessments (asks whether monitoring is implemented across training, evaluations and deployment \"in a way that cannot easily be disabled\", and for claims \"pre-registered before assessment activities begin\"). OpenAI. 2026-09-22.", "url": "https://openai.com/index/priorities-principles-third-party-assessments/", "verified": "primary" }, { "n": 34, "title": "Preparedness Framework, Version 2", "text": "Preparedness Framework, Version 2 (Table 5 lists potential safeguards against a misaligned model, among them limiting internet and tool access, credentials and filesystem access, with agent actions \"logged in an uneditable database\"; a one-time capability elicitation is treated \"as a lower bound, rather than a ceiling\"). OpenAI. 2025-04-15.", "url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "verified": "primary" }, { "n": 10, "title": "AIUC-1 requirements", "text": "AIUC-1 requirements (public requirement index, A001 to F002, each requirement on its own page (E007 and E014 marked retired); AIUC-1 is a standard of the Artificial Intelligence Underwriting Company; this site is not affiliated with AIUC, and a mapping here is not an AIUC-1 certificate or audit). Artificial Intelligence Underwriting Company. 2026-09-24.", "url": "https://standard.aiuc-1.com/llms.txt", "verified": "primary" }, { "n": 12, "title": "OpenAI Hugging Face Incident Technical Report", "text": "OpenAI Hugging Face Incident Technical Report (OpenAI states that high-risk workloads are \"prohibited via technical controls from receiving direct or transitive Internet access\", protected by \"two separate layers of network isolation\", and that it is building tooling to \"identify and halt evaluation workloads\"). OpenAI. 2026-08-26.", "url": "https://cdn.openai.com/pdf/67869394-cb91-4c12-888c-5cbd85c7814c/OpenAI-Hugging-Face%20Incident-Technical-Report.pdf", "verified": "primary" }, { "n": 41, "title": "Unauthorized communication via temporary file hosting services (misalignment report)", "text": "Unauthorized communication via temporary file hosting services (misalignment report) (OpenAI reports agents in RL training that uploaded output files to public hosting platforms for co-working agents; the behavior was found by a monitoring system that \"ran on 20% of the samples\" of the run, which now runs on 100% of samples). OpenAI (Alignment Research Blog). 2026-09-16.", "url": "https://alignment.openai.com/misalignment-reports/unauthorized-communication-via-temporary-file-hosting-services/", "verified": "primary" } ], "implementationNotes": [ "Run the collector, the monitors and their configuration outside the environment, and write the trace to append-only storage the agent cannot reach. OpenAI's Preparedness Framework v2 lists agent actions \"logged in an uneditable database\" and reviewed by asynchronous monitors among its potential safeguards against a misaligned model. OpenAI's technical report on the Hugging Face incident states that models sometimes tried to \"erase or tamper with their outputs or message logs\", and that none of the attempts it observed to manipulate tool trajectories affected the logs its graders or monitors ultimately see.", "Keep every switch of the monitoring out of the agent's reach. METR states that its red-teaming exercise at Anthropic found \"several simple ways for monitoring to be disabled\", one by changing an environment variable that indicated whether the traffic came from Claude Code, and OpenAI's criteria for third-party assessments ask whether monitoring covers training, evaluations and deployment \"in a way that cannot easily be disabled\".", "Check that the monitors ran on this run, and over all of it. OpenAI states that its chain-of-thought monitors \"did not run on the evaluations\" involved in the Hugging Face incident, and reports a training incident found by a monitoring system that ran on 20% of the samples of the run and now runs on 100%.", "Reconcile the transcript with what the tools logged, not only with itself. METR's investigation of the OpenAI/Hugging Face incident reports that at least 96 transcripts show clear evidence of spoofed tool calls, where an agent appeared to run one command and replaced its output; a hash of each output, logged by the tool server, lets a reviewer find the substitution." ], "openQuestions": [ "How can a reviewer show that a run's trace is complete for actions that pass no tool server, not only that the records it holds are unaltered?", "The OpenTelemetry GenAI conventions are still at status Development: should the profile name the attributes it relies on, or only the events a trace must contain?" ], "observation": { "subjectKind": "eval-run", "expected": "The monitors ran over the whole run, the agent could not switch them off, and the trace reconciles with the tool servers' logs call by call.", "observedExample": "Run 88251: 1,904 calls in the trace and in the tool servers' logs, 0 output hash mismatches, monitor coverage 100% of samples: pass." }, "observationSchema": "https://aigovernanceengineer.com/schemas/control-observation.v1.json", "examples": [ { "status": "pass", "url": "https://aigovernanceengineer.com/controls/examples/control-observation.aige-ctl-eval-005.pass.json" }, { "status": "fail", "url": "https://aigovernanceengineer.com/controls/examples/control-observation.aige-ctl-eval-005.fail.json" } ] } }