diff --git a/CHANGELOG.md b/CHANGELOG.md index bacf4ec..bbabb3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,12 @@ separately (currently `1.2.0`) and changes additively. - **Codex account IDs.** Codex hook capture reads `creator_account_id` and `creator_user_id` from the first line of the session file its hook input names, as `account_id` and `user_id` with source `session_meta`. They are opaque IDs - with no domain or address, and the format is not a stable interface. `auth.json` - is never read. + with no domain or address, and the format is not a stable interface. +- **Codex account domain.** Codex hook capture reads the `email` claim of `id_token` + in `auth.json` and keeps only the domain, with source `codex_auth`. That file holds + live credentials, so nothing else in it is copied, stored, logged or sent, and + `AGENTMETRY_CODEX_AUTH_DOMAIN=0` turns this off. An API-key sign-in reports no + domain. Endpoint security tools may flag a process reading that file. - **Antigravity account domain.** Antigravity hook capture reads the last sign-in from its undocumented `app_storage.json`, labelled `local_config`, so it means the account last signed in on the machine rather than proof of who made a call. diff --git a/README.md b/README.md index 830bdc1..7646207 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ These are the numbers from commands, not from a slide. That block is what `agentmetry benchmark` printed from the PyPI 0.9.4 package and from this tree. "Rules covered 13" counts rule ids named in corpus expectations. Twelve of those are published rules. The thirteenth is the experimental rule. Two published rules have no corpus case: `host-subagent-swarm-burst` and `off-hours-activity`. They are named in `apps/orchestrator/tools/generate_sigma_pack.py` so the Sigma pack still has them. -2269 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The +2294 tests collected (`pytest --collect-only -q` from `apps/orchestrator`). The [0.10.6 release PR's CI run](https://github.com/blitzcrieg1/agentmetry/actions/runs/42082231888) on Python 3.12 reported 2089 passed and 19 skipped on Linux and macOS, 2102 passed and 6 skipped on Windows, and 82.13% line coverage on Linux diff --git a/apps/dashboard/lib/identity.ts b/apps/dashboard/lib/identity.ts index 24a3328..778b616 100644 --- a/apps/dashboard/lib/identity.ts +++ b/apps/dashboard/lib/identity.ts @@ -121,7 +121,7 @@ export const AGENT_COVERAGE_LABEL: Record = { const NOT_REPORTED_SENTENCE: Record = { cursor: "No account in this window. Cursor sends one only when it knows the signed-in user.", claude: "No account in this window. The lookup is skipped when an API key, token, cloud provider or other endpoint is in use.", - codex: "No account ID in this window. Codex's IDs come from its session file, which may not exist yet on a session's first event. Events from before this was added have none.", + codex: "No account in this window. Codex's domain comes from its sign-in file, and an API-key sign-in has none. Its IDs come from its session file. Events from before this was added have neither.", antigravity: "No account in this window. Antigravity's comes from the last sign-in on this machine, in a file that can change between versions. Events from before this was added have none.", }; @@ -147,6 +147,7 @@ const SOURCE_LABEL: Record = { otel: "Sent in the agent's telemetry", local_config: "Last account signed in on this machine", session_meta: "Read from the agent's session file", + codex_auth: "Current sign-in in Codex's auth file, domain only", }; export function accountSourceLabel(source: string): string { diff --git a/apps/dashboard/tests/identity.test.ts b/apps/dashboard/tests/identity.test.ts index 036db2e..5c07221 100644 --- a/apps/dashboard/tests/identity.test.ts +++ b/apps/dashboard/tests/identity.test.ts @@ -150,7 +150,7 @@ describe("agent coverage sentences", () => { expect(cursor).toContain("Cursor"); expect(cursor).not.toContain("Codex"); const codex = agentCoverageSentence(agent("not_reported", { name: "codex" })); - expect(codex).toContain("session file"); + expect(codex).toContain("sign-in file"); expect(codex).not.toContain("Cursor"); const claude = agentCoverageSentence(agent("not_reported", { name: "claude" })); expect(claude).toContain("API key"); @@ -187,6 +187,10 @@ describe("account labels", () => { expect(accountSourceLabel("local_config")).toBe("Last account signed in on this machine"); }); + it("says Codex's sign-in file gives a domain only", () => { + expect(accountSourceLabel("codex_auth")).toContain("domain only"); + }); + it("passes through a source it does not know", () => { expect(accountSourceLabel("something_new")).toBe("something_new"); }); diff --git a/apps/orchestrator/agentmetry/core/account.py b/apps/orchestrator/agentmetry/core/account.py index 6eb56a1..3deb8d5 100644 --- a/apps/orchestrator/agentmetry/core/account.py +++ b/apps/orchestrator/agentmetry/core/account.py @@ -7,6 +7,7 @@ from __future__ import annotations +import base64 import json import os import sys @@ -27,6 +28,8 @@ #: The first line of a Codex session file also holds its base instructions. _SESSION_META_MAX_BYTES = 1024 * 1024 _WINDOWS_EXTENDED_PREFIX = chr(92) * 2 + "?" + chr(92) +_CODEX_AUTH_MAX_BYTES = 256 * 1024 +_FALSE = frozenset({"0", "false", "no", "off"}) _CLAUDE_AUTH_OVERRIDE_VARS = ( "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", @@ -231,7 +234,7 @@ def _local_antigravity_account() -> dict[str, str]: Its hook input has no account field. This one key names the account last signed in on the machine, not necessarily the one that made the call, and - the key can change between versions. Credential stores are never opened. + the key can change between versions. This file holds no credentials. """ try: path = _antigravity_storage_path() @@ -245,6 +248,42 @@ def _local_antigravity_account() -> dict[str, str]: return {} +def _codex_home() -> Path: + configured = os.environ.get("CODEX_HOME", "").strip() + return (Path(configured) if configured else Path.home() / ".codex").resolve() + + +def _codex_auth_account() -> dict[str, str]: + """Take the email domain from the identity token in Codex's `auth.json`. + + This file holds live credentials, so the handling is deliberately narrow. + Only the `email` claim of `id_token` is used and the domain is all that is + kept. Nothing else in the file is copied, stored, logged or sent, and an + error here carries no content from it. The token's signature is not checked, + so this is what the file says, not proof of who made a call. It stops on an + API-key sign-in, and AGENTMETRY_CODEX_AUTH_DOMAIN=0 turns it off. + """ + if os.environ.get("AGENTMETRY_CODEX_AUTH_DOMAIN", "").strip().lower() in _FALSE: + return {} + try: + path = _codex_home() / "auth.json" + if not path.is_file() or path.stat().st_size > _CODEX_AUTH_MAX_BYTES: + return {} + parsed = json.loads(path.read_bytes()) + if not isinstance(parsed, dict) or parsed.get("auth_mode") != "chatgpt": + return {} + tokens = parsed.get("tokens") + id_token = tokens.get("id_token") if isinstance(tokens, dict) else None + if not isinstance(id_token, str) or id_token.count(".") != 2: + return {} + part = id_token.split(".")[1] + claims = json.loads(base64.urlsafe_b64decode(part + "=" * (-len(part) % 4))) + email = claims.get("email") if isinstance(claims, dict) else None + return account_from_email(email, "codex_auth") + except (OSError, UnicodeError, ValueError, TypeError, AttributeError): + return {} + + def _codex_session_file(home: Path, transcript_path: Any, session_id: Any) -> Path | None: """Find the session file, from the path the hook named or else from its ID. @@ -274,13 +313,10 @@ def _codex_session_account(transcript_path: Any, session_id: Any = "") -> dict[s Hook input names the file, but that format is not a stable interface and can change between versions. Only a session file under Codex's own home is opened, so a crafted path cannot point the hook at another file. Nothing is - read past the first line, and `auth.json` is never touched: it holds - credentials, and a security sensor has no business opening it. + read past the first line. """ try: - configured = os.environ.get("CODEX_HOME", "").strip() - home = (Path(configured) if configured else Path.home() / ".codex").resolve() - path = _codex_session_file(home, transcript_path, session_id) + path = _codex_session_file(_codex_home(), transcript_path, session_id) if path is None: return {} with path.open("rb") as handle: @@ -311,7 +347,9 @@ def account_for_hook(source_app: str, data: dict[str, Any]) -> dict[str, str]: if source == "claude": return _local_claude_account() if source == "codex": - return _codex_session_account(data.get("transcript_path"), data.get("session_id")) + ids = _codex_session_account(data.get("transcript_path"), data.get("session_id")) + signed_in = _codex_auth_account() + return {**ids, **signed_in} if signed_in else ids if source == "antigravity": return _local_antigravity_account() return {} diff --git a/apps/orchestrator/agentmetry/core/audit/identity_view.py b/apps/orchestrator/agentmetry/core/audit/identity_view.py index 5bae02a..00843f6 100644 --- a/apps/orchestrator/agentmetry/core/audit/identity_view.py +++ b/apps/orchestrator/agentmetry/core/audit/identity_view.py @@ -166,6 +166,9 @@ def build_identity_view( ) if organization_id: record["organization_ids"].add(organization_id) + # A domain row can start from an agent that reports no IDs. + record["account_id"] = record["account_id"] or account_id + record["user_id"] = record["user_id"] or user_id record["events"] += 1 record["agents"].add(app) _widen(record, timestamp) diff --git a/apps/orchestrator/tests/test_codex_account.py b/apps/orchestrator/tests/test_codex_account.py index 63297cf..482f64d 100644 --- a/apps/orchestrator/tests/test_codex_account.py +++ b/apps/orchestrator/tests/test_codex_account.py @@ -7,6 +7,7 @@ from __future__ import annotations +import base64 import json import sys from pathlib import Path @@ -40,6 +41,144 @@ def codex_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: return home +_AUTH_SECRETS = ("synthetic-access-token", "synthetic-refresh-token", "synthetic-api-key", "Synthetic Name") + + +def _jwt(claims: dict) -> str: + def part(value: dict) -> str: + return base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip("=") + + return f"{part({'alg': 'none'})}.{part(claims)}.signature" + + +def _auth(home: Path, *, mode: str = "chatgpt", email: object = "dev@example.com", extra: dict | None = None) -> Path: + claims = {"email": email, "name": "Synthetic Name", "sub": "subject-1"} + body = { + "OPENAI_API_KEY": "synthetic-api-key", + "auth_mode": mode, + "tokens": { + "id_token": _jwt(claims), + "access_token": "synthetic-access-token", + "refresh_token": "synthetic-refresh-token", + "account_id": "token-account", + }, + **(extra or {}), + } + path = home / "auth.json" + path.write_text(json.dumps(body), encoding="utf-8") + return path + + +def test_only_the_domain_leaves_auth_json(codex_home: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("AGENTMETRY_LOG_ACCOUNT_EMAIL", raising=False) + _auth(codex_home) + block = account_for_hook("codex", {}) + assert block == {"domain": "example.com", "source": "codex_auth"} + text = json.dumps(block) + for secret in (*_AUTH_SECRETS, "dev@example.com", "subject-1", "token-account"): + assert secret not in text + + +def test_the_domain_and_the_ids_arrive_together(codex_home: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("AGENTMETRY_LOG_ACCOUNT_EMAIL", raising=False) + _auth(codex_home) + path = _session(codex_home) + assert account_for_hook("codex", {"transcript_path": str(path)}) == { + "source": "codex_auth", + "account_id": "workspace-1", + "user_id": "user-1", + "domain": "example.com", + } + + +def test_nothing_from_auth_json_reaches_the_stored_event_but_the_domain( + codex_home: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("AGENTMETRY_SOURCE_APP", "codex") + monkeypatch.delenv("AGENTMETRY_LOG_ACCOUNT_EMAIL", raising=False) + _auth(codex_home) + payload = hook.map_hook( + "PostToolUse", + {"session_id": "s1", "hook_event_name": "PostToolUse", "tool_name": "Bash", "tool_input": {"command": "git status"}}, + ) + assert payload is not None + event = build_external_canonical(ExternalIngestBody(**payload).model_dump(exclude_none=True)) + assert event["account"]["domain"] == "example.com" + for text in (json.dumps(payload), json.dumps(event)): + for secret in (*_AUTH_SECRETS, "dev@example.com", "subject-1", "token-account"): + assert secret not in text + + +def test_an_api_key_sign_in_reports_no_domain(codex_home: Path): + _auth(codex_home, mode="apikey") + assert account_for_hook("codex", {}) == {} + + +def test_it_can_be_turned_off(codex_home: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AGENTMETRY_CODEX_AUTH_DOMAIN", "0") + _auth(codex_home) + assert account_for_hook("codex", {}) == {} + + +def test_full_email_still_needs_its_own_opt_in(codex_home: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AGENTMETRY_LOG_ACCOUNT_EMAIL", "1") + _auth(codex_home) + assert account_for_hook("codex", {})["email"] == "dev@example.com" + + +@pytest.mark.parametrize( + "tokens", + [None, "text", [], {}, {"id_token": None}, {"id_token": 7}, {"id_token": "not-a-jwt"}, {"id_token": "a.%%%.c"}, {"id_token": "a.bm90IGpzb24.c"}], +) +def test_a_broken_token_leaves_the_block_absent(codex_home: Path, tokens): + (codex_home / "auth.json").write_text(json.dumps({"auth_mode": "chatgpt", "tokens": tokens}), encoding="utf-8") + assert account_for_hook("codex", {}) == {} + + +@pytest.mark.parametrize("email", [None, 7, "not-an-address", "a@b@c.example", ""]) +def test_a_bad_email_claim_leaves_the_block_absent(codex_home: Path, email): + _auth(codex_home, email=email) + assert account_for_hook("codex", {}) == {} + + +@pytest.mark.parametrize("content", ["{not json", "[]", '"text"']) +def test_a_broken_auth_file_leaves_the_block_absent(codex_home: Path, content: str): + (codex_home / "auth.json").write_text(content, encoding="utf-8") + assert account_for_hook("codex", {}) == {} + + +def test_a_huge_auth_file_is_not_read(codex_home: Path): + _auth(codex_home, extra={"pad": "x" * (256 * 1024 + 10)}) + assert account_for_hook("codex", {}) == {} + + +def test_codex_joins_the_personal_section_with_its_ids_kept(): + def row(app: str, **kw) -> dict: + return { + "source_app": app, + "action_type": "tool_called", + "correlation_id": app, + "timestamp_utc": "2026-11-07T10:00:00+00:00", + "adapter": f"{app}_hook", + "domain": "gmail.com", + **kw, + } + + view = build_identity_view( + [ + row("cursor", account_source="hook_payload"), + row("codex", account_source="codex_auth", account_id="workspace-1", user_id="user-1"), + ], + days=7, + ) + assert len(view["accounts"]) == 1 + account = view["accounts"][0] + assert account["kind"] == "personal" + assert account["agents"] == ["codex", "cursor"] + assert account["account_id"] == "workspace-1" + assert account["user_id"] == "user-1" + + def test_the_ids_come_from_the_session_file(codex_home: Path): path = _session(codex_home) assert account_for_hook("codex", {"transcript_path": str(path)}) == { diff --git a/docs/agentmetry-event-schema.md b/docs/agentmetry-event-schema.md index 4c88965..54ee315 100644 --- a/docs/agentmetry-event-schema.md +++ b/docs/agentmetry-event-schema.md @@ -142,7 +142,7 @@ schema version remains `1.2.0`. ``` `domain` is lowercased and bounded. `source` identifies the reporting surface: -`hook_payload`, `otel`, `local_config` or `session_meta`. `organization_id` is +`hook_payload`, `otel`, `local_config`, `session_meta` or `codex_auth`. `organization_id` is included only when the agent reports one. `account_id` and `user_id` are opaque IDs that name no domain or address. The full `email` key is absent unless `AGENTMETRY_LOG_ACCOUNT_EMAIL=1` is explicitly set. That opt-in is personal @@ -162,9 +162,12 @@ whose first line carries `creator_account_id` and `creator_user_id`. If the path is missing or unusable, the file is found by session ID in its name. Those arrive as `account_id` and `user_id` with source `session_meta`. That format is not a stable interface and can change between versions, and only a file under Codex's -home (`CODEX_HOME`, else `~/.codex`) is opened. `auth.json` is never read, because -it holds credentials. Codex therefore reports no domain, so a company matches its -account IDs against its own list. Antigravity's hook input has no account field, +home (`CODEX_HOME`, else `~/.codex`) is opened. Its domain comes from `auth.json`, +which holds live credentials: only the `email` claim of `id_token` is read, only the +domain is kept with source `codex_auth`, and nothing else in the file is copied, +stored, logged or sent. The token's signature is not checked, an API-key sign-in +reports no domain, and `AGENTMETRY_CODEX_AUTH_DOMAIN=0` turns this off. Endpoint +security tools may flag a process reading that file. Antigravity's hook input has no account field, so its domain comes from `jetski.onboarding.lastLoginUsername` in its undocumented `app_storage.json`, labelled `local_config`. Like Claude's, that means the account last signed in on the machine, and the key can change between versions. Only