Recognize sign-in failures for every agent and fall through (#19)

Co-authored-by: Danny Kim <temeddix@gmail.com> Co-committed-by: Danny Kim <temeddix@gmail.com>
2026-10-12 02:51:28 +00:00
parent ea20f6d629
commit f001a71c61
11 changed files with 413 additions and 36 deletions
@@ -0,0 +1,24 @@
+---
+description: How codex, claude, and opencode report rejected credentials, and how the runner detects them
+---
+
+Observed 2026-10 (codex 0.153/0.162, Claude Code 2.1.28x, opencode 1.18/1.19):
+
+- Codex: `codex login status` only reads the local auth.json, so a
+  server-revoked ChatGPT login passes it. `codex exec` then exits 1 with stderr
+  ERROR lines such as `refresh_token_invalidated` / "Your session has ended.
+  Please log in again.", `auth error code: token_revoked`, `401 Unauthorized`; a
+  bad API key gives `invalid_api_key`. Codex echoes the whole prompt and tool
+  output to stderr, so `isCodexAuthFailure` only matches ERROR lines and the
+  runner strips the prompt first.
+- Claude: a bad `CLAUDE_CODE_OAUTH_TOKEN` yields a stream-json `result` with
+  `is_error: true`, `api_error_status: 401`, and "Failed to authenticate. API
+  Error: 401 OAuth access token is invalid."; exit code 1, empty stderr.
+- OpenCode: a bad OpenRouter key exits 1 with empty stderr and a stdout JSON
+  event
+  `{"type":"error","error":{"name":"APIError","data":{"statusCode":401,
+  "message":"User not found."}}}`.
+  An empty key gives an opaque UnknownError.
+
+Sign-in failures become `AgentAuthError` and fall through to the next profile
+like usage exhaustion; the failure comment lists only fixed labels per profile.
@@ -10,13 +10,31 @@ export function codexAuthentication(token?: string): CodexAuthentication {
    : { kind: "api-key", token };
 }

+// Codex logs revoked, expired, or reused ChatGPT sign-ins and rejected API keys
+// on its own ERROR lines; tool output and the echoed prompt must not count.
+const CODEX_AUTH_FAILURE =
+  /^(?:\S+\s+)?ERROR\b.*(?:\b401\b|unauthorized|invalid_refresh_token|refresh_token_(?:expired|reused|invalidated)|token_(?:revoked|expired)|invalid_api_key|(?:log|sign) in again)/im;
+
+export function isCodexAuthFailure(error: string): boolean {
+  return CODEX_AUTH_FAILURE.test(error);
+}
+
+// Claude Code reports a rejected CLAUDE_CODE_OAUTH_TOKEN as an is_error
+// stream-json result, e.g. "Failed to authenticate. API Error: 401 OAuth
+// access token is invalid." with api_error_status 401.
+export function isClaudeAuthFailure(
+  result: { api_error_status?: number | null; result?: string },
+): boolean {
+  return result.api_error_status === 401 ||
+    /failed to authenticate|oauth (?:access )?token (?:is invalid|has expired)|invalid api key|authentication_error|please run \/login|not logged in/i
+      .test(result.result ?? "");
+}
+
 export async function retryInvalidToken<
  T extends { status: { success: boolean }; error: string },
 >(attempt: () => Promise<T>, relogin: () => Promise<void>): Promise<T> {
  let result = await attempt();
-  if (
-    !result.status.success && result.error.includes("invalid_refresh_token")
-  ) {
+  if (!result.status.success && isCodexAuthFailure(result.error)) {
    await relogin();
    result = await attempt();
  }
@@ -1,6 +1,8 @@
 import { assertEquals, assertRejects } from "jsr:@std/assert@1";
 import {
  codexAuthentication,
+  isClaudeAuthFailure,
+  isCodexAuthFailure,
  persistCodexAuth,
  retryInvalidToken,
  stageCodexAuth,
@@ -13,7 +15,9 @@ Deno.test("retries once after an invalid refresh token", async () => {
    () =>
      Promise.resolve({
        status: { success: false },
-        error: attempts++ === 0 ? "invalid_refresh_token" : "still failed",
+        error: attempts++ === 0
+          ? "ERROR: invalid_refresh_token"
+          : "still failed",
      }),
    () => {
      relogins++;
@@ -48,6 +52,80 @@ Deno.test("does not retry an unrelated failure", async () => {
  assertEquals({ attempts, relogins }, { attempts: 1, relogins: 0 });
 });

+// Verbatim stderr lines from the revoked-login job on lead/memona-home#36.
+const REVOKED_CODEX = [
+  `2026-10-12T02:28:04.702126Z ERROR codex_models_manager::manager: failed to refresh available models: unexpected status 401 Unauthorized: Encountered invalidated oauth token for user, failing request, url: https://chatgpt.com/backend-api/codex/models?client_version=0.162.1, cf-ray: a492ab48fdefea25-ICN, request id: req_4caa104fe9184a63bb8877f42817172d, auth error: 401, auth error code: token_revoked`,
+  `2026-10-12T02:28:05.082447Z ERROR codex_login::auth::manager: Failed to refresh token status=401 Unauthorized detail=TokenErrorDetail { error_code: Some("refresh_token_invalidated"), error_message: Some("Your session has ended. Please log in again."), .. }`,
+  `2026-10-12T02:28:05.451678Z ERROR rmcp::transport::worker: worker quit with fatal: Transport channel closed, when UnexpectedServerResponse("HTTP 401: {\\n  \\"error\\": {\\n    \\"message\\": \\"Encountered invalidated oauth token for user, failing request\\",\\n    \\"type\\": null,\\n    \\"code\\": \\"token_revoked\\",\\n    \\"param\\": null\\n  },\\n  \\"status\\": 401\\n}")`,
+  "ERROR: workspace routing discovery unauthorized (401)",
+];
+
+Deno.test("recognizes each Codex sign-in failure line", () => {
+  for (
+    const line of [
+      ...REVOKED_CODEX,
+      `2026-10-12T02:28:05Z ERROR codex_login::auth::manager: Failed to refresh token status=401 Unauthorized detail=TokenErrorDetail { error_code: Some("refresh_token_expired"), .. }`,
+      `ERROR: Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again. (refresh_token_reused)`,
+      "ERROR: unexpected status 401 Unauthorized: Incorrect API key provided: sk-inval*d123., auth error: 401, auth error code: invalid_api_key",
+    ]
+  ) {
+    assertEquals(isCodexAuthFailure(`exec\n${line}\n`), true, line);
+  }
+});
+
+Deno.test("ignores auth words outside Codex error lines", () => {
+  assertEquals(
+    isCodexAuthFailure(
+      "exec\ncurl returned 401 Unauthorized\ntoken_revoked in fixture\n" +
+        "ERROR: tests failed\n",
+    ),
+    false,
+  );
+});
+
+Deno.test("re-logs in after the job 9837 revoked-session stderr", async () => {
+  let relogins = 0;
+  let attempts = 0;
+  const result = await retryInvalidToken(
+    () =>
+      Promise.resolve({
+        status: { success: attempts++ > 0 },
+        error: attempts === 1 ? REVOKED_CODEX.join("\n") : "",
+      }),
+    () => {
+      relogins++;
+      return Promise.resolve();
+    },
+  );
+
+  assertEquals({ relogins, success: result.status.success }, {
+    relogins: 1,
+    success: true,
+  });
+});
+
+Deno.test("recognizes a rejected Claude OAuth token", () => {
+  assertEquals(
+    isClaudeAuthFailure({
+      api_error_status: 401,
+      result:
+        "Failed to authenticate. API Error: 401 OAuth access token is invalid.",
+    }),
+    true,
+  );
+  assertEquals(
+    isClaudeAuthFailure({
+      result:
+        "OAuth token has expired. Please obtain a new token or refresh your existing token.",
+    }),
+    true,
+  );
+  assertEquals(
+    isClaudeAuthFailure({ api_error_status: 429, result: "API Error: 429" }),
+    false,
+  );
+});
+
 Deno.test("selects API-key login only for a configured Codex token", () => {
  assertEquals(codexAuthentication("selected-key"), {
    kind: "api-key",
@@ -1,3 +1,5 @@
+import { AgentAuthError } from "./profiles.ts";
+
 export function openCodeArgs(model: string, prompt: string): string[] {
  return [
    "run",
@@ -43,6 +45,38 @@ export async function collectOpenCodeOutput(
  return { stdout: output, stderr: error };
 }

+// OpenCode reports provider failures, including a rejected OpenRouter key
+// (401 "User not found."), as a JSON error event on stdout and exits 1 with an
+// empty stderr.
+export function openCodeError(output: string): Error | undefined {
+  let failure: Error | undefined;
+  for (const line of output.split("\n")) {
+    if (line.trim() === "") continue;
+    let event: unknown;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (typeof event !== "object" || event === null) continue;
+    const { type, error } = event as Record<string, unknown>;
+    if (type !== "error" || typeof error !== "object" || error === null) {
+      continue;
+    }
+    const { name, data } = error as Record<string, unknown>;
+    const { message, statusCode } = (data ?? {}) as Record<string, unknown>;
+    const detail = `opencode ${typeof name === "string" ? name : "error"}${
+      typeof statusCode === "number" ? ` (API error ${statusCode})` : ""
+    }: ${typeof message === "string" ? message : "unknown error"}`;
+    failure = statusCode === 401 || statusCode === 403
+      ? new AgentAuthError("opencode sign-in failed", {
+        cause: new Error(detail),
+      })
+      : new Error(detail);
+  }
+  return failure;
+}
+
 export function parseOpenCodeEvents(output: string): string {
  let result: string | undefined;
  for (const line of output.split("\n")) {
@@ -54,6 +88,9 @@ export function parseOpenCodeEvents(output: string): string {
    const text = (part as Record<string, unknown>).text;
    if (typeof text === "string" && text.trim() !== "") result = text;
  }
-  if (result === undefined) throw new Error("OpenCode produced no text output");
+  if (result === undefined) {
+    throw openCodeError(output) ??
+      new Error("OpenCode produced no text output");
+  }
  return result;
 }
@@ -1,9 +1,16 @@
-import { assertEquals, assertRejects } from "jsr:@std/assert@1";
+import {
+  assertEquals,
+  assertInstanceOf,
+  assertRejects,
+  assertThrows,
+} from "jsr:@std/assert@1";
 import {
  collectOpenCodeOutput,
  openCodeArgs,
+  openCodeError,
  parseOpenCodeEvents,
 } from "./opencode.ts";
+import { AgentAuthError } from "./profiles.ts";

 Deno.test("builds a non-interactive auto-approved OpenCode command", () => {
  assertEquals(
@@ -74,3 +81,40 @@ Deno.test("streams OpenCode output while collecting it", async () => {
  assertEquals(result, { stdout: "first\nsecond\n", stderr: "warning\n" });
  assertEquals(shown, ["out:first\n", "err:warning\n", "out:second\n"]);
 });
+
+const openRouterError = (statusCode: number, message: string): string =>
+  JSON.stringify({
+    type: "error",
+    sessionID: "ses_test",
+    error: {
+      name: "APIError",
+      data: {
+        message,
+        statusCode,
+        isRetryable: false,
+        responseBody: JSON.stringify({ error: { message, code: statusCode } }),
+      },
+    },
+  });
+
+Deno.test("reports a rejected OpenRouter key as a sign-in failure", () => {
+  const error = openCodeError(openRouterError(401, "User not found."));
+  assertInstanceOf(error, AgentAuthError);
+  assertEquals(error.message, "opencode sign-in failed");
+  assertEquals(
+    (error.cause as Error).message,
+    "opencode APIError (API error 401): User not found.",
+  );
+});
+
+Deno.test("surfaces an OpenCode error event instead of empty output", () => {
+  assertThrows(
+    () => parseOpenCodeEvents(openRouterError(429, "Rate limit exceeded")),
+    Error,
+    "API error 429",
+  );
+  assertEquals(
+    openCodeError('{"type":"text","part":{"text":"ok"}}'),
+    undefined,
+  );
+});
@@ -4,7 +4,12 @@ import {
  type RuntimeManifest,
  type RuntimeOutcome,
 } from "./runtime.ts";
-import { withProfile } from "./profiles.ts";
+import {
+  failureFooter,
+  type FailureReason,
+  type ProfileFailure,
+  withProfile,
+} from "./profiles.ts";

 export type PostContext = {
  apiUrl: string;
@@ -26,7 +31,10 @@ const VERDICTS: Record<string, [event: string, mark: string]> = {
  Approved: ["APPROVED", "✅"],
  "Changes requested": ["REQUEST_CHANGES", "🛑"],
 };
-const FAILURE = "AI bot failed.\n\n> Profile: none · all profiles failed";
+const PROFILE_TYPES = ["claude", "codex", "opencode"];
+const FAILURE_REASONS = ["sign-in failed", "usage exhausted", "failed"];
+const failureComment = (failures: ProfileFailure[]): string =>
+  `AI bot failed.\n\n${failureFooter(failures)}`;

 class GiteaRequestError extends Error {}

@@ -61,7 +69,28 @@ async function readOutcome(path: string): Promise<RuntimeOutcome> {
    throw new Error("invalid agent outcome");
  }
  const outcome = value as Record<string, unknown>;
-  if (outcome.ok === false) return { ok: false };
+  if (outcome.ok === false) {
+    // Only known labels reach the comment, never raw agent output.
+    const failures = Array.isArray(outcome.failures) ? outcome.failures : [];
+    return {
+      ok: false,
+      failures: failures.flatMap((entry: unknown) => {
+        const { type, model, reason } = (entry ?? {}) as Record<
+          string,
+          unknown
+        >;
+        return typeof type === "string" && PROFILE_TYPES.includes(type) &&
+            typeof model === "string" && typeof reason === "string" &&
+            FAILURE_REASONS.includes(reason)
+          ? [{
+            type: type as ProfileFailure["type"],
+            model,
+            reason: reason as FailureReason,
+          }]
+          : [];
+      }),
+    };
+  }
  if (
    outcome.ok !== true || typeof outcome.body !== "string" ||
    typeof outcome.reviewPath !== "string" ||
@@ -152,11 +181,11 @@ export async function postRuntime(
    try {
      outcome = await readOutcome(manifest.outcomeFile);
    } catch (error) {
-      await postComment(FAILURE);
+      await postComment(failureComment([]));
      throw new Error("agent phase failed", { cause: error });
    }
    if (!outcome.ok) {
-      await postComment(FAILURE);
+      await postComment(failureComment(outcome.failures));
      throw new Error("agent phase failed");
    }

@@ -208,7 +237,9 @@ export async function postRuntime(
      });
    } catch (error) {
      if (error instanceof GiteaRequestError) throw error;
-      await postComment(FAILURE);
+      await postComment(
+        failureComment([{ ...outcome.profile, reason: "failed" }]),
+      );
      throw error;
    }
  } finally {
@@ -10,7 +10,7 @@ const context: PostContext = {
 };

 async function fixture(
-  source: { ok: false } | { ok: true; body: string },
+  source: Extract<RuntimeOutcome, { ok: false }> | { ok: true; body: string },
  review?: string,
  anchors?: unknown,
 ): Promise<{
@@ -138,8 +138,15 @@ Deno.test("retries a rejected inline review without anchors", async () => {
  assert(await missing(result.runtimeDir));
 });

-Deno.test("reports a generic agent failure and keeps the action failed", async () => {
-  const result = await fixture({ ok: false });
+Deno.test("reports each tried profile and keeps the action failed", async () => {
+  const result = await fixture({
+    ok: false,
+    failures: [
+      { type: "codex", model: "gpt-test", reason: "sign-in failed" },
+      { type: "claude", model: "claude-test", reason: "usage exhausted" },
+      { type: "opencode", model: "openrouter/test", reason: "failed" },
+    ],
+  });
  const bodies: unknown[] = [];

  await assertRejects(
@@ -153,14 +160,47 @@ Deno.test("reports a generic agent failure and keeps the action failed", async (
  );

  assertEquals(bodies, [{
-    body: "AI bot failed.\n\n> Profile: none · all profiles failed",
+    body: "AI bot failed.\n\n> Tried: `codex` · `gpt-test` (sign-in failed), " +
+      "`claude` · `claude-test` (usage exhausted), " +
+      "`opencode` · `openrouter/test` (failed)",
  }]);
  assert(!JSON.stringify(bodies).includes("reviewer-secret"));
  assert(await missing(result.runtimeDir));
 });

+Deno.test("drops failure entries that are not known labels", async () => {
+  const result = await fixture({ ok: false, failures: [] });
+  await Deno.writeTextFile(
+    result.outcomeFile,
+    JSON.stringify({
+      ok: false,
+      failures: [
+        { type: "codex", model: "gpt-test", reason: "token sk-secret leaked" },
+        { type: "shell", model: "x", reason: "failed" },
+        { type: "claude", model: "claude-test", reason: "sign-in failed" },
+      ],
+    }),
+  );
+  const bodies: unknown[] = [];
+
+  await assertRejects(
+    () =>
+      postRuntime(result.manifestPath, context, (_url, init) => {
+        bodies.push(JSON.parse(String(init?.body)));
+        return Promise.resolve(new Response(null, { status: 201 }));
+      }),
+    Error,
+    "agent phase failed",
+  );
+
+  assertEquals(bodies, [{
+    body:
+      "AI bot failed.\n\n> Tried: `claude` · `claude-test` (sign-in failed)",
+  }]);
+});
+
 Deno.test("reports a generic failure when the runner wrote no outcome", async () => {
-  const result = await fixture({ ok: false });
+  const result = await fixture({ ok: false, failures: [] });
  await Deno.remove(result.outcomeFile);
  const bodies: unknown[] = [];

@@ -174,9 +214,7 @@ Deno.test("reports a generic failure when the runner wrote no outcome", async ()
    "agent phase failed",
  );

-  assertEquals(bodies, [{
-    body: "AI bot failed.\n\n> Profile: none · all profiles failed",
-  }]);
+  assertEquals(bodies, [{ body: "AI bot failed.\n\n> Profile: none" }]);
  assert(await missing(result.runtimeDir));
 });

@@ -252,7 +290,7 @@ Deno.test("reports an attributed failure when a pull request review is missing",
  );

  assertEquals(bodies, [{
-    body: "AI bot failed.\n\n> Profile: none · all profiles failed",
+    body: "AI bot failed.\n\n> Tried: `codex` · `gpt-test` (failed)",
  }]);
  assert(await missing(result.runtimeDir));
 });
@@ -17,26 +17,52 @@ export function withProfile(
  return `${body}\n\n${profileFooter(profile)}`;
 }

+export type FailureReason = "sign-in failed" | "usage exhausted" | "failed";
+
+export type ProfileFailure = Pick<BotProfile, "type" | "model"> & {
+  reason: FailureReason;
+};
+
+// Raised once an agent's credentials are rejected and any re-login is spent.
+export class AgentAuthError extends Error {}
+
 function isUsageExhausted(error: unknown): boolean {
  return /(?:usage|rate) limit|hit your (?:weekly )?limit|insufficient_quota|exceeded your current quota|too many requests|(?:api error|http error|http|status(?: code)?)[ :]+4(?:02|29)\b/i
    .test(String(error));
 }

+function failureReason(error: unknown): FailureReason {
+  if (error instanceof AgentAuthError) return "sign-in failed";
+  return isUsageExhausted(error) ? "usage exhausted" : "failed";
+}
+
+export function failureFooter(failures: ProfileFailure[]): string {
+  if (failures.length === 0) return "> Profile: none";
+  return `> Tried: ${
+    failures
+      .map(({ type, model, reason }) =>
+        `\`${type}\` · \`${model}\` (${reason})`
+      )
+      .join(", ")
+  }`;
+}
+
 export async function runProfiles<
  T,
  P extends Pick<BotProfile, "type" | "model">,
 >(
  profiles: P[],
  attempt: (profile: P) => Promise<T>,
+  failures: ProfileFailure[] = [],
 ): Promise<T> {
  for (const [index, profile] of profiles.entries()) {
    try {
      return await attempt(profile);
    } catch (error) {
-      if (!isUsageExhausted(error) || index === profiles.length - 1) {
-        throw error;
-      }
-      console.error(`${profile.type} usage exhausted; trying next profile`);
+      const reason = failureReason(error);
+      failures.push({ type: profile.type, model: profile.model, reason });
+      if (reason === "failed" || index === profiles.length - 1) throw error;
+      console.error(`${profile.type} ${reason}; trying next profile`);
    }
  }
  throw new Error("profiles must contain at least one profile");
@@ -1,6 +1,9 @@
 import { assertEquals, assertRejects } from "jsr:@std/assert@1";
 import {
+  AgentAuthError,
  agentEnvironment,
+  failureFooter,
+  type ProfileFailure,
  profileFooter,
  runProfiles,
  sanitizedEnvironment,
@@ -70,6 +73,59 @@ Deno.test("recognizes provider API exhaustion status", async () => {
  assertEquals(results, ["fallback", "fallback"]);
 });

+Deno.test("falls through after a sign-in failure and records each try", async () => {
+  const failures: ProfileFailure[] = [];
+  const result = await runProfiles(
+    [
+      { type: "codex", model: "gpt-5.6-terra" },
+      { type: "claude", model: "claude-sonnet-5", token: "first" },
+      { type: "opencode", model: "openrouter/qwen/qwen3.8-27b" },
+    ],
+    (profile) => {
+      if (profile.type === "codex") {
+        throw new AgentAuthError("codex sign-in failed");
+      }
+      if (profile.type === "claude") throw new Error("API Error: 429");
+      return Promise.resolve("fallback");
+    },
+    failures,
+  );
+
+  assertEquals(result, "fallback");
+  assertEquals(failures, [
+    { type: "codex", model: "gpt-5.6-terra", reason: "sign-in failed" },
+    { type: "claude", model: "claude-sonnet-5", reason: "usage exhausted" },
+  ]);
+  assertEquals(
+    failureFooter(failures),
+    "> Tried: `codex` · `gpt-5.6-terra` (sign-in failed), " +
+      "`claude` · `claude-sonnet-5` (usage exhausted)",
+  );
+});
+
+Deno.test("throws the last sign-in failure when every profile fails", async () => {
+  const failures: ProfileFailure[] = [];
+  await assertRejects(
+    () =>
+      runProfiles(
+        [
+          { type: "codex", model: "gpt-5.6-terra" },
+          { type: "claude", model: "claude-sonnet-5", token: "first" },
+        ],
+        (profile) => {
+          throw new AgentAuthError(`${profile.type} sign-in failed`);
+        },
+        failures,
+      ),
+    AgentAuthError,
+    "claude sign-in failed",
+  );
+  assertEquals(failures.map(({ reason }) => reason), [
+    "sign-in failed",
+    "sign-in failed",
+  ]);
+});
+
 Deno.test("preserves an ordinary agent failure", async () => {
  const visited: string[] = [];
  await assertRejects(
@@ -4,6 +4,8 @@ import { TextLineStream } from "jsr:@std/streams@1/text-line-stream";
 import {
  type CodexAuthentication,
  codexAuthentication,
+  isClaudeAuthFailure,
+  isCodexAuthFailure,
  persistCodexAuth,
  retryInvalidToken,
  stageCodexAuth,
@@ -14,7 +16,9 @@ import {
  readProfileFile,
 } from "./isolation.ts";
 import {
+  AgentAuthError,
  agentEnvironment,
+  type ProfileFailure,
  runProfiles,
  sanitizedEnvironment,
  withProfile,
@@ -22,6 +26,7 @@ import {
 import {
  collectOpenCodeOutput,
  openCodeArgs,
+  openCodeError,
  parseOpenCodeEvents,
 } from "./opencode.ts";
 import {
@@ -195,7 +200,12 @@ async function runClaude(
    stderr: "piped",
  }).spawn();
  let result:
-    | { is_error?: boolean; result?: string; subtype: string }
+    | {
+      is_error?: boolean;
+      result?: string;
+      subtype: string;
+      api_error_status?: number | null;
+    }
    | undefined;
  let error = "";
  const copyError = claude.stderr.pipeThrough(new TextDecoderStream()).pipeTo(
@@ -220,7 +230,13 @@ async function runClaude(
  }
  const [status] = await Promise.all([claude.status, copyError]);
  if (result?.is_error) {
-    throw new Error(result.result ?? `claude ended with ${result.subtype}`);
+    const failure = new Error(
+      result.result ?? `claude ended with ${result.subtype}`,
+    );
+    if (isClaudeAuthFailure(result)) {
+      throw new AgentAuthError("claude sign-in failed", { cause: failure });
+    }
+    throw failure;
  }
  if (result?.result === undefined) {
    throw new Error(
@@ -257,12 +273,14 @@ async function codexDeviceLogin(
      status.then(() => true),
      new Promise<boolean>((resolve) => setTimeout(() => resolve(false), 100)),
    ]);
-    if (exited) throw new Error("AI bot login produced no instructions");
+    if (exited) {
+      throw new AgentAuthError("AI bot login produced no instructions");
+    }
  }
  await postAuthorComment(withProfile(shown, profile));
  const [result] = await Promise.all([status, drained]);
  if (!result.success) {
-    throw new Error(`codex login exited with ${result.code}`);
+    throw new AgentAuthError(`codex login exited with ${result.code}`);
  }
 }

@@ -347,7 +365,8 @@ async function runCodex(
      error += decoder.decode(chunk, { stream: true });
    }
    error += decoder.decode();
-    return { status: await codex.status, error };
+    // Codex echoes the prompt, whose comments may quote auth errors.
+    return { status: await codex.status, error: error.replaceAll(prompt, "") };
  };
  const result = authentication.kind === "api-key"
    ? await attempt()
@@ -356,9 +375,13 @@ async function runCodex(
      await codexDeviceLogin(profile, environment);
    });
  if (!result.status.success) {
-    throw new Error(
+    const failure = new Error(
      result.error.trim() || `codex exited with ${result.status.code}`,
    );
+    if (isCodexAuthFailure(result.error)) {
+      throw new AgentAuthError("codex sign-in failed", { cause: failure });
+    }
+    throw failure;
  }
  return outputPath;
 }
@@ -393,7 +416,8 @@ async function runOpenCode(
  const [status, streams] = await Promise.all([process.status, output]);
  const error = streams.stderr.trim();
  if (!status.success) {
-    throw new Error(error || `opencode exited with ${status.code}`);
+    throw openCodeError(streams.stdout) ??
+      new Error(error || `opencode exited with ${status.code}`);
  }
  return parseOpenCodeEvents(streams.stdout);
 }
@@ -455,14 +479,15 @@ async function writeOutcome(outcome: RuntimeOutcome): Promise<void> {
  await Deno.chmod(MANIFEST.outcomeFile, 0o600);
 }

+const failures: ProfileFailure[] = [];
 try {
  await configureGitAuthor();
  await writeOutcome({
    ok: true,
-    ...await runProfiles(MANIFEST.profiles, runProfile),
+    ...await runProfiles(MANIFEST.profiles, runProfile, failures),
  });
 } catch (error) {
  console.error(error);
-  await writeOutcome({ ok: false });
+  await writeOutcome({ ok: false, failures });
  throw error;
 }
@@ -1,6 +1,6 @@
 import { join } from "jsr:@std/path@1";
 import { parseCredentials } from "./credentials.ts";
-import type { BotProfile } from "./profiles.ts";
+import type { BotProfile, ProfileFailure } from "./profiles.ts";

 const FIRST_PROFILE_UID = 20_000;

@@ -28,7 +28,7 @@ export type RuntimeOutcome =
    anchorsPath: string;
    profile: Pick<RuntimeProfile, "type" | "model">;
  }
-  | { ok: false };
+  | { ok: false; failures: ProfileFailure[] };

 async function writePrivate(
  path: string,