skill-eval: surface provider retries and add a --case filter
A transient overload emptied a judge reply and reported "unclear" with no way to tell the provider apart from the skill. Assisted-by: Claude Opus 5 via pi
diff --git a/ai/pi/extensions/skill-eval/README.md b/ai/pi/extensions/skill-eval/README.md
index fed98f91..531c91e3 100644
--- a/ai/pi/extensions/skill-eval/README.md
+++ b/ai/pi/extensions/skill-eval/README.md
@@ -87,18 +87,34 @@ problem in the file, so one run of the command surfaces all of them.
/skill-eval run jj --only trigger # skip the expensive half
/skill-eval run jj --repeat 3 # attempts per case, to expose flakiness
/skill-eval run jj --jobs 1 # cases in flight at once (default 3)
+/skill-eval run jj --case pitfall # only cases whose prompt or note matches
+/skill-eval run jj --case 'file show' # quote a filter containing spaces
```
The command confirms the run count first. Trigger cases cost one model run
-each; adherence cases cost two, because grading is its own run. A report lands
-in `$XDG_STATE_HOME/pi/skill-eval/reports/` and, when anything failed, the
-failures arrive as a follow-up message so the conversation can fix the text.
+each; adherence cases cost two, because grading is its own run. Use `--case`
+while authoring, so iterating on one case doesn't re-bill the ones that already
+pass. A report lands in `$XDG_STATE_HOME/pi/skill-eval/reports/` and, when
+anything failed, the failures arrive as a follow-up message so the conversation
+can fix the text.
+
+A report that opens with a provider-retry line is not a verdict on the skill.
+Overload and rate limiting inside a nested run can empty a reply, which lands as
+`unclear` rather than `fail`; re-run those cases with `--case` before believing
+them.
Trigger misses usually mean the `description` field needs the vocabulary of the
prompt that missed. False positives on negative cases mean it claims too much
ground. Adherence failures point at the body: an instruction stated once in
passing, buried under prose, or contradicted elsewhere in the file.
+Write adherence cases where the skill has to *overcome* something, not where it
+merely agrees with the model. Asking whether a strong model can restate advice
+it just read tells you little; asking it to do something its training says is
+spelled differently tells you whether the skill lands. `ai/skills/jj/evals.yml`
+is the worked example — its second half draws on `pitfalls.md`, where jj's real
+behavior contradicts a git-shaped instinct.
+
## Layout
| File | Contents |
diff --git a/ai/pi/extensions/skill-eval/index.ts b/ai/pi/extensions/skill-eval/index.ts
index d4ae4269..a42ee3e7 100644
--- a/ai/pi/extensions/skill-eval/index.ts
+++ b/ai/pi/extensions/skill-eval/index.ts
@@ -20,7 +20,7 @@ import type {
ExtensionCommandContext,
} from "@mariozechner/pi-coding-agent";
import { EVALS_FILENAME, loadEvalSuite } from "./evals.ts";
-import { formatEvalReport, runEvals, summarizeFailures, type EvalRunConfig } from "./run.ts";
+import { filterSuite, formatEvalReport, runEvals, summarizeFailures, type EvalRunConfig } from "./run.ts";
// ---------- Types ----------
@@ -390,7 +390,14 @@ export function buildReviewPrompt(
export type ParsedCommand =
| { action: "triage" }
| { action: "review"; target: string }
- | { action: "run"; target: string; repeat: number; jobs: number; only?: "trigger" | "adherence" }
+ | {
+ action: "run";
+ target: string;
+ repeat: number;
+ jobs: number;
+ only?: "trigger" | "adherence";
+ cases: string[];
+ }
| { action: "error"; message: string };
const DEFAULT_REPEAT = 1;
@@ -401,8 +408,39 @@ const DEFAULT_JOBS = 3;
* behavioral evals; anything else is treated as a review target, which keeps the
* original bare-target form working.
*/
+/** Split on whitespace, but keep quoted runs together so a filter can hold spaces. */
+export function tokenizeArgs(args: string): string[] {
+ const tokens: string[] = [];
+ let current = "";
+ let quote: '"' | "'" | null = null;
+ let started = false;
+
+ for (const char of args.trim()) {
+ if (quote) {
+ if (char === quote) quote = null;
+ else current += char;
+ continue;
+ }
+ if (char === '"' || char === "'") {
+ quote = char;
+ started = true;
+ continue;
+ }
+ if (/\s/.test(char)) {
+ if (started) tokens.push(current);
+ current = "";
+ started = false;
+ continue;
+ }
+ current += char;
+ started = true;
+ }
+ if (started) tokens.push(current);
+ return tokens;
+}
+
export function parseCommandArgs(args: string): ParsedCommand {
- const tokens = args.trim().split(/\s+/).filter(Boolean);
+ const tokens = tokenizeArgs(args);
if (tokens.length === 0) return { action: "triage" };
const [head, ...rest] = tokens;
@@ -421,9 +459,18 @@ export function parseCommandArgs(args: string): ParsedCommand {
let repeat = DEFAULT_REPEAT;
let jobs = DEFAULT_JOBS;
let only: "trigger" | "adherence" | undefined;
+ const cases: string[] = [];
for (let i = 0; i < rest.length; i++) {
const token = rest[i];
+ if (token === "--case") {
+ const value = rest[++i];
+ if (value === undefined || value.startsWith("-")) {
+ return { action: "error", message: "--case needs a substring to match against case prompts and notes." };
+ }
+ cases.push(value);
+ continue;
+ }
if (token === "--repeat" || token === "--jobs") {
const value = Number(rest[++i]);
if (!Number.isInteger(value) || value < 1) {
@@ -449,7 +496,7 @@ export function parseCommandArgs(args: string): ParsedCommand {
if (target === undefined) {
return { action: "error", message: "run needs a skill name: /skill-eval run <skill>." };
}
- return { action: "run", target, repeat, jobs, ...(only ? { only } : {}) };
+ return { action: "run", target, repeat, jobs, cases, ...(only ? { only } : {}) };
}
if (tokens.length > 1) return { action: "error", message: `Unknown subcommand "${head}".` };
@@ -569,7 +616,7 @@ function announce(ctx: ExtensionCommandContext, text: string, level: "info" | "w
async function handleRun(
pi: ExtensionAPI,
ctx: ExtensionCommandContext,
- parsed: { target: string; repeat: number; jobs: number; only?: "trigger" | "adherence" },
+ parsed: { target: string; repeat: number; jobs: number; only?: "trigger" | "adherence"; cases: string[] },
): Promise<void> {
const target = await resolveEvalTarget(ctx, parsed.target);
if (!target.ok) {
@@ -582,7 +629,7 @@ async function handleRun(
announce(ctx, loaded.errors.join("\n"), "error");
return;
}
- const suite = loaded.suite;
+ const suite = filterSuite(loaded.suite, parsed.cases);
const config = evalConfig(ctx, parsed.repeat, parsed.jobs);
if (!config) {
@@ -594,7 +641,8 @@ async function handleRun(
const adherenceCases = parsed.only === "trigger" ? 0 : suite.adherence.length;
if (triggerCases + adherenceCases === 0) {
const kind = parsed.only ? `${parsed.only} ` : "";
- announce(ctx, `${EVALS_FILENAME} for ${target.meta.name} has no ${kind}cases to run.`, "error");
+ const filtered = parsed.cases.length > 0 ? ` matching ${parsed.cases.map((c) => `"${c}"`).join(" or ")}` : "";
+ announce(ctx, `${EVALS_FILENAME} for ${target.meta.name} has no ${kind}cases${filtered} to run.`, "error");
return;
}
@@ -655,7 +703,7 @@ export default async function (pi: ExtensionAPI) {
pi.registerCommand("skill-eval", {
description:
- "triage | review <target> | run <skill> [--repeat N] [--jobs N] [--only trigger|adherence]",
+ "triage | review <target> | run <skill> [--repeat N] [--jobs N] [--only trigger|adherence] [--case <substring>]",
getArgumentCompletions: (prefix: string) => {
const items = [
{ value: "triage", label: "triage — rank skills and AGENTS.md files by trim potential" },
diff --git a/ai/pi/extensions/skill-eval/run.test.ts b/ai/pi/extensions/skill-eval/run.test.ts
index 8aa0565f..ea3ac7b5 100644
--- a/ai/pi/extensions/skill-eval/run.test.ts
+++ b/ai/pi/extensions/skill-eval/run.test.ts
@@ -8,6 +8,8 @@ import {
formatEvalReport,
loadedSkill,
mapWithConcurrency,
+ filterSuite,
+ matchesFilters,
parseJudgeVerdicts,
runEvals,
skillLoadPath,
@@ -103,9 +105,9 @@ const report: EvalReport = {
thinking: "high",
repeat: 1,
trigger: [
- { kind: "trigger", expected: "load", prompt: "calibrate the bench widget", attempts: [{ loaded: true }], passes: 1, costUsd: 0.01 },
- { kind: "trigger", expected: "load", prompt: "check widget torque", attempts: [{ loaded: false }], passes: 0, costUsd: 0.01 },
- { kind: "trigger", expected: "skip", prompt: "what is 2 + 2", attempts: [{ loaded: true }], passes: 0, costUsd: 0.01 },
+ { kind: "trigger", expected: "load", prompt: "calibrate the bench widget", attempts: [{ loaded: true, retries: [] }], passes: 1, costUsd: 0.01 },
+ { kind: "trigger", expected: "load", prompt: "check widget torque", attempts: [{ loaded: false, retries: [] }], passes: 0, costUsd: 0.01 },
+ { kind: "trigger", expected: "skip", prompt: "what is 2 + 2", attempts: [{ loaded: true, retries: [] }], passes: 0, costUsd: 0.01 },
],
adherence: [
{
@@ -114,6 +116,7 @@ const report: EvalReport = {
attempts: [
{
transcript: "[assistant] wrangle --torque 42",
+ retries: [],
expectations: [
{ expectation: "names the torque value 42", verdict: "pass", reason: "said 42" },
{ expectation: "reports newton-metres", verdict: "fail", reason: "no units given" },
@@ -153,12 +156,38 @@ test("summarizeFailures reports each miss and each unmet expectation", () => {
test("summarizeFailures says nothing when every case passed", () => {
const clean: EvalReport = {
...report,
- trigger: [{ kind: "trigger", expected: "load", prompt: "p", attempts: [{ loaded: true }], passes: 1, costUsd: 0 }],
+ trigger: [
+ { kind: "trigger", expected: "load", prompt: "p", attempts: [{ loaded: true, retries: [] }], passes: 1, costUsd: 0 },
+ ],
adherence: [],
};
assert.deepEqual(summarizeFailures(clean), []);
});
+test("formatEvalReport warns when the provider retried, so scores aren't misread", () => {
+ const retried: EvalReport = {
+ ...report,
+ trigger: [
+ {
+ kind: "trigger",
+ expected: "load",
+ prompt: "p",
+ attempts: [{ loaded: false, retries: [{ attempt: 1, maxAttempts: 3, errorMessage: "overloaded_error" }] }],
+ passes: 0,
+ costUsd: 0,
+ },
+ ],
+ adherence: [],
+ };
+ const markdown = formatEvalReport(retried);
+ assert.match(markdown, /1 provider retry during this run, first: overloaded_error/);
+ assert.match(markdown, /may reflect the provider, not the skill/);
+});
+
+test("formatEvalReport stays quiet about retries when there were none", () => {
+ assert.doesNotMatch(formatEvalReport(report), /provider retr/);
+});
+
/**
* End-to-end orchestration against a stub pi entrypoint: no API credit, but it
* exercises argument construction, sandboxing, setup scripts, and judging.
@@ -325,3 +354,31 @@ test("runEvals reports progress for every attempt", async () => {
await rm(dir, { recursive: true, force: true });
}
});
+
+test("matchesFilters looks at both the prompt and the note, case-insensitively", () => {
+ const testCase = { prompt: "Squash @ into its parent", note: "pitfall: editor hang" };
+ assert.equal(matchesFilters(testCase, []), true);
+ assert.equal(matchesFilters(testCase, ["PITFALL"]), true);
+ assert.equal(matchesFilters(testCase, ["squash @"]), true);
+ assert.equal(matchesFilters(testCase, ["rebase"]), false);
+ assert.equal(matchesFilters(testCase, ["rebase", "editor"]), true);
+});
+
+test("filterSuite narrows every section and leaves an unfiltered suite alone", () => {
+ const suite = {
+ triggerPositive: [{ prompt: "squash this" }, { prompt: "push a stack" }],
+ triggerNegative: [{ prompt: "git submodule" }],
+ adherence: [
+ { prompt: "squash into parent", expect: ["a"], tools: ["read"] },
+ { prompt: "push bottom-first", expect: ["b"], tools: ["read"] },
+ ],
+ };
+
+ const filtered = filterSuite(suite, ["squash"]);
+ assert.deepEqual(filtered.triggerPositive, [{ prompt: "squash this" }]);
+ assert.deepEqual(filtered.triggerNegative, []);
+ assert.equal(filtered.adherence.length, 1);
+ assert.equal(filtered.adherence[0].prompt, "squash into parent");
+
+ assert.deepEqual(filterSuite(suite, []), suite);
+});
diff --git a/ai/pi/extensions/skill-eval/run.ts b/ai/pi/extensions/skill-eval/run.ts
index 4fc06a7f..ae05b589 100644
--- a/ai/pi/extensions/skill-eval/run.ts
+++ b/ai/pi/extensions/skill-eval/run.ts
@@ -18,7 +18,7 @@ import { mkdtemp, mkdir, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, dirname, join, relative, resolve, sep } from "node:path";
import type { AdherenceCase, EvalSuite, TriggerCase } from "./evals.ts";
-import { formatTranscript, runSetupScript, runSubagent, type SubagentResult } from "./subagent.ts";
+import { formatTranscript, runSetupScript, runSubagent, type RetryNote, type SubagentResult } from "./subagent.ts";
export interface EvalRunConfig {
provider: string;
@@ -43,6 +43,7 @@ export interface ExpectationResult {
export interface TriggerAttempt {
loaded: boolean;
+ retries: RetryNote[];
error?: string;
}
@@ -59,6 +60,7 @@ export interface TriggerResult {
export interface AdherenceAttempt {
expectations: ExpectationResult[];
transcript: string;
+ retries: RetryNote[];
error?: string;
}
@@ -114,6 +116,23 @@ export function loadedSkill(toolCalls: { name: string; args: Record<string, unkn
});
}
+/** Case matches when any filter appears in its prompt or note, compared case-insensitively. */
+export function matchesFilters(testCase: { prompt: string; note?: string }, filters: string[]): boolean {
+ if (filters.length === 0) return true;
+ const haystack = `${testCase.prompt}\n${testCase.note ?? ""}`.toLowerCase();
+ return filters.some((filter) => haystack.includes(filter.toLowerCase()));
+}
+
+/** Narrow a suite to the cases matching `--case` filters, for iterating on one case. */
+export function filterSuite(suite: EvalSuite, filters: string[]): EvalSuite {
+ if (filters.length === 0) return suite;
+ return {
+ triggerPositive: suite.triggerPositive.filter((c) => matchesFilters(c, filters)),
+ triggerNegative: suite.triggerNegative.filter((c) => matchesFilters(c, filters)),
+ adherence: suite.adherence.filter((c) => matchesFilters(c, filters)),
+ };
+}
+
export function buildJudgePrompt(
skillName: string,
task: string,
@@ -143,10 +162,18 @@ export function buildJudgePrompt(
].join("\n");
}
-/** Pull the verdict array out of a judge reply, tolerating code fences and surrounding prose. */
-export function parseJudgeVerdicts(text: string, expectations: string[]): ExpectationResult[] {
+/**
+ * Pull the verdict array out of a judge reply, tolerating code fences and
+ * surrounding prose. `context` describes the judge run itself, so an empty or
+ * truncated reply says why instead of just "unclear".
+ */
+export function parseJudgeVerdicts(text: string, expectations: string[], context?: string): ExpectationResult[] {
const fallback = (reason: string): ExpectationResult[] =>
- expectations.map((expectation) => ({ expectation, verdict: "unclear" as Verdict, reason }));
+ expectations.map((expectation) => ({
+ expectation,
+ verdict: "unclear" as Verdict,
+ reason: context ? `${reason} (${context})` : reason,
+ }));
const start = text.indexOf("[");
const end = text.lastIndexOf("]");
@@ -201,6 +228,7 @@ async function runTriggerAttempt(
return {
attempt: {
loaded: loadedSkill(result.toolCalls, skillPath),
+ retries: result.retries,
...(result.error ? { error: result.error } : {}),
},
costUsd: result.costUsd,
@@ -213,7 +241,7 @@ async function judge(
testCase: AdherenceCase,
transcript: string,
config: EvalRunConfig,
-): Promise<{ expectations: ExpectationResult[]; costUsd: number }> {
+): Promise<{ expectations: ExpectationResult[]; costUsd: number; retries: RetryNote[] }> {
return await withSandbox("judge", async (cwd) => {
const result = await runSubagent({
cwd,
@@ -234,12 +262,27 @@ async function judge(
reason: `judge run failed: ${result.error}`,
})),
costUsd: result.costUsd,
+ retries: result.retries,
};
}
- return { expectations: parseJudgeVerdicts(result.finalText, testCase.expect), costUsd: result.costUsd };
+ return {
+ expectations: parseJudgeVerdicts(result.finalText, testCase.expect, describeJudgeRun(result)),
+ costUsd: result.costUsd,
+ retries: result.retries,
+ };
});
}
+/** Why a judge reply might be unusable: truncation, retries, or an empty response. */
+function describeJudgeRun(result: SubagentResult): string {
+ const parts = [`stopReason ${result.stopReason ?? "unknown"}`];
+ if (result.retries.length > 0) {
+ parts.push(`${result.retries.length} provider retr${result.retries.length === 1 ? "y" : "ies"}: ${result.retries[0].errorMessage}`);
+ }
+ parts.push(result.finalText.trim() ? `reply began "${firstLine(result.finalText, 60)}"` : "reply was empty");
+ return parts.join("; ");
+}
+
async function runAdherenceAttempt(
testCase: AdherenceCase,
skillName: string,
@@ -257,6 +300,7 @@ async function runAdherenceAttempt(
attempt: {
expectations: unclearAll(testCase.expect, `setup script failed: ${setup.output || "no output"}`),
transcript: "",
+ retries: [],
error: `setup script failed: ${setup.output || "no output"}`,
},
costUsd: 0,
@@ -281,7 +325,7 @@ async function runAdherenceAttempt(
} catch (error) {
const message = (error as Error).message;
return {
- attempt: { expectations: unclearAll(testCase.expect, message), transcript: "", error: message },
+ attempt: { expectations: unclearAll(testCase.expect, message), transcript: "", retries: [], error: message },
costUsd: 0,
};
}
@@ -292,6 +336,7 @@ async function runAdherenceAttempt(
attempt: {
expectations: unclearAll(testCase.expect, `run failed: ${result.error}`),
transcript,
+ retries: result.retries,
error: result.error,
},
costUsd: result.costUsd,
@@ -300,7 +345,11 @@ async function runAdherenceAttempt(
const graded = await judge(skillName, testCase, transcript, config);
return {
- attempt: { expectations: graded.expectations, transcript },
+ attempt: {
+ expectations: graded.expectations,
+ transcript,
+ retries: [...result.retries, ...graded.retries],
+ },
costUsd: result.costUsd + graded.costUsd,
};
});
@@ -442,6 +491,15 @@ export function formatEvalReport(report: EvalReport): string {
);
lines.push("");
+ const retries = [...report.trigger, ...report.adherence].flatMap((row) => row.attempts.flatMap((a) => a.retries));
+ if (retries.length > 0) {
+ lines.push(
+ `${retries.length} provider retr${retries.length === 1 ? "y" : "ies"} during this run, ` +
+ `first: ${retries[0].errorMessage}. Scores below may reflect the provider, not the skill.`,
+ );
+ lines.push("");
+ }
+
if (report.trigger.length > 0) {
lines.push(`## Trigger fidelity — ${triggerPasses}/${triggerTotal}`);
lines.push("");
diff --git a/ai/pi/extensions/skill-eval/skill-eval.test.ts b/ai/pi/extensions/skill-eval/skill-eval.test.ts
index 56fadeca..e69ce33b 100644
--- a/ai/pi/extensions/skill-eval/skill-eval.test.ts
+++ b/ai/pi/extensions/skill-eval/skill-eval.test.ts
@@ -319,7 +319,7 @@ test("buildReviewPrompt notes no usage signal for AGENTS.md", () => {
assert.match(prompt, /No usage signal available/);
});
-import { parseCommandArgs } from "./index.ts";
+import { parseCommandArgs, tokenizeArgs } from "./index.ts";
test("parseCommandArgs treats no arguments and an explicit subcommand as triage", () => {
assert.deepEqual(parseCommandArgs(""), { action: "triage" });
@@ -333,16 +333,27 @@ test("parseCommandArgs keeps the bare-target form working as a review", () => {
});
test("parseCommandArgs reads run flags and applies defaults", () => {
- assert.deepEqual(parseCommandArgs("run jj"), { action: "run", target: "jj", repeat: 1, jobs: 3 });
+ assert.deepEqual(parseCommandArgs("run jj"), { action: "run", target: "jj", repeat: 1, jobs: 3, cases: [] });
assert.deepEqual(parseCommandArgs("run jj --repeat 3 --jobs 1 --only trigger"), {
action: "run",
target: "jj",
repeat: 3,
jobs: 1,
+ cases: [],
only: "trigger",
});
});
+test("parseCommandArgs collects repeated --case filters", () => {
+ assert.deepEqual(parseCommandArgs("run jj --case pitfall --case squash"), {
+ action: "run",
+ target: "jj",
+ repeat: 1,
+ jobs: 3,
+ cases: ["pitfall", "squash"],
+ });
+});
+
test("parseCommandArgs rejects malformed input with a specific message", () => {
const cases: [string, RegExp][] = [
["run", /needs a skill name/],
@@ -351,6 +362,8 @@ test("parseCommandArgs rejects malformed input with a specific message", () => {
["run jj --jobs two", /positive integer/],
["run jj --only sideways", /trigger or adherence/],
["run jj --wat", /Unknown flag --wat/],
+ ["run jj --case", /--case needs a substring/],
+ ["run jj --case --jobs 2", /--case needs a substring/],
["triage jj", /takes no arguments/],
["review", /exactly one target/],
["review a b", /exactly one target/],
@@ -362,3 +375,21 @@ test("parseCommandArgs rejects malformed input with a specific message", () => {
if (parsed.action === "error") assert.match(parsed.message, pattern);
}
});
+
+test("tokenizeArgs keeps quoted runs together and drops the quotes", () => {
+ assert.deepEqual(tokenizeArgs("run jj --case 'Squash @ into'"), ["run", "jj", "--case", "Squash @ into"]);
+ assert.deepEqual(tokenizeArgs('run jj --case "file show"'), ["run", "jj", "--case", "file show"]);
+ assert.deepEqual(tokenizeArgs(" run jj "), ["run", "jj"]);
+ assert.deepEqual(tokenizeArgs(""), []);
+ assert.deepEqual(tokenizeArgs("run jj --case ''"), ["run", "jj", "--case", ""]);
+});
+
+test("parseCommandArgs accepts a quoted multi-word case filter", () => {
+ assert.deepEqual(parseCommandArgs("run jj --case 'Squash @ into'"), {
+ action: "run",
+ target: "jj",
+ repeat: 1,
+ jobs: 3,
+ cases: ["Squash @ into"],
+ });
+});
diff --git a/ai/pi/extensions/skill-eval/subagent.test.ts b/ai/pi/extensions/skill-eval/subagent.test.ts
index df399e9f..fe2b3ee0 100644
--- a/ai/pi/extensions/skill-eval/subagent.test.ts
+++ b/ai/pi/extensions/skill-eval/subagent.test.ts
@@ -6,8 +6,11 @@ import { join } from "node:path";
import {
buildSubagentArgs,
collectEvents,
+ extractApiError,
extractCost,
extractFinalText,
+ extractRetries,
+ extractStopReason,
extractToolCalls,
formatTranscript,
runSetupScript,
@@ -192,3 +195,46 @@ test("runSubagent kills a run that outlives its timeout", async () => {
assert.equal(result.timedOut, true);
assert.match(result.error ?? "", /exceeded 200ms/);
});
+
+test("extractRetries and extractApiError surface provider trouble inside a run", () => {
+ const events = [
+ { type: "auto_retry_start", attempt: 1, maxAttempts: 3, delayMs: 1000, errorMessage: "overloaded_error" },
+ { type: "auto_retry_start", attempt: 2, maxAttempts: 3, delayMs: 2000, errorMessage: "overloaded_error" },
+ { type: "auto_retry_end", success: false, attempt: 2, finalError: "still overloaded" },
+ { type: "agent_end", messages: [] },
+ ];
+ assert.deepEqual(extractRetries(events), [
+ { attempt: 1, maxAttempts: 3, errorMessage: "overloaded_error" },
+ { attempt: 2, maxAttempts: 3, errorMessage: "overloaded_error" },
+ ]);
+ assert.equal(extractApiError(events), "still overloaded");
+});
+
+test("extractApiError reports an errored final message and stays quiet on success", () => {
+ const errored = [
+ {
+ type: "agent_end",
+ messages: [{ role: "assistant", content: [], stopReason: "error", errorMessage: "context window exceeded" }],
+ },
+ ];
+ assert.equal(extractApiError(errored), "context window exceeded");
+ assert.equal(extractStopReason(errored), "error");
+ assert.equal(extractApiError([agentEnd]), undefined);
+ assert.equal(extractStopReason([agentEnd]), undefined);
+});
+
+test("runSubagent reports retries, stop reason, and elapsed time", async () => {
+ const script = `
+process.stdin.on("data", () => {});
+process.stdin.on("end", () => {
+ const emit = (o) => process.stdout.write(JSON.stringify(o) + "\\n");
+ emit({ type: "auto_retry_start", attempt: 1, maxAttempts: 3, delayMs: 10, errorMessage: "overloaded_error" });
+ emit({ type: "agent_end", messages: [{ role: "assistant", content: [{ type: "text", text: "done" }], stopReason: "stop", usage: { cost: { total: 0.01 } } }] });
+});
+`;
+ const result = await withFakePi(script, () => runSubagent({ ...baseOptions, cwd: "/tmp", stdin: "hi" }));
+ assert.equal(result.error, undefined);
+ assert.equal(result.stopReason, "stop");
+ assert.deepEqual(result.retries, [{ attempt: 1, maxAttempts: 3, errorMessage: "overloaded_error" }]);
+ assert.ok(result.elapsedMs >= 0);
+});
diff --git a/ai/pi/extensions/skill-eval/subagent.ts b/ai/pi/extensions/skill-eval/subagent.ts
index caab00a1..ffdce964 100644
--- a/ai/pi/extensions/skill-eval/subagent.ts
+++ b/ai/pi/extensions/skill-eval/subagent.ts
@@ -33,6 +33,12 @@ export interface SubagentToolCall {
args: Record<string, unknown>;
}
+export interface RetryNote {
+ attempt: number;
+ maxAttempts: number;
+ errorMessage: string;
+}
+
export interface SubagentResult {
exitCode: number | null;
timedOut: boolean;
@@ -41,6 +47,11 @@ export interface SubagentResult {
toolCalls: SubagentToolCall[];
finalText: string;
costUsd: number;
+ elapsedMs: number;
+ /** Provider retries pi performed inside the run, e.g. overload or rate limiting. */
+ retries: RetryNote[];
+ /** Stop reason of the run's last assistant message, useful when the reply is empty. */
+ stopReason?: string;
/** Set when the run itself failed, as opposed to the graded behavior failing. */
error?: string;
}
@@ -118,6 +129,44 @@ export function extractFinalText(events: any[]): string {
return "";
}
+export function extractRetries(events: any[]): RetryNote[] {
+ return events
+ .filter((event) => event.type === "auto_retry_start")
+ .map((event) => ({
+ attempt: event.attempt ?? 0,
+ maxAttempts: event.maxAttempts ?? 0,
+ errorMessage: String(event.errorMessage ?? "unknown provider error"),
+ }));
+}
+
+/** The provider error that ended the run, when retries were exhausted or the model reported one. */
+export function extractApiError(events: any[]): string | undefined {
+ for (let i = events.length - 1; i >= 0; i--) {
+ const event = events[i];
+ if (event.type === "auto_retry_end" && event.success === false) {
+ return String(event.finalError ?? "provider retries exhausted");
+ }
+ }
+ const messages = finalMessages(events);
+ for (let i = messages.length - 1; i >= 0; i--) {
+ const message = messages[i];
+ if (message.role !== "assistant") continue;
+ if (message.stopReason === "error" || message.stopReason === "aborted") {
+ return String(message.errorMessage ?? `run ended with stopReason ${message.stopReason}`);
+ }
+ return undefined;
+ }
+ return undefined;
+}
+
+export function extractStopReason(events: any[]): string | undefined {
+ const messages = finalMessages(events);
+ for (let i = messages.length - 1; i >= 0; i--) {
+ if (messages[i].role === "assistant") return messages[i].stopReason;
+ }
+ return undefined;
+}
+
export function extractCost(events: any[]): number {
let total = 0;
for (const message of finalMessages(events)) {
@@ -170,6 +219,7 @@ export async function runSubagent(opts: SubagentOptions): Promise<SubagentResult
delete env.PI_SESSION_FILE;
delete env.PI_SESSION_ID;
+ const startedAt = Date.now();
return await new Promise<SubagentResult>((resolve) => {
const child = spawn(command, args, { cwd: opts.cwd, env, stdio: ["pipe", "pipe", "pipe"] });
let stdout = "";
@@ -193,6 +243,8 @@ export async function runSubagent(opts: SubagentOptions): Promise<SubagentResult
if (timer) clearTimeout(timer);
opts.signal?.removeEventListener("abort", onAbort);
const events = collectEvents(stdout);
+ const apiError = extractApiError(events);
+ const stopReason = extractStopReason(events);
resolve({
exitCode,
timedOut,
@@ -201,7 +253,10 @@ export async function runSubagent(opts: SubagentOptions): Promise<SubagentResult
toolCalls: extractToolCalls(events),
finalText: extractFinalText(events),
costUsd: extractCost(events),
- ...(error ? { error } : {}),
+ elapsedMs: Date.now() - startedAt,
+ retries: extractRetries(events),
+ ...(stopReason ? { stopReason } : {}),
+ ...(error || apiError ? { error: error ?? apiError } : {}),
});
};