Assisted-by: Claude Opus 5 via pi

diff --git a/ai/pi/extensions/skill-eval/README.md b/ai/pi/extensions/skill-eval/README.md
index 975773ab..49d34d8f 100644
--- a/ai/pi/extensions/skill-eval/README.md
+++ b/ai/pi/extensions/skill-eval/README.md
@@ -110,9 +110,18 @@ cheapest trim signal the run produces, and it cuts both ways: either the file
 isn't earning its place, or no case reaches the behavior it covers.
 
 A report that opens with a provider-retry line is not a verdict on the skill.
-Overload and rate limiting inside a nested run can empty a reply, which lands as
-`unclear` rather than `fail`; re-run those cases with `--case` before believing
-them.
+Overload, rate limiting, or the five-minute per-run timeout can kill a nested
+run before anything reaches the judge. Those cases report once as `run error`,
+list their expectations as `ungraded` rather than `fail`, and print the tail of
+the transcript under "Where it stopped" so you can tell a thrashing agent from
+one that just ran out of clock. When every failure in a run is a dead run, the
+follow-up asks for a re-run instead of edits — unproven is not disproven, and a
+timeout says nothing about the text. Re-run with `--case` before believing any
+of it.
+
+`ungraded` and `unclear` are different: `ungraded` means no verdict exists,
+while `unclear` means the judge read the transcript and it didn't settle the
+expectation. Only the second is a signal about the skill.
 
 Trigger misses usually mean the `description` field needs the vocabulary of the
 prompt that missed. False positives on negative cases mean it claims too much
diff --git a/ai/pi/extensions/skill-eval/run.test.ts b/ai/pi/extensions/skill-eval/run.test.ts
index f45dfdfc..923cf171 100644
--- a/ai/pi/extensions/skill-eval/run.test.ts
+++ b/ai/pi/extensions/skill-eval/run.test.ts
@@ -9,6 +9,7 @@ import {
   buildTrimFollowUp,
   bundledReads,
   formatEvalReport,
+  hasGradedFailure,
   listBundledFiles,
   loadedSkill,
   mapWithConcurrency,
@@ -18,6 +19,7 @@ import {
   runEvals,
   skillLoadPath,
   summarizeFailures,
+  transcriptTail,
   unusedBundledFiles,
   type EvalReport,
 } from "./run.ts";
@@ -232,6 +234,105 @@ test("formatEvalReport skips trim signals for a skill with no bundled files", ()
   assert.doesNotMatch(formatEvalReport({ ...report, bundledFiles: [] }), /Trim signals/);
 });
 
+// A run killed by the timeout: every expectation carries the same reason and
+// nothing reached the judge.
+const deadReason = "run failed: run exceeded 300000ms and was killed";
+const dead: EvalReport = {
+  ...report,
+  trigger: [],
+  adherence: [
+    {
+      kind: "adherence",
+      prompt: "calibrate it and report the torque",
+      attempts: [
+        {
+          transcript: "[assistant] reading torque.md\n[tool] read torque.md\n[assistant] checking the bench",
+          reads: ["torque.md"],
+          retries: [],
+          error: "run exceeded 300000ms and was killed",
+          expectations: [
+            { expectation: "names the torque value 42", verdict: "unclear", reason: deadReason },
+            { expectation: "reports newton-metres", verdict: "unclear", reason: deadReason },
+          ],
+        },
+      ],
+      passes: 0,
+      costUsd: 0.05,
+    },
+  ],
+};
+
+test("transcriptTail keeps the end and says how much it dropped", () => {
+  const long = Array.from({ length: 25 }, (_, i) => `line ${i + 1}`).join("\n");
+  const tail = transcriptTail(long, 5);
+  assert.equal(tail[0], "…20 earlier line(s) elided");
+  assert.equal(tail.at(-1), "line 25");
+  assert.deepEqual(transcriptTail("only\n\n", 5), ["only"]);
+  assert.deepEqual(transcriptTail("", 5), []);
+});
+
+test("summarizeFailures collapses a dead run into one note instead of one per expectation", () => {
+  const notes = summarizeFailures(dead);
+  assert.equal(notes.length, 1);
+  assert.match(notes[0], /No verdict on "calibrate it and report the torque"/);
+  assert.match(notes[0], /2 expectation\(s\) ungraded/);
+  assert.match(notes[0], /says nothing about the skill/);
+});
+
+test("summarizeFailures does not call a dead trigger run a description miss", () => {
+  const deadTrigger: EvalReport = {
+    ...report,
+    adherence: [],
+    trigger: [
+      {
+        kind: "trigger",
+        expected: "load",
+        prompt: "check widget torque",
+        attempts: [{ loaded: false, reads: [], retries: [], error: "run exceeded 300000ms and was killed" }],
+        passes: 0,
+        costUsd: 0.01,
+      },
+    ],
+  };
+  const notes = summarizeFailures(deadTrigger);
+  assert.equal(notes.length, 1);
+  assert.match(notes[0], /No verdict on "check widget torque"/);
+  assert.doesNotMatch(notes[0], /did not fire/);
+});
+
+test("hasGradedFailure separates real verdicts from dead runs", () => {
+  assert.equal(hasGradedFailure(report), true);
+  assert.equal(hasGradedFailure(dead), false);
+  assert.equal(hasGradedFailure(clean), false);
+});
+
+test("formatEvalReport prints a dead run once and shows where it stopped", () => {
+  const markdown = formatEvalReport(dead);
+  assert.match(markdown, /- run error: run exceeded 300000ms and was killed/);
+  assert.match(markdown, /2 expectation\(s\) ungraded, so this case says nothing about the skill/);
+  assert.match(markdown, /  - ungraded: names the torque value 42/);
+  assert.match(markdown, /Where it stopped:/);
+  assert.match(markdown, /\[assistant\] checking the bench/);
+  // The identical reason should not be repeated once per expectation.
+  assert.equal(markdown.match(/run exceeded 300000ms/g)?.length, 1);
+});
+
+test("formatEvalReport says so when a dead run produced no output at all", () => {
+  const silent: EvalReport = {
+    ...dead,
+    adherence: [{ ...dead.adherence[0], attempts: [{ ...dead.adherence[0].attempts[0], transcript: "" }] }],
+  };
+  assert.match(formatEvalReport(silent), /The run produced no output before it died\./);
+});
+
+test("buildFailureFollowUp asks for a re-run when nothing was graded", () => {
+  const message = buildFailureFollowUp(dead, summarizeFailures(dead), "/reports/eval.md");
+  assert.match(message, /unproven, not\ndisproven/);
+  assert.match(message, /--case <filter>` before/);
+  assert.match(message, /a timeout is not a\nverdict on the text/);
+  assert.doesNotMatch(message, /propose specific edits/);
+});
+
 test("buildFailureFollowUp asks for cuts before additions", () => {
   const message = buildFailureFollowUp(report, summarizeFailures(report), "/reports/eval.md");
   assert.match(message, /# Eval failures: widget-wrangler/);
diff --git a/ai/pi/extensions/skill-eval/run.ts b/ai/pi/extensions/skill-eval/run.ts
index caa09782..a7df7ec3 100644
--- a/ai/pi/extensions/skill-eval/run.ts
+++ b/ai/pi/extensions/skill-eval/run.ts
@@ -529,6 +529,21 @@ function firstLine(text: string, max = 80): string {
   return line.length > max ? `${line.slice(0, max - 1)}…` : line;
 }
 
+/**
+ * The end of a transcript from a run that died. A killed run stalls at its last
+ * action, so the tail is the diagnostic — it separates an agent thrashing on
+ * tool calls from one that simply ran out of clock.
+ */
+export function transcriptTail(transcript: string, maxLines = 20): string[] {
+  const all = transcript.split("\n");
+  while (all.length > 0 && all.at(-1)!.trim() === "") all.pop();
+  const tail = all.slice(-maxLines);
+  while (tail.length > 0 && tail[0].trim() === "") tail.shift();
+  if (tail.length === 0) return [];
+  const elided = all.length - tail.length;
+  return elided > 0 ? [`…${elided} earlier line(s) elided`, ...tail] : tail;
+}
+
 /** Bundled references no attempt opened — the cheapest trim candidates in the skill. */
 export function unusedBundledFiles(report: EvalReport): string[] {
   const opened = new Set<string>();
@@ -602,7 +617,33 @@ export function formatEvalReport(report: EvalReport): string {
       lines.push("");
       row.attempts.forEach((attempt, i) => {
         if (row.attempts.length > 1) lines.push(`Attempt ${i + 1}:`);
-        if (attempt.error) lines.push(`- run error: ${attempt.error}`);
+        // A failed run grades nothing, so every expectation carries the same
+        // reason. Print it once: repeating it per expectation reads as several
+        // independent findings instead of one dead run.
+        if (attempt.error) {
+          const count = attempt.expectations.length;
+          lines.push(`- run error: ${attempt.error}`);
+          lines.push(
+            `- ${count} expectation(s) ungraded, so this case says nothing about the skill. Re-run it before ` +
+              `changing the text.`,
+          );
+          for (const expectation of attempt.expectations) {
+            lines.push(`  - ungraded: ${expectation.expectation}`);
+          }
+          const tail = transcriptTail(attempt.transcript);
+          lines.push("");
+          if (tail.length > 0) {
+            lines.push("Where it stopped:");
+            lines.push("");
+            lines.push("```");
+            lines.push(...tail);
+            lines.push("```");
+          } else {
+            lines.push("The run produced no output before it died.");
+          }
+          lines.push("");
+          return;
+        }
         for (const expectation of attempt.expectations) {
           lines.push(`- ${expectation.verdict}: ${expectation.expectation} — ${expectation.reason}`);
         }
@@ -629,11 +670,40 @@ export function formatEvalReport(report: EvalReport): string {
   return lines.join("\n");
 }
 
+/**
+ * Whether any failure in the report is a verdict on the skill rather than a dead
+ * run. A killed or errored run fails every expectation under it for the same
+ * reason, which says nothing about the text.
+ */
+export function hasGradedFailure(report: EvalReport): boolean {
+  for (const row of report.trigger) {
+    if (row.passes === row.attempts.length) continue;
+    if (row.attempts.some((a) => !a.error)) return true;
+  }
+  for (const row of report.adherence) {
+    for (const attempt of row.attempts) {
+      if (attempt.error) continue;
+      if (attempt.expectations.some((e) => e.verdict !== "pass")) return true;
+    }
+  }
+  return false;
+}
+
 /** Failure detail worth pulling into the conversation so the skill text can be fixed. */
 export function summarizeFailures(report: EvalReport): string[] {
   const notes: string[] = [];
   for (const row of report.trigger) {
     if (row.passes === row.attempts.length) continue;
+    const errored = row.attempts.filter((a) => a.error);
+    // A run that died never chose whether to load the skill, so reporting it as
+    // a trigger miss invents a verdict the run didn't reach.
+    if (errored.length === row.attempts.length) {
+      notes.push(
+        `No verdict on "${firstLine(row.prompt, 120)}": the run failed (${errored[0].error}). ` +
+          `Re-run this case; it says nothing about the description.`,
+      );
+      continue;
+    }
     const loads = row.attempts.filter((a) => a.loaded).length;
     notes.push(
       row.expected === "load"
@@ -643,6 +713,16 @@ export function summarizeFailures(report: EvalReport): string[] {
   }
   for (const row of report.adherence) {
     for (const attempt of row.attempts) {
+      // One note per dead run, not one per expectation: the reason is identical
+      // across them, and repeating it reads as several independent findings.
+      if (attempt.error) {
+        const count = attempt.expectations.length;
+        notes.push(
+          `No verdict on "${firstLine(row.prompt, 80)}": the run failed (${attempt.error}), leaving ` +
+            `${count} expectation(s) ungraded. Re-run this case; it says nothing about the skill.`,
+        );
+        continue;
+      }
       for (const expectation of attempt.expectations) {
         if (expectation.verdict === "pass") continue;
         notes.push(
@@ -660,6 +740,22 @@ export function summarizeFailures(report: EvalReport): string[] {
  * ones.
  */
 export function buildFailureFollowUp(report: EvalReport, failures: string[], reportPath: string): string {
+  // With nothing graded, asking for edits invites rewriting the skill against a
+  // blank transcript. Ask for a re-run instead.
+  const ask = hasGradedFailure(report)
+    ? [
+        "Read the skill and propose specific edits to its text that would fix these, or say why a case is",
+        "wrong about the skill rather than the skill being wrong.",
+        "",
+        "Reach the fix by sharpening or cutting what's already there before adding anything. If a failure",
+        "genuinely needs new instruction, name something else in the file to delete alongside it.",
+      ]
+    : [
+        "Every failure above is a run that died, so nothing was graded and the skill text is unproven, not",
+        `disproven. Re-run the affected cases with \`/skill-eval run ${report.skillName} --case <filter>\` before`,
+        "changing a word of the skill. If they keep dying, fix the case or the harness — a timeout is not a",
+        "verdict on the text.",
+      ];
   return [
     `# Eval failures: ${report.skillName}`,
     "",
@@ -667,11 +763,7 @@ export function buildFailureFollowUp(report: EvalReport, failures: string[], rep
     "",
     ...failures.map((note) => `- ${note}`),
     "",
-    "Read the skill and propose specific edits to its text that would fix these, or say why a case is",
-    "wrong about the skill rather than the skill being wrong.",
-    "",
-    "Reach the fix by sharpening or cutting what's already there before adding anything. If a failure",
-    "genuinely needs new instruction, name something else in the file to delete alongside it.",
+    ...ask,
   ].join("\n");
 }