/**
 * Behavioral evals: run a skill's text through nested pi processes.
 *
 * Trigger cases measure whether the description makes the agent reach for the
 * skill, so the run keeps ordinary skill discovery on (competing descriptions
 * are part of the test) and allows only `read`, then checks whether the skill
 * files were read.
 *
 * Adherence cases force the skill in with `/skill:<name>`, let the agent work
 * in a scratch directory, and hand the transcript to a judge run that grades
 * each declared expectation.
 *
 * Runs are hermetic apart from globally discovered skills: no context files, no
 * extensions, no session, and a fresh sandbox directory per case.
 *
 * Scores are the point, but the run also feeds trimming: it records which
 * bundled files anything opened, and a clean suite is the license to cut.
 */

import { mkdtemp, mkdir, readdir, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, dirname, join, relative, resolve, sep } from "node:path";
import { EVALS_FILENAME, type AdherenceCase, type EvalSuite, type TriggerCase } from "./evals.ts";
import { formatTranscript, runSetupScript, runSubagent, type RetryNote, type SubagentResult } from "./subagent.ts";

export interface EvalRunConfig {
  provider: string;
  model: string;
  thinking?: string;
  /** Attempts per case. Repeats surface flaky trigger behavior. */
  repeat: number;
  /** Cases in flight at once. */
  jobs: number;
  timeoutMs: number;
  signal?: AbortSignal;
  onProgress?: (done: number, total: number, label: string) => void;
}

export type Verdict = "pass" | "fail" | "unclear";

export interface ExpectationResult {
  expectation: string;
  verdict: Verdict;
  reason: string;
}

export interface TriggerAttempt {
  loaded: boolean;
  /** Bundled files this attempt opened, relative to the skill directory. */
  reads: string[];
  retries: RetryNote[];
  error?: string;
}

export interface TriggerResult {
  kind: "trigger";
  expected: "load" | "skip";
  prompt: string;
  note?: string;
  attempts: TriggerAttempt[];
  passes: number;
  costUsd: number;
}

export interface AdherenceAttempt {
  expectations: ExpectationResult[];
  transcript: string;
  /** Bundled files this attempt opened, relative to the skill directory. */
  reads: string[];
  retries: RetryNote[];
  error?: string;
}

export interface AdherenceResult {
  kind: "adherence";
  prompt: string;
  note?: string;
  attempts: AdherenceAttempt[];
  passes: number;
  costUsd: number;
}

export interface EvalReport {
  skillName: string;
  skillPath: string;
  model: string;
  thinking?: string;
  repeat: number;
  /** Files bundled beside SKILL.md, relative to its directory. Empty for a bare `.md` skill. */
  bundledFiles: string[];
  trigger: TriggerResult[];
  adherence: AdherenceResult[];
  costUsd: number;
  runs: number;
  startedAt: Date;
  finishedAt: Date;
}

/**
 * What to hand `--skill`, so the target loads even when it isn't discoverable
 * from the sandbox. Pi accepts either a file or a directory, and a bare `.md`
 * skill has no directory of its own.
 */
export function skillLoadPath(skillPath: string): string {
  return basename(skillPath) === "SKILL.md" ? dirname(skillPath) : skillPath;
}

/**
 * Which bundled references a run opened, relative to the skill directory. A
 * bare `.md` skill shares its directory with unrelated skills, so it has no
 * bundle to attribute reads to.
 */
export function bundledReads(
  toolCalls: { name: string; args: Record<string, unknown> }[],
  skillPath: string,
): string[] {
  if (basename(skillPath) !== "SKILL.md") return [];
  const bundleDir = resolve(dirname(skillPath));
  const reads = new Set<string>();
  for (const call of toolCalls) {
    if (call.name !== "read") continue;
    const raw = call.args?.path;
    if (typeof raw !== "string") continue;
    const rel = relative(bundleDir, resolve(raw));
    if (rel === "" || rel.startsWith("..")) continue;
    reads.add(rel.split(sep).join("/"));
  }
  return [...reads].sort();
}

/** True when the run read the skill itself, or one of its bundled references. */
export function loadedSkill(toolCalls: { name: string; args: Record<string, unknown> }[], skillPath: string): boolean {
  const target = resolve(skillPath);
  const readTarget = toolCalls.some(
    (call) => call.name === "read" && typeof call.args?.path === "string" && resolve(call.args.path) === target,
  );
  return readTarget || bundledReads(toolCalls, skillPath).length > 0;
}

/**
 * Files bundled beside SKILL.md, as trim candidates: a reference no case ever
 * opens is either dead weight or a gap in the suite. `evals.yml` is the suite
 * itself, so it never counts.
 */
export async function listBundledFiles(skillPath: string): Promise<string[]> {
  if (basename(skillPath) !== "SKILL.md") return [];
  const root = resolve(dirname(skillPath));

  const walk = async (dir: string): Promise<string[]> => {
    let entries;
    try {
      entries = await readdir(dir, { withFileTypes: true });
    } catch {
      return [];
    }
    const found: string[] = [];
    for (const entry of entries) {
      if (entry.name.startsWith(".")) continue;
      const full = join(dir, entry.name);
      if (entry.isDirectory()) found.push(...(await walk(full)));
      else if (entry.isFile()) found.push(relative(root, full).split(sep).join("/"));
    }
    return found;
  };

  const files = await walk(root);
  return files.filter((file) => file !== "SKILL.md" && file !== EVALS_FILENAME).sort();
}

/** Case matches when any filter appears in its prompt or note, compared case-insensitively. */
export function matchesFilters(testCase: { prompt: string; note?: string }, filters: string[]): boolean {
  if (filters.length === 0) return true;
  const haystack = `${testCase.prompt}\n${testCase.note ?? ""}`.toLowerCase();
  return filters.some((filter) => haystack.includes(filter.toLowerCase()));
}

/** Narrow a suite to the cases matching `--case` filters, for iterating on one case. */
export function filterSuite(suite: EvalSuite, filters: string[]): EvalSuite {
  if (filters.length === 0) return suite;
  return {
    triggerPositive: suite.triggerPositive.filter((c) => matchesFilters(c, filters)),
    triggerNegative: suite.triggerNegative.filter((c) => matchesFilters(c, filters)),
    adherence: suite.adherence.filter((c) => matchesFilters(c, filters)),
  };
}

export function buildJudgePrompt(
  skillName: string,
  task: string,
  transcript: string,
  expectations: string[],
): string {
  return [
    `You are grading one run of a coding agent that was given the "${skillName}" skill and then a task.`,
    "",
    "Grade each expectation independently against the transcript. Judge only what the transcript",
    'shows: "pass" when the transcript clearly meets the expectation, "fail" when it clearly does not,',
    '"unclear" when the transcript does not settle it. Do not credit intentions the agent never acted on.',
    "",
    "Task given to the agent:",
    task,
    "",
    "Expectations:",
    ...expectations.map((item, i) => `${i + 1}. ${item}`),
    "",
    "Transcript:",
    "<<<TRANSCRIPT",
    transcript || "(the agent produced no output)",
    "TRANSCRIPT",
    "",
    "Reply with only a JSON array, one object per expectation, in the same order:",
    '[{"index": 1, "verdict": "pass" | "fail" | "unclear", "reason": "one sentence"}]',
  ].join("\n");
}

/**
 * Pull the verdict array out of a judge reply, tolerating code fences and
 * surrounding prose. `context` describes the judge run itself, so an empty or
 * truncated reply says why instead of just "unclear".
 */
export function parseJudgeVerdicts(text: string, expectations: string[], context?: string): ExpectationResult[] {
  const fallback = (reason: string): ExpectationResult[] =>
    expectations.map((expectation) => ({
      expectation,
      verdict: "unclear" as Verdict,
      reason: context ? `${reason} (${context})` : reason,
    }));

  const start = text.indexOf("[");
  const end = text.lastIndexOf("]");
  if (start === -1 || end <= start) return fallback("judge did not return a JSON array");

  let parsed: unknown;
  try {
    parsed = JSON.parse(text.slice(start, end + 1));
  } catch {
    return fallback("judge returned unparseable JSON");
  }
  if (!Array.isArray(parsed)) return fallback("judge did not return a JSON array");

  return expectations.map((expectation, i) => {
    const entry: any = parsed.find((item: any) => Number(item?.index) === i + 1) ?? parsed[i];
    const verdict: Verdict =
      entry?.verdict === "pass" || entry?.verdict === "fail" || entry?.verdict === "unclear"
        ? entry.verdict
        : "unclear";
    const reason = typeof entry?.reason === "string" ? entry.reason : "judge gave no reason";
    return { expectation, verdict, reason };
  });
}

async function withSandbox<T>(prefix: string, fn: (dir: string) => Promise<T>): Promise<T> {
  const dir = await mkdtemp(join(tmpdir(), `skill-eval-${prefix}-`));
  try {
    return await fn(dir);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
}

async function runTriggerAttempt(
  testCase: TriggerCase,
  skillPath: string,
  config: EvalRunConfig,
): Promise<{ attempt: TriggerAttempt; costUsd: number }> {
  return await withSandbox("trigger", async (cwd) => {
    const result = await runSubagent({
      cwd,
      provider: config.provider,
      model: config.model,
      thinking: config.thinking,
      tools: ["read"],
      skillDirs: [skillLoadPath(skillPath)],
      discoverSkills: true,
      stdin: testCase.prompt,
      timeoutMs: config.timeoutMs,
      signal: config.signal,
    });
    return {
      attempt: {
        loaded: loadedSkill(result.toolCalls, skillPath),
        reads: bundledReads(result.toolCalls, skillPath),
        retries: result.retries,
        ...(result.error ? { error: result.error } : {}),
      },
      costUsd: result.costUsd,
    };
  });
}

async function judge(
  skillName: string,
  testCase: AdherenceCase,
  transcript: string,
  config: EvalRunConfig,
): Promise<{ expectations: ExpectationResult[]; costUsd: number; retries: RetryNote[] }> {
  return await withSandbox("judge", async (cwd) => {
    const result = await runSubagent({
      cwd,
      provider: config.provider,
      model: config.model,
      thinking: config.thinking,
      tools: [],
      discoverSkills: false,
      stdin: buildJudgePrompt(skillName, testCase.prompt, transcript, testCase.expect),
      timeoutMs: config.timeoutMs,
      signal: config.signal,
    });
    if (result.error) {
      return {
        expectations: testCase.expect.map((expectation) => ({
          expectation,
          verdict: "unclear" as Verdict,
          reason: `judge run failed: ${result.error}`,
        })),
        costUsd: result.costUsd,
        retries: result.retries,
      };
    }
    return {
      expectations: parseJudgeVerdicts(result.finalText, testCase.expect, describeJudgeRun(result)),
      costUsd: result.costUsd,
      retries: result.retries,
    };
  });
}

/** Why a judge reply might be unusable: truncation, retries, or an empty response. */
function describeJudgeRun(result: SubagentResult): string {
  const parts = [`stopReason ${result.stopReason ?? "unknown"}`];
  if (result.retries.length > 0) {
    parts.push(`${result.retries.length} provider retr${result.retries.length === 1 ? "y" : "ies"}: ${result.retries[0].errorMessage}`);
  }
  parts.push(result.finalText.trim() ? `reply began "${firstLine(result.finalText, 60)}"` : "reply was empty");
  return parts.join("; ");
}

async function runAdherenceAttempt(
  testCase: AdherenceCase,
  skillName: string,
  skillPath: string,
  config: EvalRunConfig,
): Promise<{ attempt: AdherenceAttempt; costUsd: number }> {
  return await withSandbox("adherence", async (cwd) => {
    const work = join(cwd, "work");
    await mkdir(work, { recursive: true });

    if (testCase.setup) {
      const setup = await runSetupScript(testCase.setup, work);
      if (!setup.ok) {
        return {
          attempt: {
            expectations: unclearAll(testCase.expect, `setup script failed: ${setup.output || "no output"}`),
            transcript: "",
            reads: [],
            retries: [],
            error: `setup script failed: ${setup.output || "no output"}`,
          },
          costUsd: 0,
        };
      }
    }

    let result: SubagentResult;
    try {
      result = await runSubagent({
        cwd: work,
        provider: config.provider,
        model: config.model,
        thinking: config.thinking,
        tools: testCase.tools,
        skillDirs: [skillLoadPath(skillPath)],
        discoverSkills: false,
        messages: [`/skill:${skillName}`, testCase.prompt],
        timeoutMs: config.timeoutMs,
        signal: config.signal,
      });
    } catch (error) {
      const message = (error as Error).message;
      return {
        attempt: { expectations: unclearAll(testCase.expect, message), transcript: "", reads: [], retries: [], error: message },
        costUsd: 0,
      };
    }

    const transcript = formatTranscript(result.events);
    if (result.error) {
      return {
        attempt: {
          expectations: unclearAll(testCase.expect, `run failed: ${result.error}`),
          transcript,
          reads: bundledReads(result.toolCalls, skillPath),
          retries: result.retries,
          error: result.error,
        },
        costUsd: result.costUsd,
      };
    }

    const graded = await judge(skillName, testCase, transcript, config);
    return {
      attempt: {
        expectations: graded.expectations,
        transcript,
        reads: bundledReads(result.toolCalls, skillPath),
        retries: [...result.retries, ...graded.retries],
      },
      costUsd: result.costUsd + graded.costUsd,
    };
  });
}

function unclearAll(expectations: string[], reason: string): ExpectationResult[] {
  return expectations.map((expectation) => ({ expectation, verdict: "unclear" as Verdict, reason }));
}

/** Run tasks with bounded concurrency, preserving input order in the results. */
export async function mapWithConcurrency<T, R>(
  items: T[],
  limit: number,
  fn: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
  const results = new Array<R>(items.length);
  let next = 0;
  const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
    while (next < items.length) {
      const index = next++;
      results[index] = await fn(items[index], index);
    }
  });
  await Promise.all(workers);
  return results;
}

type Unit =
  | { kind: "trigger"; expected: "load" | "skip"; testCase: TriggerCase }
  | { kind: "adherence"; testCase: AdherenceCase };

export interface RunEvalsOptions {
  skillName: string;
  skillPath: string;
  suite: EvalSuite;
  config: EvalRunConfig;
  only?: "trigger" | "adherence";
}

export async function runEvals({ skillName, skillPath, suite, config, only }: RunEvalsOptions): Promise<EvalReport> {
  const startedAt = new Date();

  const units: Unit[] = [];
  if (only !== "adherence") {
    for (const testCase of suite.triggerPositive) units.push({ kind: "trigger", expected: "load", testCase });
    for (const testCase of suite.triggerNegative) units.push({ kind: "trigger", expected: "skip", testCase });
  }
  if (only !== "trigger") {
    for (const testCase of suite.adherence) units.push({ kind: "adherence", testCase });
  }

  const total = units.length * config.repeat;
  let done = 0;
  const tick = (label: string) => config.onProgress?.(++done, total, label);

  const results = await mapWithConcurrency(units, config.jobs, async (unit) => {
    if (unit.kind === "trigger") {
      const attempts: TriggerAttempt[] = [];
      let costUsd = 0;
      for (let i = 0; i < config.repeat; i++) {
        const { attempt, costUsd: cost } = await runTriggerAttempt(unit.testCase, skillPath, config);
        attempts.push(attempt);
        costUsd += cost;
        tick(`trigger: ${unit.testCase.prompt}`);
      }
      const passes = attempts.filter((a) => (unit.expected === "load" ? a.loaded : !a.loaded)).length;
      const result: TriggerResult = {
        kind: "trigger",
        expected: unit.expected,
        prompt: unit.testCase.prompt,
        ...(unit.testCase.note ? { note: unit.testCase.note } : {}),
        attempts,
        passes,
        costUsd,
      };
      return result;
    }

    const attempts: AdherenceAttempt[] = [];
    let costUsd = 0;
    for (let i = 0; i < config.repeat; i++) {
      const { attempt, costUsd: cost } = await runAdherenceAttempt(unit.testCase, skillName, skillPath, config);
      attempts.push(attempt);
      costUsd += cost;
      tick(`adherence: ${unit.testCase.prompt}`);
    }
    const passes = attempts.filter((a) => a.expectations.every((e) => e.verdict === "pass")).length;
    const result: AdherenceResult = {
      kind: "adherence",
      prompt: unit.testCase.prompt,
      ...(unit.testCase.note ? { note: unit.testCase.note } : {}),
      attempts,
      passes,
      costUsd,
    };
    return result;
  });

  const trigger = results.filter((r): r is TriggerResult => r.kind === "trigger");
  const adherence = results.filter((r): r is AdherenceResult => r.kind === "adherence");

  return {
    skillName,
    skillPath,
    model: `${config.provider}/${config.model}`,
    thinking: config.thinking,
    repeat: config.repeat,
    bundledFiles: await listBundledFiles(skillPath),
    trigger,
    adherence,
    costUsd: results.reduce((sum, r) => sum + r.costUsd, 0),
    runs: total,
    startedAt,
    finishedAt: new Date(),
  };
}

// ---------- Reporting ----------

function firstLine(text: string, max = 80): string {
  const line = text.split("\n")[0].trim();
  return line.length > max ? `${line.slice(0, max - 1)}…` : line;
}

/**
 * The end of a transcript from a run that died. A killed run stalls at its last
 * action, so the tail is the diagnostic — it separates an agent thrashing on
 * tool calls from one that simply ran out of clock.
 */
export function transcriptTail(transcript: string, maxLines = 20): string[] {
  const all = transcript.split("\n");
  while (all.length > 0 && all.at(-1)!.trim() === "") all.pop();
  const tail = all.slice(-maxLines);
  while (tail.length > 0 && tail[0].trim() === "") tail.shift();
  if (tail.length === 0) return [];
  const elided = all.length - tail.length;
  return elided > 0 ? [`…${elided} earlier line(s) elided`, ...tail] : tail;
}

/** Bundled references no attempt opened — the cheapest trim candidates in the skill. */
export function unusedBundledFiles(report: EvalReport): string[] {
  const opened = new Set<string>();
  for (const row of [...report.trigger, ...report.adherence]) {
    for (const attempt of row.attempts) {
      for (const file of attempt.reads) opened.add(file);
    }
  }
  return report.bundledFiles.filter((file) => !opened.has(file));
}

export function formatEvalReport(report: EvalReport): string {
  const lines: string[] = [];
  const triggerPasses = report.trigger.reduce((sum, r) => sum + r.passes, 0);
  const triggerTotal = report.trigger.length * report.repeat;
  const adherencePasses = report.adherence.reduce((sum, r) => sum + r.passes, 0);
  const adherenceTotal = report.adherence.length * report.repeat;
  const seconds = Math.round((report.finishedAt.getTime() - report.startedAt.getTime()) / 1000);

  lines.push(`# Skill eval: ${report.skillName}`);
  lines.push("");
  lines.push(`${report.skillPath}`);
  lines.push("");
  lines.push(
    `${report.model}${report.thinking ? ` (thinking: ${report.thinking})` : ""} · ${report.runs} case run(s) · ` +
      `${report.repeat} attempt(s) per case · $${report.costUsd.toFixed(4)} · ${seconds}s`,
  );
  lines.push("");

  const retries = [...report.trigger, ...report.adherence].flatMap((row) => row.attempts.flatMap((a) => a.retries));
  if (retries.length > 0) {
    lines.push(
      `${retries.length} provider retr${retries.length === 1 ? "y" : "ies"} during this run, ` +
        `first: ${retries[0].errorMessage}. Scores below may reflect the provider, not the skill.`,
    );
    lines.push("");
  }

  if (report.trigger.length > 0) {
    lines.push(`## Trigger fidelity — ${triggerPasses}/${triggerTotal}`);
    lines.push("");
    lines.push("| Expected | Loaded | Prompt |");
    lines.push("|----------|--------|--------|");
    for (const row of report.trigger) {
      const loads = row.attempts.filter((a) => a.loaded).length;
      lines.push(
        `| ${row.expected === "load" ? "load" : "skip"} | ${loads}/${row.attempts.length} | ${firstLine(row.prompt)} |`,
      );
    }
    const failures = report.trigger.filter((r) => r.passes < r.attempts.length);
    if (failures.length > 0) {
      lines.push("");
      lines.push("Misses:");
      lines.push("");
      for (const row of failures) {
        const verb = row.expected === "load" ? "did not load" : "loaded when it should not have";
        lines.push(`- ${verb}: ${firstLine(row.prompt, 120)}`);
        for (const attempt of row.attempts) {
          if (attempt.error) lines.push(`  - run error: ${attempt.error}`);
        }
      }
    }
    lines.push("");
  }

  if (report.adherence.length > 0) {
    lines.push(`## Instruction adherence — ${adherencePasses}/${adherenceTotal}`);
    lines.push("");
    for (const row of report.adherence) {
      lines.push(`### ${firstLine(row.prompt, 120)}`);
      lines.push("");
      row.attempts.forEach((attempt, i) => {
        if (row.attempts.length > 1) lines.push(`Attempt ${i + 1}:`);
        // A failed run grades nothing, so every expectation carries the same
        // reason. Print it once: repeating it per expectation reads as several
        // independent findings instead of one dead run.
        if (attempt.error) {
          const count = attempt.expectations.length;
          lines.push(`- run error: ${attempt.error}`);
          lines.push(
            `- ${count} expectation(s) ungraded, so this case says nothing about the skill. Re-run it before ` +
              `changing the text.`,
          );
          for (const expectation of attempt.expectations) {
            lines.push(`  - ungraded: ${expectation.expectation}`);
          }
          const tail = transcriptTail(attempt.transcript);
          lines.push("");
          if (tail.length > 0) {
            lines.push("Where it stopped:");
            lines.push("");
            lines.push("```");
            lines.push(...tail);
            lines.push("```");
          } else {
            lines.push("The run produced no output before it died.");
          }
          lines.push("");
          return;
        }
        for (const expectation of attempt.expectations) {
          lines.push(`- ${expectation.verdict}: ${expectation.expectation} — ${expectation.reason}`);
        }
        lines.push("");
      });
    }
  }

  const unused = unusedBundledFiles(report);
  if (report.bundledFiles.length > 0) {
    const total = report.bundledFiles.length;
    const them = unused.length === 1 ? "it" : "them";
    lines.push("## Trim signals");
    lines.push("");
    lines.push(
      unused.length === 0
        ? `Some run opened ${total === 1 ? "the one bundled file" : `all ${total} bundled files`}.`
        : `Never opened, ${unused.length} of ${total} bundled ${total === 1 ? "file" : "files"}: ` +
          `${unused.join(", ")}. Either no case reaches ${them} or the skill doesn't need ${them}.`,
    );
    lines.push("");
  }

  return lines.join("\n");
}

/**
 * Whether any failure in the report is a verdict on the skill rather than a dead
 * run. A killed or errored run fails every expectation under it for the same
 * reason, which says nothing about the text.
 */
export function hasGradedFailure(report: EvalReport): boolean {
  for (const row of report.trigger) {
    if (row.passes === row.attempts.length) continue;
    if (row.attempts.some((a) => !a.error)) return true;
  }
  for (const row of report.adherence) {
    for (const attempt of row.attempts) {
      if (attempt.error) continue;
      if (attempt.expectations.some((e) => e.verdict !== "pass")) return true;
    }
  }
  return false;
}

/** Failure detail worth pulling into the conversation so the skill text can be fixed. */
export function summarizeFailures(report: EvalReport): string[] {
  const notes: string[] = [];
  for (const row of report.trigger) {
    if (row.passes === row.attempts.length) continue;
    const errored = row.attempts.filter((a) => a.error);
    // A run that died never chose whether to load the skill, so reporting it as
    // a trigger miss invents a verdict the run didn't reach.
    if (errored.length === row.attempts.length) {
      notes.push(
        `No verdict on "${firstLine(row.prompt, 120)}": the run failed (${errored[0].error}). ` +
          `Re-run this case; it says nothing about the description.`,
      );
      continue;
    }
    const loads = row.attempts.filter((a) => a.loaded).length;
    notes.push(
      row.expected === "load"
        ? `The description did not fire for "${firstLine(row.prompt, 120)}" (${loads}/${row.attempts.length} loads).`
        : `The description fired on the near-miss prompt "${firstLine(row.prompt, 120)}" (${loads}/${row.attempts.length} loads).`,
    );
  }
  for (const row of report.adherence) {
    for (const attempt of row.attempts) {
      // One note per dead run, not one per expectation: the reason is identical
      // across them, and repeating it reads as several independent findings.
      if (attempt.error) {
        const count = attempt.expectations.length;
        notes.push(
          `No verdict on "${firstLine(row.prompt, 80)}": the run failed (${attempt.error}), leaving ` +
            `${count} expectation(s) ungraded. Re-run this case; it says nothing about the skill.`,
        );
        continue;
      }
      for (const expectation of attempt.expectations) {
        if (expectation.verdict === "pass") continue;
        notes.push(
          `On "${firstLine(row.prompt, 80)}", expectation "${expectation.expectation}" came back ${expectation.verdict}: ${expectation.reason}`,
        );
      }
    }
  }
  return notes;
}

/**
 * Follow-up for a run with failures. Efficacy comes first, but a fix that only
 * adds text is how a skill bloats, so the ask is to pay for new lines with old
 * ones.
 */
export function buildFailureFollowUp(report: EvalReport, failures: string[], reportPath: string): string {
  // With nothing graded, asking for edits invites rewriting the skill against a
  // blank transcript. Ask for a re-run instead.
  const ask = hasGradedFailure(report)
    ? [
        "Read the skill and propose specific edits to its text that would fix these, or say why a case is",
        "wrong about the skill rather than the skill being wrong.",
        "",
        "Reach the fix by sharpening or cutting what's already there before adding anything. If a failure",
        "genuinely needs new instruction, name something else in the file to delete alongside it.",
      ]
    : [
        "Every failure above is a run that died, so nothing was graded and the skill text is unproven, not",
        `disproven. Re-run the affected cases with \`/skill-eval run ${report.skillName} --case <filter>\` before`,
        "changing a word of the skill. If they keep dying, fix the case or the harness — a timeout is not a",
        "verdict on the text.",
      ];
  return [
    `# Eval failures: ${report.skillName}`,
    "",
    `Behavioral evals for ${report.skillPath} came back with failures. Full report: ${reportPath}`,
    "",
    ...failures.map((note) => `- ${note}`),
    "",
    ...ask,
  ].join("\n");
}

/**
 * Follow-up for a clean run. A green suite is the safety net that makes cutting
 * cheap: cut, re-run, and the score says whether the text was load-bearing.
 */
export function buildTrimFollowUp(report: EvalReport, lineCount: number, reportPath: string): string {
  const unused = unusedBundledFiles(report);
  return [
    `# Trim pass: ${report.skillName}`,
    "",
    `Every case passed for ${report.skillPath} (${report.runs} run(s)). Full report: ${reportPath}`,
    "",
    `The suite is now a safety net, so cuts are cheap to verify: cut, re-run \`/skill-eval run ${report.skillName}\`,`,
    `and a still-green score says the text wasn't load-bearing. The skill file is ${lineCount} lines.`,
    ...(unused.length > 0
      ? ["", `No run opened ${unused.join(" or ")}, so start there.`]
      : []),
    "",
    "Load the `tighten-docs` skill and propose specific cuts. Leave the `description` frontmatter's",
    "trigger vocabulary alone — a smaller skill that stops loading is a worse skill.",
  ].join("\n");
}