1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
  buildFailureFollowUp,
  buildJudgePrompt,
  buildTrimFollowUp,
  bundledReads,
  formatEvalReport,
  hasGradedFailure,
  listBundledFiles,
  loadedSkill,
  mapWithConcurrency,
  filterSuite,
  matchesFilters,
  parseJudgeVerdicts,
  runEvals,
  skillLoadPath,
  summarizeFailures,
  transcriptTail,
  unusedBundledFiles,
  type EvalReport,
} from "./run.ts";

test("loadedSkill matches reads anywhere inside the skill directory", () => {
  const skillPath = "/skills/jj/SKILL.md";
  assert.equal(loadedSkill([{ name: "read", args: { path: "/skills/jj/SKILL.md" } }], skillPath), true);
  assert.equal(loadedSkill([{ name: "read", args: { path: "/skills/jj/references/rebase.md" } }], skillPath), true);
  assert.equal(loadedSkill([{ name: "read", args: { path: "/skills/jj/../jj/SKILL.md" } }], skillPath), true);
});

test("loadedSkill ignores other skills, other tools, and malformed arguments", () => {
  const skillPath = "/skills/jj/SKILL.md";
  assert.equal(loadedSkill([{ name: "read", args: { path: "/skills/jj-workspaces/SKILL.md" } }], skillPath), false);
  assert.equal(loadedSkill([{ name: "bash", args: { command: "cat /skills/jj/SKILL.md" } }], skillPath), false);
  assert.equal(loadedSkill([{ name: "read", args: {} }], skillPath), false);
  assert.equal(loadedSkill([], skillPath), false);
});

test("loadedSkill counts only the file itself for a bare .md skill", () => {
  const skillPath = "/skills/jj.md";
  assert.equal(loadedSkill([{ name: "read", args: { path: "/skills/jj.md" } }], skillPath), true);
  assert.equal(loadedSkill([{ name: "read", args: { path: "/skills/bash.md" } }], skillPath), false);
});

test("bundledReads lists skill-relative reads and ignores everything else", () => {
  const skillPath = "/skills/jj/SKILL.md";
  const calls = [
    { name: "read", args: { path: "/skills/jj/SKILL.md" } },
    { name: "read", args: { path: "/skills/jj/references/rebase.md" } },
    { name: "read", args: { path: "/skills/jj/references/rebase.md" } },
    { name: "read", args: { path: "/skills/other/SKILL.md" } },
    { name: "bash", args: { command: "cat /skills/jj/pitfalls.md" } },
  ];
  assert.deepEqual(bundledReads(calls, skillPath), ["SKILL.md", "references/rebase.md"]);
  assert.deepEqual(bundledReads(calls, "/skills/jj.md"), []);
});

test("listBundledFiles finds references and skips SKILL.md, evals.yml, and dotfiles", async () => {
  const dir = await mkdtemp(join(tmpdir(), "skill-eval-bundle-"));
  try {
    await writeFile(join(dir, "SKILL.md"), "# skill");
    await writeFile(join(dir, "evals.yml"), "trigger: {}");
    await writeFile(join(dir, "pitfalls.md"), "# pitfalls");
    await writeFile(join(dir, ".hidden"), "x");
    await mkdir(join(dir, "references"));
    await writeFile(join(dir, "references", "rebase.md"), "# rebase");

    assert.deepEqual(await listBundledFiles(join(dir, "SKILL.md")), ["pitfalls.md", "references/rebase.md"]);
    assert.deepEqual(await listBundledFiles(join(dir, "bare.md")), []);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});

test("skillLoadPath passes the bundle directory for SKILL.md and the file otherwise", () => {
  assert.equal(skillLoadPath("/skills/jj/SKILL.md"), "/skills/jj");
  assert.equal(skillLoadPath("/skills/jj.md"), "/skills/jj.md");
});

test("buildJudgePrompt numbers expectations and carries task and transcript", () => {
  const prompt = buildJudgePrompt("jj", "commit my work", "[assistant] jj commit", ["uses jj commit", "adds a trailer"]);
  assert.match(prompt, /"jj" skill/);
  assert.match(prompt, /1\. uses jj commit/);
  assert.match(prompt, /2\. adds a trailer/);
  assert.match(prompt, /commit my work/);
  assert.match(prompt, /\[assistant\] jj commit/);
});

test("buildJudgePrompt notes an empty transcript rather than sending nothing", () => {
  assert.match(buildJudgePrompt("jj", "task", "", ["something"]), /the agent produced no output/);
});

test("parseJudgeVerdicts reads a fenced array and matches verdicts by index", () => {
  const reply = [
    "Here are my verdicts:",
    "```json",
    '[{"index": 2, "verdict": "fail", "reason": "no trailer"},',
    ' {"index": 1, "verdict": "pass", "reason": "ran jj commit"}]',
    "```",
  ].join("\n");
  const results = parseJudgeVerdicts(reply, ["uses jj commit", "adds a trailer"]);
  assert.deepEqual(results, [
    { expectation: "uses jj commit", verdict: "pass", reason: "ran jj commit" },
    { expectation: "adds a trailer", verdict: "fail", reason: "no trailer" },
  ]);
});

test("parseJudgeVerdicts falls back to unclear when the reply is unusable", () => {
  for (const reply of ["I could not tell.", "[not json]", '{"verdict":"pass"}']) {
    const results = parseJudgeVerdicts(reply, ["a", "b"]);
    assert.equal(results.length, 2);
    assert.ok(results.every((r) => r.verdict === "unclear"));
  }
});

test("parseJudgeVerdicts treats unknown verdict values as unclear", () => {
  const results = parseJudgeVerdicts('[{"index":1,"verdict":"maybe","reason":"hmm"}]', ["a"]);
  assert.deepEqual(results, [{ expectation: "a", verdict: "unclear", reason: "hmm" }]);
});

test("mapWithConcurrency preserves order and respects the limit", async () => {
  let inFlight = 0;
  let peak = 0;
  const results = await mapWithConcurrency([10, 20, 30, 40, 50], 2, async (item) => {
    inFlight++;
    peak = Math.max(peak, inFlight);
    await new Promise((r) => setTimeout(r, item % 30));
    inFlight--;
    return item * 2;
  });
  assert.deepEqual(results, [20, 40, 60, 80, 100]);
  assert.equal(peak, 2);
});

const report: EvalReport = {
  skillName: "widget-wrangler",
  skillPath: "/skills/widget-wrangler/SKILL.md",
  model: "anthropic/claude-opus-5",
  thinking: "high",
  repeat: 1,
  bundledFiles: ["calibration.md", "torque.md"],
  trigger: [
    { kind: "trigger", expected: "load", prompt: "calibrate the bench widget", attempts: [{ loaded: true, reads: ["SKILL.md"], retries: [] }], passes: 1, costUsd: 0.01 },
    { kind: "trigger", expected: "load", prompt: "check widget torque", attempts: [{ loaded: false, reads: [], retries: [] }], passes: 0, costUsd: 0.01 },
    { kind: "trigger", expected: "skip", prompt: "what is 2 + 2", attempts: [{ loaded: true, reads: ["SKILL.md"], retries: [] }], passes: 0, costUsd: 0.01 },
  ],
  adherence: [
    {
      kind: "adherence",
      prompt: "calibrate it and report the torque",
      attempts: [
        {
          transcript: "[assistant] wrangle --torque 42",
          reads: ["torque.md"],
          retries: [],
          expectations: [
            { expectation: "names the torque value 42", verdict: "pass", reason: "said 42" },
            { expectation: "reports newton-metres", verdict: "fail", reason: "no units given" },
          ],
        },
      ],
      passes: 0,
      costUsd: 0.05,
    },
  ],
  costUsd: 0.08,
  runs: 4,
  startedAt: new Date("2026-01-01T00:00:00.000Z"),
  finishedAt: new Date("2026-01-01T00:01:00.000Z"),
};

test("formatEvalReport scores both sections and lists misses", () => {
  const markdown = formatEvalReport(report);
  assert.match(markdown, /# Skill eval: widget-wrangler/);
  assert.match(markdown, /anthropic\/claude-opus-5 \(thinking: high\).*\$0\.0800.*60s/);
  assert.match(markdown, /## Trigger fidelity — 1\/3/);
  assert.match(markdown, /\| load \| 0\/1 \| check widget torque \|/);
  assert.match(markdown, /did not load: check widget torque/);
  assert.match(markdown, /loaded when it should not have: what is 2 \+ 2/);
  assert.match(markdown, /## Instruction adherence — 0\/1/);
  assert.match(markdown, /- fail: reports newton-metres — no units given/);
});

test("summarizeFailures reports each miss and each unmet expectation", () => {
  const notes = summarizeFailures(report);
  assert.equal(notes.length, 3);
  assert.match(notes[0], /did not fire for "check widget torque"/);
  assert.match(notes[1], /fired on the near-miss prompt "what is 2 \+ 2"/);
  assert.match(notes[2], /"reports newton-metres" came back fail: no units given/);
});

const clean: EvalReport = {
  ...report,
  trigger: [
    {
      kind: "trigger",
      expected: "load",
      prompt: "calibrate the bench widget",
      attempts: [{ loaded: true, reads: ["SKILL.md", "calibration.md"], retries: [] }],
      passes: 1,
      costUsd: 0,
    },
  ],
  adherence: [],
};

test("summarizeFailures says nothing when every case passed", () => {
  assert.deepEqual(summarizeFailures(clean), []);
});

test("unusedBundledFiles names the references no attempt opened", () => {
  assert.deepEqual(unusedBundledFiles(report), ["calibration.md"]);
  assert.deepEqual(unusedBundledFiles(clean), ["torque.md"]);
  assert.deepEqual(unusedBundledFiles({ ...report, bundledFiles: [] }), []);
});

test("formatEvalReport flags unopened bundled files as trim candidates", () => {
  const markdown = formatEvalReport(report);
  assert.match(markdown, /## Trim signals\n\nNever opened, 1 of 2 bundled files: calibration\.md\./);
  assert.match(markdown, /Either no case reaches it or the skill doesn't need it\./);
});

test("formatEvalReport says so when every bundled file earned a read", () => {
  const covered: EvalReport = { ...clean, bundledFiles: ["calibration.md"] };
  assert.match(formatEvalReport(covered), /Some run opened the one bundled file\./);
  const both: EvalReport = { ...clean, bundledFiles: ["SKILL.md", "calibration.md"] };
  assert.match(formatEvalReport(both), /Some run opened all 2 bundled files\./);
});

test("formatEvalReport skips trim signals for a skill with no bundled files", () => {
  assert.doesNotMatch(formatEvalReport({ ...report, bundledFiles: [] }), /Trim signals/);
});

// A run killed by the timeout: every expectation carries the same reason and
// nothing reached the judge.
const deadReason = "run failed: run exceeded 300000ms and was killed";
const dead: EvalReport = {
  ...report,
  trigger: [],
  adherence: [
    {
      kind: "adherence",
      prompt: "calibrate it and report the torque",
      attempts: [
        {
          transcript: "[assistant] reading torque.md\n[tool] read torque.md\n[assistant] checking the bench",
          reads: ["torque.md"],
          retries: [],
          error: "run exceeded 300000ms and was killed",
          expectations: [
            { expectation: "names the torque value 42", verdict: "unclear", reason: deadReason },
            { expectation: "reports newton-metres", verdict: "unclear", reason: deadReason },
          ],
        },
      ],
      passes: 0,
      costUsd: 0.05,
    },
  ],
};

test("transcriptTail keeps the end and says how much it dropped", () => {
  const long = Array.from({ length: 25 }, (_, i) => `line ${i + 1}`).join("\n");
  const tail = transcriptTail(long, 5);
  assert.equal(tail[0], "…20 earlier line(s) elided");
  assert.equal(tail.at(-1), "line 25");
  assert.deepEqual(transcriptTail("only\n\n", 5), ["only"]);
  assert.deepEqual(transcriptTail("", 5), []);
});

test("summarizeFailures collapses a dead run into one note instead of one per expectation", () => {
  const notes = summarizeFailures(dead);
  assert.equal(notes.length, 1);
  assert.match(notes[0], /No verdict on "calibrate it and report the torque"/);
  assert.match(notes[0], /2 expectation\(s\) ungraded/);
  assert.match(notes[0], /says nothing about the skill/);
});

test("summarizeFailures does not call a dead trigger run a description miss", () => {
  const deadTrigger: EvalReport = {
    ...report,
    adherence: [],
    trigger: [
      {
        kind: "trigger",
        expected: "load",
        prompt: "check widget torque",
        attempts: [{ loaded: false, reads: [], retries: [], error: "run exceeded 300000ms and was killed" }],
        passes: 0,
        costUsd: 0.01,
      },
    ],
  };
  const notes = summarizeFailures(deadTrigger);
  assert.equal(notes.length, 1);
  assert.match(notes[0], /No verdict on "check widget torque"/);
  assert.doesNotMatch(notes[0], /did not fire/);
});

test("hasGradedFailure separates real verdicts from dead runs", () => {
  assert.equal(hasGradedFailure(report), true);
  assert.equal(hasGradedFailure(dead), false);
  assert.equal(hasGradedFailure(clean), false);
});

test("formatEvalReport prints a dead run once and shows where it stopped", () => {
  const markdown = formatEvalReport(dead);
  assert.match(markdown, /- run error: run exceeded 300000ms and was killed/);
  assert.match(markdown, /2 expectation\(s\) ungraded, so this case says nothing about the skill/);
  assert.match(markdown, /  - ungraded: names the torque value 42/);
  assert.match(markdown, /Where it stopped:/);
  assert.match(markdown, /\[assistant\] checking the bench/);
  // The identical reason should not be repeated once per expectation.
  assert.equal(markdown.match(/run exceeded 300000ms/g)?.length, 1);
});

test("formatEvalReport says so when a dead run produced no output at all", () => {
  const silent: EvalReport = {
    ...dead,
    adherence: [{ ...dead.adherence[0], attempts: [{ ...dead.adherence[0].attempts[0], transcript: "" }] }],
  };
  assert.match(formatEvalReport(silent), /The run produced no output before it died\./);
});

test("buildFailureFollowUp asks for a re-run when nothing was graded", () => {
  const message = buildFailureFollowUp(dead, summarizeFailures(dead), "/reports/eval.md");
  assert.match(message, /unproven, not\ndisproven/);
  assert.match(message, /--case <filter>` before/);
  assert.match(message, /a timeout is not a\nverdict on the text/);
  assert.doesNotMatch(message, /propose specific edits/);
});

test("buildFailureFollowUp asks for cuts before additions", () => {
  const message = buildFailureFollowUp(report, summarizeFailures(report), "/reports/eval.md");
  assert.match(message, /# Eval failures: widget-wrangler/);
  assert.match(message, /\/reports\/eval\.md/);
  assert.match(message, /- The description did not fire/);
  assert.match(message, /sharpening or cutting what's already there before adding anything/);
  assert.match(message, /name something else in the file to delete/);
});

test("buildTrimFollowUp turns a green suite into a trim invitation", () => {
  const message = buildTrimFollowUp(clean, 212, "/reports/eval.md");
  assert.match(message, /# Trim pass: widget-wrangler/);
  assert.match(message, /Every case passed/);
  assert.match(message, /The skill file is 212 lines/);
  assert.match(message, /No run opened torque\.md, so start there\./);
  assert.match(message, /tighten-docs/);
  assert.match(message, /trigger vocabulary alone/);
});

test("buildTrimFollowUp stays quiet about unopened files when there are none", () => {
  const message = buildTrimFollowUp({ ...clean, bundledFiles: [] }, 40, "/reports/eval.md");
  assert.doesNotMatch(message, /No run opened/);
});

test("formatEvalReport warns when the provider retried, so scores aren't misread", () => {
  const retried: EvalReport = {
    ...report,
    trigger: [
      {
        kind: "trigger",
        expected: "load",
        prompt: "p",
        attempts: [{ loaded: false, reads: [], retries: [{ attempt: 1, maxAttempts: 3, errorMessage: "overloaded_error" }] }],
        passes: 0,
        costUsd: 0,
      },
    ],
    adherence: [],
  };
  const markdown = formatEvalReport(retried);
  assert.match(markdown, /1 provider retry during this run, first: overloaded_error/);
  assert.match(markdown, /may reflect the provider, not the skill/);
});

test("formatEvalReport stays quiet about retries when there were none", () => {
  assert.doesNotMatch(formatEvalReport(report), /provider retr/);
});

/**
 * End-to-end orchestration against a stub pi entrypoint: no API credit, but it
 * exercises argument construction, sandboxing, setup scripts, and judging.
 */
const FAKE_PI = `
const args = process.argv.slice(2);
let stdin = "";
process.stdin.setEncoding("utf8");
process.stdin.on("data", (c) => (stdin += c));
process.stdin.on("end", () => {
  const emit = (o) => process.stdout.write(JSON.stringify(o) + "\\n");
  if (args.includes("--no-tools")) {
    // Judge run: pass the first expectation, fail the rest.
    const count = (stdin.match(/^\\d+\\. /gm) || []).length;
    const verdicts = Array.from({ length: count }, (_, i) => ({
      index: i + 1,
      verdict: i === 0 ? "pass" : "fail",
      reason: "stub verdict",
    }));
    emit({ type: "agent_end", messages: [{ role: "assistant", content: [{ type: "text", text: JSON.stringify(verdicts) }], usage: { cost: { total: 0.001 } } }] });
    return;
  }
  const skillDir = args[args.indexOf("--skill") + 1];
  if (args.some((a) => a.startsWith("/skill:"))) {
    // Adherence run: record the sandbox contents so setup scripts are observable.
    const fs = require("node:fs");
    emit({ type: "agent_end", messages: [{ role: "assistant", content: [{ type: "text", text: "sandbox: " + fs.readdirSync(process.cwd()).join(",") }], usage: { cost: { total: 0.002 } } }] });
    return;
  }
  // Trigger run: load the skill only when the prompt mentions widgets.
  if (/widget/.test(stdin)) emit({ type: "tool_execution_start", toolName: "read", args: { path: skillDir + "/SKILL.md" } });
  emit({ type: "agent_end", messages: [{ role: "assistant", content: [{ type: "text", text: "done" }], usage: { cost: { total: 0.003 } } }] });
});
`;

test("runEvals scores trigger and adherence cases end to end", async () => {
  const dir = await mkdtemp(join(tmpdir(), "skill-eval-run-"));
  const previous = process.env.PI_BIN;
  try {
    const fakePi = join(dir, "fake-pi.js");
    await writeFile(fakePi, FAKE_PI);
    process.env.PI_BIN = fakePi;

    const result = await runEvals({
      skillName: "widget-wrangler",
      skillPath: join(dir, "SKILL.md"),
      suite: {
        triggerPositive: [{ prompt: "calibrate the widget" }],
        triggerNegative: [{ prompt: "what is 2 + 2" }, { prompt: "the widget question again" }],
        adherence: [
          { prompt: "calibrate it", expect: ["first", "second"], tools: ["read"], setup: "touch fixture.txt" },
        ],
      },
      config: { provider: "anthropic", model: "test-model", thinking: "high", repeat: 1, jobs: 2, timeoutMs: 10_000 },
    });

    assert.equal(result.runs, 4);
    assert.deepEqual(
      result.trigger.map((r) => [r.expected, r.passes]),
      [
        ["load", 1],
        ["skip", 1],
        ["skip", 0],
      ],
    );
    assert.deepEqual(
      result.adherence[0].attempts[0].expectations.map((e) => e.verdict),
      ["pass", "fail"],
    );
    assert.match(result.adherence[0].attempts[0].transcript, /sandbox: fixture\.txt/);
    assert.ok(result.costUsd > 0);
  } finally {
    if (previous === undefined) delete process.env.PI_BIN;
    else process.env.PI_BIN = previous;
    await rm(dir, { recursive: true, force: true });
  }
});

test("runEvals marks a case unclear when its setup script fails", async () => {
  const dir = await mkdtemp(join(tmpdir(), "skill-eval-run-"));
  const previous = process.env.PI_BIN;
  try {
    const fakePi = join(dir, "fake-pi.js");
    await writeFile(fakePi, FAKE_PI);
    process.env.PI_BIN = fakePi;

    const result = await runEvals({
      skillName: "widget-wrangler",
      skillPath: join(dir, "SKILL.md"),
      suite: {
        triggerPositive: [],
        triggerNegative: [],
        adherence: [{ prompt: "calibrate it", expect: ["first"], tools: ["read"], setup: "exit 7" }],
      },
      config: { provider: "anthropic", model: "test-model", repeat: 1, jobs: 1, timeoutMs: 10_000 },
    });

    const attempt = result.adherence[0].attempts[0];
    assert.match(attempt.error ?? "", /setup script failed/);
    assert.equal(attempt.expectations[0].verdict, "unclear");
  } finally {
    if (previous === undefined) delete process.env.PI_BIN;
    else process.env.PI_BIN = previous;
    await rm(dir, { recursive: true, force: true });
  }
});

test("runEvals honors --only by skipping the other section", async () => {
  const dir = await mkdtemp(join(tmpdir(), "skill-eval-run-"));
  const previous = process.env.PI_BIN;
  try {
    const fakePi = join(dir, "fake-pi.js");
    await writeFile(fakePi, FAKE_PI);
    process.env.PI_BIN = fakePi;

    const suite = {
      triggerPositive: [{ prompt: "calibrate the widget" }],
      triggerNegative: [],
      adherence: [{ prompt: "calibrate it", expect: ["first"], tools: ["read"] }],
    };
    const config = { provider: "anthropic", model: "test-model", repeat: 1, jobs: 1, timeoutMs: 10_000 };

    const triggerOnly = await runEvals({ skillName: "w", skillPath: join(dir, "SKILL.md"), suite, config, only: "trigger" });
    assert.equal(triggerOnly.trigger.length, 1);
    assert.equal(triggerOnly.adherence.length, 0);

    const adherenceOnly = await runEvals({ skillName: "w", skillPath: join(dir, "SKILL.md"), suite, config, only: "adherence" });
    assert.equal(adherenceOnly.trigger.length, 0);
    assert.equal(adherenceOnly.adherence.length, 1);
  } finally {
    if (previous === undefined) delete process.env.PI_BIN;
    else process.env.PI_BIN = previous;
    await rm(dir, { recursive: true, force: true });
  }
});

test("runEvals reports progress for every attempt", async () => {
  const dir = await mkdtemp(join(tmpdir(), "skill-eval-run-"));
  const previous = process.env.PI_BIN;
  try {
    const fakePi = join(dir, "fake-pi.js");
    await writeFile(fakePi, FAKE_PI);
    process.env.PI_BIN = fakePi;

    const seen: string[] = [];
    await runEvals({
      skillName: "widget-wrangler",
      skillPath: join(dir, "SKILL.md"),
      suite: { triggerPositive: [{ prompt: "calibrate the widget" }], triggerNegative: [], adherence: [] },
      config: {
        provider: "anthropic",
        model: "test-model",
        repeat: 2,
        jobs: 1,
        timeoutMs: 10_000,
        onProgress: (done, total, label) => seen.push(`${done}/${total} ${label}`),
      },
    });

    assert.deepEqual(seen, ["1/2 trigger: calibrate the widget", "2/2 trigger: calibrate the widget"]);
  } finally {
    if (previous === undefined) delete process.env.PI_BIN;
    else process.env.PI_BIN = previous;
    await rm(dir, { recursive: true, force: true });
  }
});

test("matchesFilters looks at both the prompt and the note, case-insensitively", () => {
  const testCase = { prompt: "Squash @ into its parent", note: "pitfall: editor hang" };
  assert.equal(matchesFilters(testCase, []), true);
  assert.equal(matchesFilters(testCase, ["PITFALL"]), true);
  assert.equal(matchesFilters(testCase, ["squash @"]), true);
  assert.equal(matchesFilters(testCase, ["rebase"]), false);
  assert.equal(matchesFilters(testCase, ["rebase", "editor"]), true);
});

test("filterSuite narrows every section and leaves an unfiltered suite alone", () => {
  const suite = {
    triggerPositive: [{ prompt: "squash this" }, { prompt: "push a stack" }],
    triggerNegative: [{ prompt: "git submodule" }],
    adherence: [
      { prompt: "squash into parent", expect: ["a"], tools: ["read"] },
      { prompt: "push bottom-first", expect: ["b"], tools: ["read"] },
    ],
  };

  const filtered = filterSuite(suite, ["squash"]);
  assert.deepEqual(filtered.triggerPositive, [{ prompt: "squash this" }]);
  assert.deepEqual(filtered.triggerNegative, []);
  assert.equal(filtered.adherence.length, 1);
  assert.equal(filtered.adherence[0].prompt, "squash into parent");

  assert.deepEqual(filterSuite(suite, []), suite);
});