1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
/**
 * Subagent runner.
 *
 * Pi has no subagents, so a "subagent" here is a nested `pi -p --mode json`
 * process: no session file, no extensions, no context files, an explicit tool
 * allowlist, and a scratch cwd. Everything the caller needs — tool calls,
 * final text, spend — comes from parsing the JSON event stream on stdout.
 */

import { spawn } from "node:child_process";

export interface SubagentOptions {
  cwd: string;
  provider: string;
  model: string;
  thinking?: string;
  /** Tool allowlist. An empty list disables tools entirely. */
  tools: string[];
  /** Extra skill paths to load on top of ordinary discovery. */
  skillDirs?: string[];
  /** Whether ordinary skill discovery runs, which keeps description competition realistic. */
  discoverSkills: boolean;
  /** Messages processed in order, each as its own turn. Must not start with "-" or "@". */
  messages?: string[];
  /** Prompt text piped in, which avoids argv escaping limits entirely. */
  stdin?: string;
  timeoutMs?: number;
  signal?: AbortSignal;
}

export interface SubagentToolCall {
  name: string;
  args: Record<string, unknown>;
}

export interface RetryNote {
  attempt: number;
  maxAttempts: number;
  errorMessage: string;
}

export interface SubagentResult {
  exitCode: number | null;
  timedOut: boolean;
  stderr: string;
  events: any[];
  toolCalls: SubagentToolCall[];
  finalText: string;
  costUsd: number;
  elapsedMs: number;
  /** Provider retries pi performed inside the run, e.g. overload or rate limiting. */
  retries: RetryNote[];
  /** Stop reason of the run's last assistant message, useful when the reply is empty. */
  stopReason?: string;
  /** Set when the run itself failed, as opposed to the graded behavior failing. */
  error?: string;
}

export function buildSubagentArgs(opts: SubagentOptions): string[] {
  const args = [
    "-p",
    "--mode",
    "json",
    "--no-session",
    "--no-extensions",
    "--no-context-files",
    "--no-prompt-templates",
    "--no-approve",
    "--offline",
    "--provider",
    opts.provider,
    "--model",
    opts.model,
  ];
  if (opts.thinking) args.push("--thinking", opts.thinking);
  if (opts.tools.length === 0) args.push("--no-tools");
  else args.push("--tools", opts.tools.join(","));
  if (!opts.discoverSkills) args.push("--no-skills");
  for (const dir of opts.skillDirs ?? []) args.push("--skill", dir);
  for (const message of opts.messages ?? []) args.push(message);
  return args;
}

/** Resolve the pi entrypoint of the running process so subagents match this build. */
export function piExecutable(): { command: string; prefixArgs: string[] } {
  const cli = process.env.PI_BIN ?? process.argv[1];
  if (cli) return { command: process.execPath, prefixArgs: [cli] };
  return { command: "pi", prefixArgs: [] };
}

export function collectEvents(stdout: string): any[] {
  const events: any[] = [];
  for (const line of stdout.split("\n")) {
    if (!line.trim()) continue;
    try {
      events.push(JSON.parse(line));
    } catch {
      // Non-JSON noise on stdout isn't fatal; the stream is line-oriented.
    }
  }
  return events;
}

export function extractToolCalls(events: any[]): SubagentToolCall[] {
  return events
    .filter((event) => event.type === "tool_execution_start")
    .map((event) => ({ name: event.toolName, args: event.args ?? {} }));
}

function finalMessages(events: any[]): any[] {
  for (let i = events.length - 1; i >= 0; i--) {
    if (events[i].type === "agent_end" && Array.isArray(events[i].messages)) return events[i].messages;
  }
  return [];
}

export function extractFinalText(events: any[]): string {
  const messages = finalMessages(events);
  for (let i = messages.length - 1; i >= 0; i--) {
    const message = messages[i];
    if (message.role !== "assistant" || !Array.isArray(message.content)) continue;
    const text = message.content
      .filter((block: any) => block.type === "text" && typeof block.text === "string")
      .map((block: any) => block.text)
      .join("\n")
      .trim();
    if (text) return text;
  }
  return "";
}

export function extractRetries(events: any[]): RetryNote[] {
  return events
    .filter((event) => event.type === "auto_retry_start")
    .map((event) => ({
      attempt: event.attempt ?? 0,
      maxAttempts: event.maxAttempts ?? 0,
      errorMessage: String(event.errorMessage ?? "unknown provider error"),
    }));
}

/** The provider error that ended the run, when retries were exhausted or the model reported one. */
export function extractApiError(events: any[]): string | undefined {
  for (let i = events.length - 1; i >= 0; i--) {
    const event = events[i];
    if (event.type === "auto_retry_end" && event.success === false) {
      return String(event.finalError ?? "provider retries exhausted");
    }
  }
  const messages = finalMessages(events);
  for (let i = messages.length - 1; i >= 0; i--) {
    const message = messages[i];
    if (message.role !== "assistant") continue;
    if (message.stopReason === "error" || message.stopReason === "aborted") {
      return String(message.errorMessage ?? `run ended with stopReason ${message.stopReason}`);
    }
    return undefined;
  }
  return undefined;
}

export function extractStopReason(events: any[]): string | undefined {
  const messages = finalMessages(events);
  for (let i = messages.length - 1; i >= 0; i--) {
    if (messages[i].role === "assistant") return messages[i].stopReason;
  }
  return undefined;
}

export function extractCost(events: any[]): number {
  let total = 0;
  for (const message of finalMessages(events)) {
    const cost = message?.usage?.cost?.total;
    if (typeof cost === "number") total += cost;
  }
  return total;
}

/** Readable transcript of a run, for handing to a judge. */
export function formatTranscript(events: any[], maxResultChars = 1500): string {
  const lines: string[] = [];
  for (const message of finalMessages(events)) {
    if (message.role === "user" && Array.isArray(message.content)) {
      const text = message.content
        .filter((block: any) => block.type === "text")
        .map((block: any) => block.text)
        .join("\n")
        .trim();
      if (text) lines.push(`[user]\n${text}`);
    } else if (message.role === "assistant" && Array.isArray(message.content)) {
      for (const block of message.content) {
        if (block.type === "text" && block.text?.trim()) lines.push(`[assistant]\n${block.text.trim()}`);
        if (block.type === "toolCall") {
          lines.push(`[tool call: ${block.name}]\n${truncate(JSON.stringify(block.arguments ?? {}), maxResultChars)}`);
        }
      }
    } else if (message.role === "toolResult" && Array.isArray(message.content)) {
      const text = message.content
        .filter((block: any) => block.type === "text")
        .map((block: any) => block.text)
        .join("\n");
      lines.push(`[tool result: ${message.toolName}${message.isError ? " (error)" : ""}]\n${truncate(text, maxResultChars)}`);
    }
  }
  return lines.join("\n\n");
}

function truncate(text: string, max: number): string {
  if (text.length <= max) return text;
  return `${text.slice(0, max)}\n… [${text.length - max} more characters]`;
}

export async function runSubagent(opts: SubagentOptions): Promise<SubagentResult> {
  const { command, prefixArgs } = piExecutable();
  const args = [...prefixArgs, ...buildSubagentArgs(opts)];

  const env = { ...process.env, NO_COLOR: "1", PI_OFFLINE: "1" };
  // A nested run must not inherit this session's identity.
  delete env.PI_SESSION_FILE;
  delete env.PI_SESSION_ID;

  const startedAt = Date.now();
  return await new Promise<SubagentResult>((resolve) => {
    const child = spawn(command, args, { cwd: opts.cwd, env, stdio: ["pipe", "pipe", "pipe"] });
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    let settled = false;

    const timer = opts.timeoutMs
      ? setTimeout(() => {
          timedOut = true;
          child.kill("SIGKILL");
        }, opts.timeoutMs)
      : undefined;

    const onAbort = () => child.kill("SIGKILL");
    opts.signal?.addEventListener("abort", onAbort, { once: true });

    const finish = (exitCode: number | null, error?: string) => {
      if (settled) return;
      settled = true;
      if (timer) clearTimeout(timer);
      opts.signal?.removeEventListener("abort", onAbort);
      const events = collectEvents(stdout);
      const apiError = extractApiError(events);
      const stopReason = extractStopReason(events);
      resolve({
        exitCode,
        timedOut,
        stderr: stderr.trim(),
        events,
        toolCalls: extractToolCalls(events),
        finalText: extractFinalText(events),
        costUsd: extractCost(events),
        elapsedMs: Date.now() - startedAt,
        retries: extractRetries(events),
        ...(stopReason ? { stopReason } : {}),
        ...(error || apiError ? { error: error ?? apiError } : {}),
      });
    };

    child.stdout.setEncoding("utf8");
    child.stdout.on("data", (chunk) => (stdout += chunk));
    child.stderr.setEncoding("utf8");
    child.stderr.on("data", (chunk) => (stderr += chunk));
    child.on("error", (error) => finish(null, error.message));
    child.on("close", (code) => {
      if (timedOut) finish(code, `run exceeded ${opts.timeoutMs}ms and was killed`);
      else if (code !== 0) finish(code, `pi exited with code ${code}${stderr.trim() ? `: ${stderr.trim()}` : ""}`);
      else finish(code);
    });

    if (opts.stdin !== undefined) child.stdin.end(opts.stdin);
    else child.stdin.end();
  });
}

/** Run a case's setup script in its sandbox. */
export async function runSetupScript(script: string, cwd: string): Promise<{ ok: boolean; output: string }> {
  return await new Promise((resolve) => {
    const child = spawn("bash", ["-euo", "pipefail", "-c", script], { cwd, stdio: ["ignore", "pipe", "pipe"] });
    let output = "";
    child.stdout.setEncoding("utf8");
    child.stdout.on("data", (chunk) => (output += chunk));
    child.stderr.setEncoding("utf8");
    child.stderr.on("data", (chunk) => (output += chunk));
    child.on("error", (error) => resolve({ ok: false, output: error.message }));
    child.on("close", (code) => resolve({ ok: code === 0, output: output.trim() }));
  });
}