fix: stop orphan reconciliation from killing healthy in-flight runs

Runs now record owner_pid (the Node process that started them) at creation
time. reconcileOrphanRuns skips runs whose owner process is still alive —
previously a newly loaded worker (dev compile / instrumentation) would see a
running run with pid not yet written (during the atlas stage), mark it failed
mid-flight, and disable the stop button, while sling kept running in the
background. Legacy rows with null owner_pid are still reconciled as before.
This commit is contained in:
tigerenwork 2026-08-09 23:07:24 +08:00
parent 088925cee4
commit 0b7dc0e86e
2 changed files with 22 additions and 7 deletions

View File

@ -82,6 +82,7 @@ export interface Run {
log_path: string | null; log_path: string | null;
error: string | null; error: string | null;
pid: number | null; pid: number | null;
owner_pid: number | null;
} }
function createDb(): Database.Database { function createDb(): Database.Database {
@ -123,7 +124,8 @@ function createDb(): Database.Database {
finished_at TEXT, finished_at TEXT,
log_path TEXT, log_path TEXT,
error TEXT, error TEXT,
pid INTEGER pid INTEGER,
owner_pid INTEGER
); );
`); `);
// Lightweight migrations for databases created before these columns existed. // Lightweight migrations for databases created before these columns existed.
@ -162,6 +164,11 @@ function createDb(): Database.Database {
"ALTER TABLE pipelines ADD COLUMN env TEXT NOT NULL DEFAULT '{}'", "ALTER TABLE pipelines ADD COLUMN env TEXT NOT NULL DEFAULT '{}'",
"env" "env"
); );
addColumnIfMissing(
"runs",
"ALTER TABLE runs ADD COLUMN owner_pid INTEGER",
"owner_pid"
);
return db; return db;
} }
@ -294,11 +301,12 @@ export function createRun(input: {
started_at?: string; started_at?: string;
log_path?: string; log_path?: string;
pid?: number; pid?: number;
owner_pid?: number;
}): Run { }): Run {
const res = db const res = db
.prepare( .prepare(
`INSERT INTO runs (pipeline_id, status, started_at, log_path, pid) `INSERT INTO runs (pipeline_id, status, started_at, log_path, pid, owner_pid)
VALUES (@pipeline_id, @status, @started_at, @log_path, @pid)` VALUES (@pipeline_id, @status, @started_at, @log_path, @pid, @owner_pid)`
) )
.run({ .run({
pipeline_id: input.pipeline_id, pipeline_id: input.pipeline_id,
@ -306,6 +314,7 @@ export function createRun(input: {
started_at: input.started_at ?? null, started_at: input.started_at ?? null,
log_path: input.log_path ?? null, log_path: input.log_path ?? null,
pid: input.pid ?? null, pid: input.pid ?? null,
owner_pid: input.owner_pid ?? null,
}); });
return getRun(Number(res.lastInsertRowid))!; return getRun(Number(res.lastInsertRowid))!;
} }

View File

@ -84,6 +84,7 @@ class JobManager {
pipeline_id: pipelineId, pipeline_id: pipelineId,
status: "running", status: "running",
started_at: new Date().toISOString(), started_at: new Date().toISOString(),
owner_pid: process.pid,
}); });
const runsDir = path.join(DATA_DIR, "runs"); const runsDir = path.join(DATA_DIR, "runs");
fs.mkdirSync(runsDir, { recursive: true }); fs.mkdirSync(runsDir, { recursive: true });
@ -324,16 +325,21 @@ export const jobManager: JobManager =
g.__jobManager ?? (g.__jobManager = new JobManager()); g.__jobManager ?? (g.__jobManager = new JobManager());
// Reconcile runs left in running/queued by a previous server process. // Reconcile runs left in running/queued by a previous server process.
// A run whose recorded pid is dead is stale (the subprocess died with the // Each run records owner_pid (the Node process that started it). A run whose
// old server or never existed) -> failed. A run whose pid is still alive is // owner is still alive belongs to a live worker of the current server — skip
// an orphaned sling/atlas subprocess that kept running after the old server // it, otherwise a newly loaded worker would kill healthy runs mid-flight.
// exited -> SIGTERM it and mark the run cancelled. // Only runs with a dead (or legacy null) owner are reconciled: dead child pid
// -> failed; live child pid -> orphaned sling/atlas subprocess, SIGTERM it
// and mark cancelled.
function reconcileOrphanRuns() { function reconcileOrphanRuns() {
const stale = db const stale = db
.prepare("SELECT * FROM runs WHERE status IN ('running', 'queued')") .prepare("SELECT * FROM runs WHERE status IN ('running', 'queued')")
.all() as Run[]; .all() as Run[];
const now = new Date().toISOString(); const now = new Date().toISOString();
for (const run of stale) { for (const run of stale) {
if (run.owner_pid != null && isProcessAlive(run.owner_pid)) {
continue;
}
const alive = run.pid != null && isProcessAlive(run.pid); const alive = run.pid != null && isProcessAlive(run.pid);
if (alive) { if (alive) {
try { try {