fix: stop orphan reconciliation from killing healthy in-flight runs
Runs now record owner_pid (the Node process that started them) at creation time. reconcileOrphanRuns skips runs whose owner process is still alive — previously a newly loaded worker (dev compile / instrumentation) would see a running run with pid not yet written (during the atlas stage), mark it failed mid-flight, and disable the stop button, while sling kept running in the background. Legacy rows with null owner_pid are still reconciled as before.
This commit is contained in:
parent
088925cee4
commit
0b7dc0e86e
15
lib/db.ts
15
lib/db.ts
|
|
@ -82,6 +82,7 @@ export interface Run {
|
||||||
log_path: string | null;
|
log_path: string | null;
|
||||||
error: string | null;
|
error: string | null;
|
||||||
pid: number | null;
|
pid: number | null;
|
||||||
|
owner_pid: number | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function createDb(): Database.Database {
|
function createDb(): Database.Database {
|
||||||
|
|
@ -123,7 +124,8 @@ function createDb(): Database.Database {
|
||||||
finished_at TEXT,
|
finished_at TEXT,
|
||||||
log_path TEXT,
|
log_path TEXT,
|
||||||
error TEXT,
|
error TEXT,
|
||||||
pid INTEGER
|
pid INTEGER,
|
||||||
|
owner_pid INTEGER
|
||||||
);
|
);
|
||||||
`);
|
`);
|
||||||
// Lightweight migrations for databases created before these columns existed.
|
// Lightweight migrations for databases created before these columns existed.
|
||||||
|
|
@ -162,6 +164,11 @@ function createDb(): Database.Database {
|
||||||
"ALTER TABLE pipelines ADD COLUMN env TEXT NOT NULL DEFAULT '{}'",
|
"ALTER TABLE pipelines ADD COLUMN env TEXT NOT NULL DEFAULT '{}'",
|
||||||
"env"
|
"env"
|
||||||
);
|
);
|
||||||
|
addColumnIfMissing(
|
||||||
|
"runs",
|
||||||
|
"ALTER TABLE runs ADD COLUMN owner_pid INTEGER",
|
||||||
|
"owner_pid"
|
||||||
|
);
|
||||||
return db;
|
return db;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -294,11 +301,12 @@ export function createRun(input: {
|
||||||
started_at?: string;
|
started_at?: string;
|
||||||
log_path?: string;
|
log_path?: string;
|
||||||
pid?: number;
|
pid?: number;
|
||||||
|
owner_pid?: number;
|
||||||
}): Run {
|
}): Run {
|
||||||
const res = db
|
const res = db
|
||||||
.prepare(
|
.prepare(
|
||||||
`INSERT INTO runs (pipeline_id, status, started_at, log_path, pid)
|
`INSERT INTO runs (pipeline_id, status, started_at, log_path, pid, owner_pid)
|
||||||
VALUES (@pipeline_id, @status, @started_at, @log_path, @pid)`
|
VALUES (@pipeline_id, @status, @started_at, @log_path, @pid, @owner_pid)`
|
||||||
)
|
)
|
||||||
.run({
|
.run({
|
||||||
pipeline_id: input.pipeline_id,
|
pipeline_id: input.pipeline_id,
|
||||||
|
|
@ -306,6 +314,7 @@ export function createRun(input: {
|
||||||
started_at: input.started_at ?? null,
|
started_at: input.started_at ?? null,
|
||||||
log_path: input.log_path ?? null,
|
log_path: input.log_path ?? null,
|
||||||
pid: input.pid ?? null,
|
pid: input.pid ?? null,
|
||||||
|
owner_pid: input.owner_pid ?? null,
|
||||||
});
|
});
|
||||||
return getRun(Number(res.lastInsertRowid))!;
|
return getRun(Number(res.lastInsertRowid))!;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
14
lib/jobs.ts
14
lib/jobs.ts
|
|
@ -84,6 +84,7 @@ class JobManager {
|
||||||
pipeline_id: pipelineId,
|
pipeline_id: pipelineId,
|
||||||
status: "running",
|
status: "running",
|
||||||
started_at: new Date().toISOString(),
|
started_at: new Date().toISOString(),
|
||||||
|
owner_pid: process.pid,
|
||||||
});
|
});
|
||||||
const runsDir = path.join(DATA_DIR, "runs");
|
const runsDir = path.join(DATA_DIR, "runs");
|
||||||
fs.mkdirSync(runsDir, { recursive: true });
|
fs.mkdirSync(runsDir, { recursive: true });
|
||||||
|
|
@ -324,16 +325,21 @@ export const jobManager: JobManager =
|
||||||
g.__jobManager ?? (g.__jobManager = new JobManager());
|
g.__jobManager ?? (g.__jobManager = new JobManager());
|
||||||
|
|
||||||
// Reconcile runs left in running/queued by a previous server process.
|
// Reconcile runs left in running/queued by a previous server process.
|
||||||
// A run whose recorded pid is dead is stale (the subprocess died with the
|
// Each run records owner_pid (the Node process that started it). A run whose
|
||||||
// old server or never existed) -> failed. A run whose pid is still alive is
|
// owner is still alive belongs to a live worker of the current server — skip
|
||||||
// an orphaned sling/atlas subprocess that kept running after the old server
|
// it, otherwise a newly loaded worker would kill healthy runs mid-flight.
|
||||||
// exited -> SIGTERM it and mark the run cancelled.
|
// Only runs with a dead (or legacy null) owner are reconciled: dead child pid
|
||||||
|
// -> failed; live child pid -> orphaned sling/atlas subprocess, SIGTERM it
|
||||||
|
// and mark cancelled.
|
||||||
function reconcileOrphanRuns() {
|
function reconcileOrphanRuns() {
|
||||||
const stale = db
|
const stale = db
|
||||||
.prepare("SELECT * FROM runs WHERE status IN ('running', 'queued')")
|
.prepare("SELECT * FROM runs WHERE status IN ('running', 'queued')")
|
||||||
.all() as Run[];
|
.all() as Run[];
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
for (const run of stale) {
|
for (const run of stale) {
|
||||||
|
if (run.owner_pid != null && isProcessAlive(run.owner_pid)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const alive = run.pid != null && isProcessAlive(run.pid);
|
const alive = run.pid != null && isProcessAlive(run.pid);
|
||||||
if (alive) {
|
if (alive) {
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue