xorbitlab
/
sing-ui
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: stop orphan reconciliation from killing healthy in-flight runs 

Runs now record owner_pid (the Node process that started them) at creation
time. reconcileOrphanRuns skips runs whose owner process is still alive —
previously a newly loaded worker (dev compile / instrumentation) would see a
running run with pid not yet written (during the atlas stage), mark it failed
mid-flight, and disable the stop button, while sling kept running in the
background. Legacy rows with null owner_pid are still reconciled as before.
This commit is contained in:
tigerenwork 2026-08-09 23:07:24 +08:00
parent 088925cee4
commit 0b7dc0e86e
2 changed files with 22 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
lib/db.ts
View File

@ -82,6 +82,7 @@ export interface Run {
log_path: string | null;
error: string | null;
pid: number | null;
owner_pid: number | null;
}
function createDb(): Database.Database {
@ -123,7 +124,8 @@ function createDb(): Database.Database {
finished_at TEXT,
log_path TEXT,
error TEXT,
pid INTEGER
pid INTEGER,
owner_pid INTEGER
);
`);
// Lightweight migrations for databases created before these columns existed.
@ -162,6 +164,11 @@ function createDb(): Database.Database {
"ALTER TABLE pipelines ADD COLUMN env TEXT NOT NULL DEFAULT '{}'",
"env"
);
addColumnIfMissing(
"runs",
"ALTER TABLE runs ADD COLUMN owner_pid INTEGER",
"owner_pid"
);
return db;
}
@ -294,11 +301,12 @@ export function createRun(input: {
started_at?: string;
log_path?: string;
pid?: number;
owner_pid?: number;
}): Run {
const res = db
.prepare(
`INSERT INTO runs (pipeline_id, status, started_at, log_path, pid)
VALUES (@pipeline_id, @status, @started_at, @log_path, @pid)`
`INSERT INTO runs (pipeline_id, status, started_at, log_path, pid, owner_pid)
VALUES (@pipeline_id, @status, @started_at, @log_path, @pid, @owner_pid)`
)
.run({
pipeline_id: input.pipeline_id,
@ -306,6 +314,7 @@ export function createRun(input: {
started_at: input.started_at ?? null,
log_path: input.log_path ?? null,
pid: input.pid ?? null,
owner_pid: input.owner_pid ?? null,
});
return getRun(Number(res.lastInsertRowid))!;
}

14
lib/jobs.ts
View File

@ -84,6 +84,7 @@ class JobManager {
pipeline_id: pipelineId,
status: "running",
started_at: new Date().toISOString(),
owner_pid: process.pid,
});
const runsDir = path.join(DATA_DIR, "runs");
fs.mkdirSync(runsDir, { recursive: true });
@ -324,16 +325,21 @@ export const jobManager: JobManager =
g.__jobManager ?? (g.__jobManager = new JobManager());
// Reconcile runs left in running/queued by a previous server process.
// A run whose recorded pid is dead is stale (the subprocess died with the
// old server or never existed) -> failed. A run whose pid is still alive is
// an orphaned sling/atlas subprocess that kept running after the old server
// exited -> SIGTERM it and mark the run cancelled.
// Each run records owner_pid (the Node process that started it). A run whose
// owner is still alive belongs to a live worker of the current server — skip
// it, otherwise a newly loaded worker would kill healthy runs mid-flight.
// Only runs with a dead (or legacy null) owner are reconciled: dead child pid
// -> failed; live child pid -> orphaned sling/atlas subprocess, SIGTERM it
// and mark cancelled.
function reconcileOrphanRuns() {
const stale = db
.prepare("SELECT * FROM runs WHERE status IN ('running', 'queued')")
.all() as Run[];
const now = new Date().toISOString();
for (const run of stale) {
if (run.owner_pid != null && isProcessAlive(run.owner_pid)) {
continue;
}
const alive = run.pid != null && isProcessAlive(run.pid);
if (alive) {
try {