feat: table-scoped schema sync, optional FK migration, pipeline edit UI

- pipelines: new schema_scope ('selected'|'all') and include_fk fields with
  lightweight ALTER TABLE migration for existing DBs
- atlas: computeSchemaExcludes() limits diff/apply to selected tables via
  repeated --exclude (bare-name globs); avoids unsupported-type failures on
  unrelated tables (e.g. TiDB VECTOR) and prevents DROP of target-only tables
- fk toggle off appends --exclude '*[type=foreign-key]'; FKs referencing
  excluded parent tables are skipped by Atlas automatically
- pipeline form extracted to shared component; new /pipelines/[id]/edit page
  with full prefill; edit entries in list and detail pages
This commit is contained in:
tigerenwork 2026-08-09 17:35:55 +08:00
parent fcb0f2e299
commit 884eaa1ed8
11 changed files with 597 additions and 334 deletions

View File

@ -14,10 +14,7 @@ export async function GET(
if (!pipeline) { if (!pipeline) {
return Response.json({ error: "pipeline not found" }, { status: 404 }); return Response.json({ error: "pipeline not found" }, { status: 404 });
} }
const result = await schemaDiff( const result = await schemaDiff(pipeline);
pipeline.source_conn_id,
pipeline.target_conn_id
);
return Response.json(result); return Response.json(result);
} catch (err) { } catch (err) {
return Response.json({ error: (err as Error).message }, { status: 500 }); return Response.json({ error: (err as Error).message }, { status: 500 });

View File

@ -33,6 +33,8 @@ export async function PUT(request: Request, { params }: Ctx) {
"source_conn_id", "source_conn_id",
"target_conn_id", "target_conn_id",
"schema_sync", "schema_sync",
"schema_scope",
"include_fk",
"mode", "mode",
]) { ]) {
if (body[key] !== undefined) patch[key] = body[key]; if (body[key] !== undefined) patch[key] = body[key];

View File

@ -30,6 +30,8 @@ export async function POST(request: Request) {
source_conn_id: Number(body.source_conn_id), source_conn_id: Number(body.source_conn_id),
target_conn_id: Number(body.target_conn_id), target_conn_id: Number(body.target_conn_id),
schema_sync: body.schema_sync === undefined ? 1 : Number(body.schema_sync), schema_sync: body.schema_sync === undefined ? 1 : Number(body.schema_sync),
schema_scope: body.schema_scope ? String(body.schema_scope) : "selected",
include_fk: body.include_fk === undefined ? 0 : Number(body.include_fk),
mode: body.mode ? String(body.mode) : "full-refresh", mode: body.mode ? String(body.mode) : "full-refresh",
streams: normalizeStreams(body.streams), streams: normalizeStreams(body.streams),
}); });

View File

@ -0,0 +1,32 @@
"use client";
import { useEffect, useState } from "react";
import { useParams } from "next/navigation";
import { api, type Pipeline } from "@/components/api";
import { PipelineForm } from "@/components/pipeline-form";
export default function EditPipelinePage() {
const params = useParams<{ id: string }>();
const id = Number(params.id);
const [pipeline, setPipeline] = useState<Pipeline | null>(null);
const [error, setError] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
useEffect(() => {
api<Pipeline>(`/api/pipelines/${id}`)
.then(setPipeline)
.catch((e) => setError(e instanceof Error ? e.message : "加载失败"))
.finally(() => setLoading(false));
}, [id]);
if (loading) return <div className="text-sm text-muted-foreground"></div>;
if (error || !pipeline)
return (
<div className="text-sm text-red-500">
{error ? `加载失败:${error}` : "流水线不存在"}
</div>
);
return <PipelineForm initial={pipeline} />;
}

View File

@ -4,7 +4,7 @@ import { useCallback, useEffect, useMemo, useState } from "react";
import Link from "next/link"; import Link from "next/link";
import { useParams } from "next/navigation"; import { useParams } from "next/navigation";
import { toast } from "sonner"; import { toast } from "sonner";
import { Play, GitCompareArrows } from "lucide-react"; import { Play, GitCompareArrows, Pencil } from "lucide-react";
import { import {
api, api,
ApiError, ApiError,
@ -141,6 +141,12 @@ export default function PipelineDetailPage() {
<div className="flex items-center justify-between"> <div className="flex items-center justify-between">
<h1 className="text-xl font-semibold">{pipeline.name}</h1> <h1 className="text-xl font-semibold">{pipeline.name}</h1>
<div className="flex gap-2"> <div className="flex gap-2">
<Button variant="outline">
<Link href={`/pipelines/${id}/edit`} className="flex items-center gap-1">
<Pencil className="size-4" />
</Link>
</Button>
<Button variant="outline" onClick={previewDiff} disabled={diffLoading}> <Button variant="outline" onClick={previewDiff} disabled={diffLoading}>
<GitCompareArrows className="size-4" /> <GitCompareArrows className="size-4" />
{diffLoading ? "Diff 计算中…" : "Schema Diff 预览"} {diffLoading ? "Diff 计算中…" : "Schema Diff 预览"}
@ -187,6 +193,16 @@ export default function PipelineDetailPage() {
)} )}
</dd> </dd>
</div> </div>
<div>
<dt className="text-muted-foreground"></dt>
<dd className="font-medium">
{pipeline.schema_scope === "all" ? "整个库" : "仅选中的表"}
</dd>
</div>
<div>
<dt className="text-muted-foreground"></dt>
<dd className="font-medium">{pipeline.include_fk ? "是" : "否"}</dd>
</div>
<div> <div>
<dt className="text-muted-foreground"></dt> <dt className="text-muted-foreground"></dt>
<dd>{fmtTime(pipeline.created_at)}</dd> <dd>{fmtTime(pipeline.created_at)}</dd>

View File

@ -1,314 +1,7 @@
"use client"; "use client";
import { useEffect, useMemo, useState } from "react"; import { PipelineForm } from "@/components/pipeline-form";
import { useRouter } from "next/navigation";
import { toast } from "sonner";
import {
api,
type Connection,
type Pipeline,
type StreamConfig,
type StreamInfo,
} from "@/components/api";
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
const MODES = [
{ value: "full-refresh", label: "full-refresh全量覆盖默认" },
{ value: "truncate", label: "truncate清空后写入" },
{ value: "incremental", label: "incremental增量" },
{ value: "snapshot", label: "snapshot快照" },
];
export default function NewPipelinePage() { export default function NewPipelinePage() {
const router = useRouter(); return <PipelineForm />;
const [connections, setConnections] = useState<Connection[]>([]);
const [connError, setConnError] = useState<string | null>(null);
const [name, setName] = useState("");
const [sourceId, setSourceId] = useState("");
const [targetId, setTargetId] = useState("");
const [mode, setMode] = useState("full-refresh");
const [schemaSync, setSchemaSync] = useState(true);
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
const [streamsLoading, setStreamsLoading] = useState(false);
const [selected, setSelected] = useState<Record<string, StreamConfig>>({});
const [saving, setSaving] = useState(false);
useEffect(() => {
api<Connection[]>("/api/connections")
.then(setConnections)
.catch((e) => setConnError(e instanceof Error ? e.message : "加载失败"));
}, []);
// 切换源连接时同时清空已加载的表(在 onValueChange 里处理)
const selectedList = useMemo(() => Object.values(selected), [selected]);
async function loadStreams() {
if (!sourceId) return;
setStreamsLoading(true);
try {
const list = await api<StreamInfo[]>(
`/api/connections/${sourceId}/streams`
);
setStreamList(list);
if (list.length === 0) toast.info("该连接没有可同步的表");
} catch (e) {
toast.error(e instanceof Error ? e.message : "加载表失败");
} finally {
setStreamsLoading(false);
}
}
function toggleStream(s: StreamInfo, checked: boolean) {
setSelected((m) => {
const next = { ...m };
if (checked) next[s.name] = { name: s.name };
else delete next[s.name];
return next;
});
}
function selectAll(on: boolean) {
if (!streamList) return;
setSelected(
on
? Object.fromEntries(
streamList.map((s) => [s.name, selected[s.name] ?? { name: s.name }])
)
: {}
);
}
function setStreamField(
name: string,
field: "primary_key" | "update_key",
value: string
) {
setSelected((m) => ({
...m,
[name]: { ...m[name], name, [field]: value || undefined },
}));
}
async function submit() {
if (!name.trim()) return toast.error("请填写流水线名称");
if (!sourceId || !targetId) return toast.error("请选择源连接和目标连接");
if (sourceId === targetId) return toast.error("源连接和目标连接不能相同");
if (selectedList.length === 0) return toast.error("请至少选择一张表");
setSaving(true);
try {
const p = await api<Pipeline>("/api/pipelines", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
name: name.trim(),
source_conn_id: Number(sourceId),
target_conn_id: Number(targetId),
schema_sync: schemaSync ? 1 : 0,
mode,
streams: JSON.stringify(selectedList),
}),
});
toast.success("流水线已创建");
router.push(`/pipelines/${p.id}`);
} catch (e) {
toast.error(e instanceof Error ? e.message : "创建失败");
setSaving(false);
}
}
return (
<div className="flex max-w-3xl flex-col gap-6">
<h1 className="text-xl font-semibold">线</h1>
{connError ? (
<div className="text-sm text-red-500">{connError}</div>
) : null}
<Card>
<CardHeader>
<CardTitle></CardTitle>
</CardHeader>
<CardContent className="grid gap-4">
<div className="grid gap-1.5">
<Label htmlFor="p-name"></Label>
<Input
id="p-name"
value={name}
onChange={(e) => setName(e.target.value)}
placeholder="例如:测试库 → 本地 dev"
/>
</div>
<div className="grid grid-cols-2 gap-4">
<div className="grid gap-1.5">
<Label></Label>
<Select
value={sourceId}
onValueChange={(v) => {
setSourceId(String(v));
setStreamList(null);
setSelected({});
}}
>
<SelectTrigger className="w-full">
<SelectValue placeholder="选择源连接" />
</SelectTrigger>
<SelectContent>
{connections.map((c) => (
<SelectItem key={c.id} value={String(c.id)}>
{c.name}{c.database}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="grid gap-1.5">
<Label></Label>
<Select value={targetId} onValueChange={(v) => setTargetId(String(v))}>
<SelectTrigger className="w-full">
<SelectValue placeholder="选择目标连接" />
</SelectTrigger>
<SelectContent>
{connections.map((c) => (
<SelectItem key={c.id} value={String(c.id)}>
{c.name}{c.database}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
</div>
<div className="grid gap-1.5">
<Label></Label>
<Select value={mode} onValueChange={(v) => setMode(String(v))}>
<SelectTrigger className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
{MODES.map((m) => (
<SelectItem key={m.value} value={m.value}>
{m.label}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="flex items-center justify-between rounded-md border p-3">
<div className="flex flex-col gap-0.5">
<Label>Schema </Label>
<span className="text-xs text-muted-foreground">
Atlas schema//
</span>
</div>
<Switch checked={schemaSync} onCheckedChange={(c) => setSchemaSync(!!c)} />
</div>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle></CardTitle>
<CardDescription>
</CardDescription>
</CardHeader>
<CardContent className="flex flex-col gap-3">
<div className="flex items-center gap-2">
<Button
variant="outline"
onClick={loadStreams}
disabled={!sourceId || streamsLoading}
>
{streamsLoading ? "加载中…" : "加载表"}
</Button>
{streamList && streamList.length > 0 ? (
<>
<Button variant="ghost" size="sm" onClick={() => selectAll(true)}>
</Button>
<Button variant="ghost" size="sm" onClick={() => selectAll(false)}>
</Button>
<span className="text-xs text-muted-foreground">
{selectedList.length} / {streamList.length}
</span>
</>
) : null}
</div>
{streamList === null ? null : streamList.length === 0 ? (
<div className="text-sm text-muted-foreground"></div>
) : (
<div className="max-h-96 overflow-auto rounded-md border">
{streamList.map((s) => {
const checked = !!selected[s.name];
return (
<div
key={s.name}
className="flex items-center gap-3 border-b px-3 py-2 last:border-b-0"
>
<Checkbox
checked={checked}
onCheckedChange={(c) => toggleStream(s, !!c)}
/>
<span className="flex-1 font-mono text-sm">{s.name}</span>
{mode === "incremental" && checked ? (
<div className="flex items-center gap-2">
<Input
className="h-7 w-36 font-mono text-xs"
placeholder="primary_key"
value={selected[s.name].primary_key ?? ""}
onChange={(e) =>
setStreamField(s.name, "primary_key", e.target.value)
}
/>
<Input
className="h-7 w-36 font-mono text-xs"
placeholder="update_key"
value={selected[s.name].update_key ?? ""}
onChange={(e) =>
setStreamField(s.name, "update_key", e.target.value)
}
/>
</div>
) : null}
</div>
);
})}
</div>
)}
</CardContent>
</Card>
<div className="flex gap-2">
<Button onClick={submit} disabled={saving}>
{saving ? "创建中…" : "创建流水线"}
</Button>
<Button variant="outline" onClick={() => router.push("/pipelines")}>
</Button>
</div>
</div>
);
} }

View File

@ -122,6 +122,9 @@ export default function PipelinesPage() {
<Button variant="outline" size="sm"> <Button variant="outline" size="sm">
<Link href={`/pipelines/${p.id}`}></Link> <Link href={`/pipelines/${p.id}`}></Link>
</Button> </Button>
<Button variant="outline" size="sm">
<Link href={`/pipelines/${p.id}/edit`}></Link>
</Button>
<Button <Button
variant="outline" variant="outline"
size="sm" size="sm"

View File

@ -0,0 +1,405 @@
"use client";
import { useEffect, useMemo, useState } from "react";
import { useRouter } from "next/navigation";
import { toast } from "sonner";
import {
api,
type Connection,
type Pipeline,
type StreamConfig,
type StreamInfo,
} from "@/components/api";
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
const MODES = [
{ value: "full-refresh", label: "full-refresh全量覆盖默认" },
{ value: "truncate", label: "truncate清空后写入" },
{ value: "incremental", label: "incremental增量" },
{ value: "snapshot", label: "snapshot快照" },
];
export function PipelineForm({ initial }: { initial?: Pipeline }) {
const router = useRouter();
const isEdit = !!initial;
const [connections, setConnections] = useState<Connection[]>([]);
const [connError, setConnError] = useState<string | null>(null);
const [name, setName] = useState(initial?.name ?? "");
const [sourceId, setSourceId] = useState(
initial ? String(initial.source_conn_id) : ""
);
const [targetId, setTargetId] = useState(
initial ? String(initial.target_conn_id) : ""
);
const [mode, setMode] = useState(initial?.mode ?? "full-refresh");
const [schemaSync, setSchemaSync] = useState(
initial ? !!initial.schema_sync : true
);
const [schemaScope, setSchemaScope] = useState(
initial?.schema_scope ?? "selected"
);
const [includeFk, setIncludeFk] = useState(initial ? !!initial.include_fk : false);
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
const [streamsLoading, setStreamsLoading] = useState(false);
const [selected, setSelected] = useState<Record<string, StreamConfig>>(() => {
if (!initial) return {};
try {
const streams = JSON.parse(initial.streams || "[]") as StreamConfig[];
return Object.fromEntries(streams.map((s) => [s.name, s]));
} catch {
return {};
}
});
const [saving, setSaving] = useState(false);
useEffect(() => {
api<Connection[]>("/api/connections")
.then(setConnections)
.catch((e) => setConnError(e instanceof Error ? e.message : "加载失败"));
}, []);
// 编辑模式:加载现有流水线后自动拉取表列表,回填勾选状态
useEffect(() => {
if (isEdit && sourceId) void loadStreams();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [isEdit, sourceId]);
const selectedList = useMemo(() => Object.values(selected), [selected]);
async function loadStreams() {
if (!sourceId) return;
setStreamsLoading(true);
try {
const list = await api<StreamInfo[]>(
`/api/connections/${sourceId}/streams`
);
setStreamList(list);
if (list.length === 0) toast.info("该连接没有可同步的表");
} catch (e) {
toast.error(e instanceof Error ? e.message : "加载表失败");
} finally {
setStreamsLoading(false);
}
}
function toggleStream(s: StreamInfo, checked: boolean) {
setSelected((m) => {
const next = { ...m };
if (checked) next[s.name] = m[s.name] ?? { name: s.name };
else delete next[s.name];
return next;
});
}
function selectAll(on: boolean) {
if (!streamList) return;
setSelected(
on
? Object.fromEntries(
streamList.map((s) => [s.name, selected[s.name] ?? { name: s.name }])
)
: {}
);
}
function setStreamField(
name: string,
field: "primary_key" | "update_key",
value: string
) {
setSelected((m) => ({
...m,
[name]: { ...m[name], name, [field]: value || undefined },
}));
}
async function submit() {
if (!name.trim()) return toast.error("请填写流水线名称");
if (!sourceId || !targetId) return toast.error("请选择源连接和目标连接");
if (sourceId === targetId) return toast.error("源连接和目标连接不能相同");
if (selectedList.length === 0) return toast.error("请至少选择一张表");
setSaving(true);
const payload = {
name: name.trim(),
source_conn_id: Number(sourceId),
target_conn_id: Number(targetId),
schema_sync: schemaSync ? 1 : 0,
schema_scope: schemaScope,
include_fk: includeFk ? 1 : 0,
mode,
streams: JSON.stringify(selectedList),
};
try {
const p = isEdit
? await api<Pipeline>(`/api/pipelines/${initial.id}`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
})
: await api<Pipeline>("/api/pipelines", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
toast.success(isEdit ? "流水线已保存" : "流水线已创建");
router.push(`/pipelines/${p.id}`);
} catch (e) {
toast.error(e instanceof Error ? e.message : isEdit ? "保存失败" : "创建失败");
setSaving(false);
}
}
const cancelHref = isEdit ? `/pipelines/${initial.id}` : "/pipelines";
return (
<div className="flex max-w-3xl flex-col gap-6">
<h1 className="text-xl font-semibold">
{isEdit ? `编辑流水线:${initial.name}` : "新建流水线"}
</h1>
{connError ? (
<div className="text-sm text-red-500">{connError}</div>
) : null}
<Card>
<CardHeader>
<CardTitle></CardTitle>
</CardHeader>
<CardContent className="grid gap-4">
<div className="grid gap-1.5">
<Label htmlFor="p-name"></Label>
<Input
id="p-name"
value={name}
onChange={(e) => setName(e.target.value)}
placeholder="例如:测试库 → 本地 dev"
/>
</div>
<div className="grid grid-cols-2 gap-4">
<div className="grid gap-1.5">
<Label></Label>
<Select
value={sourceId}
onValueChange={(v) => {
setSourceId(String(v));
setStreamList(null);
setSelected({});
}}
>
<SelectTrigger className="w-full">
<SelectValue placeholder="选择源连接" />
</SelectTrigger>
<SelectContent>
{connections.map((c) => (
<SelectItem key={c.id} value={String(c.id)}>
{c.name}{c.database}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="grid gap-1.5">
<Label></Label>
<Select value={targetId} onValueChange={(v) => setTargetId(String(v))}>
<SelectTrigger className="w-full">
<SelectValue placeholder="选择目标连接" />
</SelectTrigger>
<SelectContent>
{connections.map((c) => (
<SelectItem key={c.id} value={String(c.id)}>
{c.name}{c.database}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
</div>
<div className="grid gap-1.5">
<Label></Label>
<Select value={mode} onValueChange={(v) => setMode(String(v))}>
<SelectTrigger className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
{MODES.map((m) => (
<SelectItem key={m.value} value={m.value}>
{m.label}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="flex items-center justify-between rounded-md border p-3">
<div className="flex flex-col gap-0.5">
<Label>Schema </Label>
<span className="text-xs text-muted-foreground">
Atlas schema//
</span>
</div>
<Switch checked={schemaSync} onCheckedChange={(c) => setSchemaSync(!!c)} />
</div>
<div
className={`flex items-center justify-between rounded-md border p-3 ${
schemaSync ? "" : "opacity-50"
}`}
>
<div className="flex flex-col gap-0.5">
<Label>Schema </Label>
<span className="text-xs text-muted-foreground">
</span>
</div>
<Select
value={schemaScope}
onValueChange={(v) => setSchemaScope(String(v))}
disabled={!schemaSync}
>
<SelectTrigger className="w-36">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="selected"></SelectItem>
<SelectItem value="all"></SelectItem>
</SelectContent>
</Select>
</div>
<div
className={`flex items-center justify-between rounded-md border p-3 ${
schemaSync ? "" : "opacity-50"
}`}
>
<div className="flex flex-col gap-0.5">
<Label></Label>
<span className="text-xs text-muted-foreground">
</span>
</div>
<Switch
checked={includeFk}
onCheckedChange={(c) => setIncludeFk(!!c)}
disabled={!schemaSync}
/>
</div>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle></CardTitle>
<CardDescription>
</CardDescription>
</CardHeader>
<CardContent className="flex flex-col gap-3">
<div className="flex items-center gap-2">
<Button
variant="outline"
onClick={loadStreams}
disabled={!sourceId || streamsLoading}
>
{streamsLoading ? "加载中…" : "加载表"}
</Button>
{streamList && streamList.length > 0 ? (
<>
<Button variant="ghost" size="sm" onClick={() => selectAll(true)}>
</Button>
<Button variant="ghost" size="sm" onClick={() => selectAll(false)}>
</Button>
<span className="text-xs text-muted-foreground">
{selectedList.length} / {streamList.length}
</span>
</>
) : null}
</div>
{streamList === null ? (
selectedList.length > 0 ? (
<div className="text-sm text-muted-foreground">
{selectedList.length}
{selectedList.map((s) => s.name).join("、")}
</div>
) : null
) : streamList.length === 0 ? (
<div className="text-sm text-muted-foreground"></div>
) : (
<div className="max-h-96 overflow-auto rounded-md border">
{streamList.map((s) => {
const checked = !!selected[s.name];
return (
<div
key={s.name}
className="flex items-center gap-3 border-b px-3 py-2 last:border-b-0"
>
<Checkbox
checked={checked}
onCheckedChange={(c) => toggleStream(s, !!c)}
/>
<span className="flex-1 font-mono text-sm">{s.name}</span>
{mode === "incremental" && checked ? (
<div className="flex items-center gap-2">
<Input
className="h-7 w-36 font-mono text-xs"
placeholder="primary_key"
value={selected[s.name].primary_key ?? ""}
onChange={(e) =>
setStreamField(s.name, "primary_key", e.target.value)
}
/>
<Input
className="h-7 w-36 font-mono text-xs"
placeholder="update_key"
value={selected[s.name].update_key ?? ""}
onChange={(e) =>
setStreamField(s.name, "update_key", e.target.value)
}
/>
</div>
) : null}
</div>
);
})}
</div>
)}
</CardContent>
</Card>
<div className="flex gap-2">
<Button onClick={submit} disabled={saving}>
{saving
? isEdit
? "保存中…"
: "创建中…"
: isEdit
? "保存修改"
: "创建流水线"}
</Button>
<Button variant="outline" onClick={() => router.push(cancelHref)}>
</Button>
</div>
</div>
);
}

View File

@ -1,6 +1,6 @@
import { execFile } from "child_process"; import { execFile } from "child_process";
import { Connection, getConnection } from "./db"; import { Connection, Pipeline, StreamConfig, getConnection } from "./db";
import { connectionUrl, execSql } from "./sling"; import { connectionUrl, discoverStreams, execSql } from "./sling";
import { resolveBinary } from "./binaries"; import { resolveBinary } from "./binaries";
// Atlas needs an empty scratch database ("dev database") to plan diffs. // Atlas needs an empty scratch database ("dev database") to plan diffs.
@ -23,8 +23,13 @@ export function prepareDevDatabase(tgt: Connection): string {
return devDatabaseUrl(tgt); return devDatabaseUrl(tgt);
} }
function baseApplyArgs(srcUrl: string, tgtUrl: string, devUrl: string): string[] { function baseApplyArgs(
return [ srcUrl: string,
tgtUrl: string,
devUrl: string,
excludes: string[]
): string[] {
const args = [
"schema", "schema",
"apply", "apply",
"--url", "--url",
@ -34,47 +39,112 @@ function baseApplyArgs(srcUrl: string, tgtUrl: string, devUrl: string): string[]
"--dev-url", "--dev-url",
devUrl, devUrl,
]; ];
// Community Atlas has no --include; --exclude globs match bare resource
// names (no schema prefix). Excluded objects are dropped from the diff
// entirely, so e.g. unsupported column types in them never surface.
for (const p of excludes) args.push("--exclude", p);
return args;
} }
export function buildApplyArgs( export function buildApplyArgs(
srcUrl: string, srcUrl: string,
tgtUrl: string, tgtUrl: string,
devUrl: string devUrl: string,
excludes: string[] = []
): string[] { ): string[] {
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl), "--auto-approve"]; return [...baseApplyArgs(srcUrl, tgtUrl, devUrl, excludes), "--auto-approve"];
} }
// atlas rejects --dry-run combined with --auto-approve. // atlas rejects --dry-run combined with --auto-approve.
export function buildDryRunArgs( export function buildDryRunArgs(
srcUrl: string, srcUrl: string,
tgtUrl: string, tgtUrl: string,
devUrl: string devUrl: string,
excludes: string[] = []
): string[] { ): string[] {
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl), "--dry-run"]; return [...baseApplyArgs(srcUrl, tgtUrl, devUrl, excludes), "--dry-run"];
}
// Compute the --exclude patterns for a pipeline's schema step. Atlas diffs
// whole databases, and tables that exist only on the target would be planned
// as DROPs — so for 'selected' scope we must exclude every non-selected table
// from BOTH sides. Bare table names only: exclude globs match resource names
// without schema prefixes.
export async function computeSchemaExcludes(
pipeline: Pipeline
): Promise<string[]> {
if ((pipeline.schema_scope || "selected") === "all") return [];
const streams = JSON.parse(pipeline.streams || "[]") as StreamConfig[];
const selected = new Set(
streams.map((s) => {
const i = s.name.lastIndexOf(".");
return i >= 0 ? s.name.slice(i + 1) : s.name;
})
);
const tableNames = (connId: number, side: string): Set<string> => {
try {
return new Set(
discoverStreams(connId).map((s) => {
const i = s.name.lastIndexOf(".");
return i >= 0 ? s.name.slice(i + 1) : s.name;
})
);
} catch (e) {
// Silently falling back would risk DROPping target-only tables.
throw new Error(
`无法获取${side}库表列表,已中止 schema 同步:${(e as Error).message}`
);
}
};
const srcTables = tableNames(pipeline.source_conn_id, "源");
const tgtTables = tableNames(pipeline.target_conn_id, "目标");
const excludes: string[] = [];
for (const t of new Set([...srcTables, ...tgtTables])) {
if (!selected.has(t)) excludes.push(t);
}
// FKs referencing excluded tables vanish from the desired state anyway;
// excluding all FKs additionally avoids touching constraints on the
// selected tables themselves.
if (!pipeline.include_fk) excludes.push("*[type=foreign-key]");
return excludes;
} }
function atlasBin(): string { function atlasBin(): string {
return resolveBinary("atlas") ?? "atlas"; return resolveBinary("atlas") ?? "atlas";
} }
export function schemaDiff( export async function schemaDiff(
sourceConnId: number, pipeline: Pipeline
targetConnId: number
): Promise<{ ok: boolean; sql: string }> { ): Promise<{ ok: boolean; sql: string }> {
const src = getConnection(sourceConnId); const src = getConnection(pipeline.source_conn_id);
const tgt = getConnection(targetConnId); const tgt = getConnection(pipeline.target_conn_id);
if (!src || !tgt) throw new Error("connection not found"); if (!src || !tgt) throw new Error("connection not found");
let excludes: string[];
try {
excludes = await computeSchemaExcludes(pipeline);
} catch (e) {
return { ok: false, sql: (e as Error).message };
}
let devUrl: string; let devUrl: string;
try { try {
devUrl = prepareDevDatabase(tgt); devUrl = prepareDevDatabase(tgt);
} catch (e) { } catch (e) {
const err = e as { stdout?: string; stderr?: string; message?: string }; const err = e as { stdout?: string; stderr?: string; message?: string };
return Promise.resolve({ return {
ok: false, ok: false,
sql: [err.stdout, err.stderr, err.message].filter(Boolean).join("\n"), sql: [err.stdout, err.stderr, err.message].filter(Boolean).join("\n"),
}); };
} }
const args = buildDryRunArgs(connectionUrl(src), connectionUrl(tgt), devUrl); const args = buildDryRunArgs(
connectionUrl(src),
connectionUrl(tgt),
devUrl,
excludes
);
return new Promise((resolve) => { return new Promise((resolve) => {
execFile( execFile(
/* turbopackIgnore: true */ atlasBin(), /* turbopackIgnore: true */ atlasBin(),

View File

@ -41,6 +41,8 @@ export interface Pipeline {
source_conn_id: number; source_conn_id: number;
target_conn_id: number; target_conn_id: number;
schema_sync: number; // 1 | 0 schema_sync: number; // 1 | 0
schema_scope: string; // 'selected' | 'all'
include_fk: number; // 1 | 0
mode: string; mode: string;
streams: string; // JSON: StreamConfig[] streams: string; // JSON: StreamConfig[]
created_at: string; created_at: string;
@ -52,6 +54,8 @@ export interface PipelineInput {
source_conn_id: number; source_conn_id: number;
target_conn_id: number; target_conn_id: number;
schema_sync?: number; schema_sync?: number;
schema_scope?: string;
include_fk?: number;
mode?: string; mode?: string;
streams?: string; streams?: string;
} }
@ -91,6 +95,8 @@ function createDb(): Database.Database {
source_conn_id INTEGER NOT NULL REFERENCES connections(id), source_conn_id INTEGER NOT NULL REFERENCES connections(id),
target_conn_id INTEGER NOT NULL REFERENCES connections(id), target_conn_id INTEGER NOT NULL REFERENCES connections(id),
schema_sync INTEGER DEFAULT 1, schema_sync INTEGER DEFAULT 1,
schema_scope TEXT NOT NULL DEFAULT 'selected',
include_fk INTEGER NOT NULL DEFAULT 0,
mode TEXT DEFAULT 'full-refresh', mode TEXT DEFAULT 'full-refresh',
streams TEXT NOT NULL DEFAULT '[]', streams TEXT NOT NULL DEFAULT '[]',
created_at TEXT DEFAULT (datetime('now')), created_at TEXT DEFAULT (datetime('now')),
@ -107,6 +113,30 @@ function createDb(): Database.Database {
pid INTEGER pid INTEGER
); );
`); `);
// Lightweight migrations for databases created before these columns existed.
// Several Next.js workers may open the DB concurrently during build, so a
// racing ALTER can fail with "duplicate column name" — safe to ignore.
const addColumnIfMissing = (sql: string, column: string) => {
const cols = new Set(
(db.pragma("table_info(pipelines)") as { name: string }[]).map(
(c) => c.name
)
);
if (cols.has(column)) return;
try {
db.exec(sql);
} catch (e) {
if (!(e as Error).message.includes("duplicate column name")) throw e;
}
};
addColumnIfMissing(
"ALTER TABLE pipelines ADD COLUMN schema_scope TEXT NOT NULL DEFAULT 'selected'",
"schema_scope"
);
addColumnIfMissing(
"ALTER TABLE pipelines ADD COLUMN include_fk INTEGER NOT NULL DEFAULT 0",
"include_fk"
);
return db; return db;
} }
@ -186,14 +216,16 @@ export function getPipeline(id: number): Pipeline | null {
export function createPipeline(input: PipelineInput): Pipeline { export function createPipeline(input: PipelineInput): Pipeline {
const res = db const res = db
.prepare( .prepare(
`INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, mode, streams) `INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, schema_scope, include_fk, mode, streams)
VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @mode, @streams)` VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @schema_scope, @include_fk, @mode, @streams)`
) )
.run({ .run({
name: input.name, name: input.name,
source_conn_id: input.source_conn_id, source_conn_id: input.source_conn_id,
target_conn_id: input.target_conn_id, target_conn_id: input.target_conn_id,
schema_sync: input.schema_sync ?? 1, schema_sync: input.schema_sync ?? 1,
schema_scope: input.schema_scope ?? "selected",
include_fk: input.include_fk ?? 0,
mode: input.mode ?? "full-refresh", mode: input.mode ?? "full-refresh",
streams: input.streams ?? "[]", streams: input.streams ?? "[]",
}); });
@ -209,6 +241,8 @@ export function updatePipeline(
"source_conn_id", "source_conn_id",
"target_conn_id", "target_conn_id",
"schema_sync", "schema_sync",
"schema_scope",
"include_fk",
"mode", "mode",
"streams", "streams",
].filter((k) => input[k as keyof PipelineInput] !== undefined); ].filter((k) => input[k as keyof PipelineInput] !== undefined);

View File

@ -12,7 +12,7 @@ import {
updateRunStatus, updateRunStatus,
} from "./db"; } from "./db";
import { SLING_HOME, buildReplicationYaml, connectionUrl } from "./sling"; import { SLING_HOME, buildReplicationYaml, connectionUrl } from "./sling";
import { buildApplyArgs, prepareDevDatabase } from "./atlas"; import { buildApplyArgs, computeSchemaExcludes, prepareDevDatabase } from "./atlas";
import { resolveBinary } from "./binaries"; import { resolveBinary } from "./binaries";
export interface JobEvent { export interface JobEvent {
@ -137,11 +137,20 @@ class JobManager {
const src = getConnection(pipeline.source_conn_id); const src = getConnection(pipeline.source_conn_id);
const tgt = getConnection(pipeline.target_conn_id); const tgt = getConnection(pipeline.target_conn_id);
if (!src || !tgt) throw new Error("pipeline connections not found"); if (!src || !tgt) throw new Error("pipeline connections not found");
const excludes = await computeSchemaExcludes(pipeline);
const scope = pipeline.schema_scope || "selected";
const tableExcludes = pipeline.include_fk
? excludes.length
: excludes.filter((p) => p !== "*[type=foreign-key]").length;
this.emit(
state,
`[schema] scope=${scope}, excluding ${tableExcludes} objects, fk=${pipeline.include_fk ? "on" : "off"}`
);
const devUrl = prepareDevDatabase(tgt); const devUrl = prepareDevDatabase(tgt);
const code = await this.runProcess( const code = await this.runProcess(
state, state,
resolveBinary("atlas") ?? "atlas", resolveBinary("atlas") ?? "atlas",
buildApplyArgs(connectionUrl(src), connectionUrl(tgt), devUrl), buildApplyArgs(connectionUrl(src), connectionUrl(tgt), devUrl, excludes),
{} {}
); );
if (code !== 0) { if (code !== 0) {