feat: table-scoped schema sync, optional FK migration, pipeline edit UI
- pipelines: new schema_scope ('selected'|'all') and include_fk fields with
lightweight ALTER TABLE migration for existing DBs
- atlas: computeSchemaExcludes() limits diff/apply to selected tables via
repeated --exclude (bare-name globs); avoids unsupported-type failures on
unrelated tables (e.g. TiDB VECTOR) and prevents DROP of target-only tables
- fk toggle off appends --exclude '*[type=foreign-key]'; FKs referencing
excluded parent tables are skipped by Atlas automatically
- pipeline form extracted to shared component; new /pipelines/[id]/edit page
with full prefill; edit entries in list and detail pages
This commit is contained in:
parent
fcb0f2e299
commit
884eaa1ed8
|
|
@ -14,10 +14,7 @@ export async function GET(
|
|||
if (!pipeline) {
|
||||
return Response.json({ error: "pipeline not found" }, { status: 404 });
|
||||
}
|
||||
const result = await schemaDiff(
|
||||
pipeline.source_conn_id,
|
||||
pipeline.target_conn_id
|
||||
);
|
||||
const result = await schemaDiff(pipeline);
|
||||
return Response.json(result);
|
||||
} catch (err) {
|
||||
return Response.json({ error: (err as Error).message }, { status: 500 });
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ export async function PUT(request: Request, { params }: Ctx) {
|
|||
"source_conn_id",
|
||||
"target_conn_id",
|
||||
"schema_sync",
|
||||
"schema_scope",
|
||||
"include_fk",
|
||||
"mode",
|
||||
]) {
|
||||
if (body[key] !== undefined) patch[key] = body[key];
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ export async function POST(request: Request) {
|
|||
source_conn_id: Number(body.source_conn_id),
|
||||
target_conn_id: Number(body.target_conn_id),
|
||||
schema_sync: body.schema_sync === undefined ? 1 : Number(body.schema_sync),
|
||||
schema_scope: body.schema_scope ? String(body.schema_scope) : "selected",
|
||||
include_fk: body.include_fk === undefined ? 0 : Number(body.include_fk),
|
||||
mode: body.mode ? String(body.mode) : "full-refresh",
|
||||
streams: normalizeStreams(body.streams),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import { useParams } from "next/navigation";
|
||||
import { api, type Pipeline } from "@/components/api";
|
||||
import { PipelineForm } from "@/components/pipeline-form";
|
||||
|
||||
export default function EditPipelinePage() {
|
||||
const params = useParams<{ id: string }>();
|
||||
const id = Number(params.id);
|
||||
|
||||
const [pipeline, setPipeline] = useState<Pipeline | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
api<Pipeline>(`/api/pipelines/${id}`)
|
||||
.then(setPipeline)
|
||||
.catch((e) => setError(e instanceof Error ? e.message : "加载失败"))
|
||||
.finally(() => setLoading(false));
|
||||
}, [id]);
|
||||
|
||||
if (loading) return <div className="text-sm text-muted-foreground">加载中…</div>;
|
||||
if (error || !pipeline)
|
||||
return (
|
||||
<div className="text-sm text-red-500">
|
||||
{error ? `加载失败:${error}` : "流水线不存在"}
|
||||
</div>
|
||||
);
|
||||
|
||||
return <PipelineForm initial={pipeline} />;
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@ import { useCallback, useEffect, useMemo, useState } from "react";
|
|||
import Link from "next/link";
|
||||
import { useParams } from "next/navigation";
|
||||
import { toast } from "sonner";
|
||||
import { Play, GitCompareArrows } from "lucide-react";
|
||||
import { Play, GitCompareArrows, Pencil } from "lucide-react";
|
||||
import {
|
||||
api,
|
||||
ApiError,
|
||||
|
|
@ -141,6 +141,12 @@ export default function PipelineDetailPage() {
|
|||
<div className="flex items-center justify-between">
|
||||
<h1 className="text-xl font-semibold">{pipeline.name}</h1>
|
||||
<div className="flex gap-2">
|
||||
<Button variant="outline">
|
||||
<Link href={`/pipelines/${id}/edit`} className="flex items-center gap-1">
|
||||
<Pencil className="size-4" />
|
||||
编辑
|
||||
</Link>
|
||||
</Button>
|
||||
<Button variant="outline" onClick={previewDiff} disabled={diffLoading}>
|
||||
<GitCompareArrows className="size-4" />
|
||||
{diffLoading ? "Diff 计算中…" : "Schema Diff 预览"}
|
||||
|
|
@ -187,6 +193,16 @@ export default function PipelineDetailPage() {
|
|||
)}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt className="text-muted-foreground">同步范围</dt>
|
||||
<dd className="font-medium">
|
||||
{pipeline.schema_scope === "all" ? "整个库" : "仅选中的表"}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt className="text-muted-foreground">包含外键</dt>
|
||||
<dd className="font-medium">{pipeline.include_fk ? "是" : "否"}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt className="text-muted-foreground">创建时间</dt>
|
||||
<dd>{fmtTime(pipeline.created_at)}</dd>
|
||||
|
|
|
|||
|
|
@ -1,314 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
api,
|
||||
type Connection,
|
||||
type Pipeline,
|
||||
type StreamConfig,
|
||||
type StreamInfo,
|
||||
} from "@/components/api";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
|
||||
const MODES = [
|
||||
{ value: "full-refresh", label: "full-refresh(全量覆盖,默认)" },
|
||||
{ value: "truncate", label: "truncate(清空后写入)" },
|
||||
{ value: "incremental", label: "incremental(增量)" },
|
||||
{ value: "snapshot", label: "snapshot(快照)" },
|
||||
];
|
||||
import { PipelineForm } from "@/components/pipeline-form";
|
||||
|
||||
export default function NewPipelinePage() {
|
||||
const router = useRouter();
|
||||
const [connections, setConnections] = useState<Connection[]>([]);
|
||||
const [connError, setConnError] = useState<string | null>(null);
|
||||
|
||||
const [name, setName] = useState("");
|
||||
const [sourceId, setSourceId] = useState("");
|
||||
const [targetId, setTargetId] = useState("");
|
||||
const [mode, setMode] = useState("full-refresh");
|
||||
const [schemaSync, setSchemaSync] = useState(true);
|
||||
|
||||
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
|
||||
const [streamsLoading, setStreamsLoading] = useState(false);
|
||||
const [selected, setSelected] = useState<Record<string, StreamConfig>>({});
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
api<Connection[]>("/api/connections")
|
||||
.then(setConnections)
|
||||
.catch((e) => setConnError(e instanceof Error ? e.message : "加载失败"));
|
||||
}, []);
|
||||
|
||||
// 切换源连接时同时清空已加载的表(在 onValueChange 里处理)
|
||||
|
||||
const selectedList = useMemo(() => Object.values(selected), [selected]);
|
||||
|
||||
async function loadStreams() {
|
||||
if (!sourceId) return;
|
||||
setStreamsLoading(true);
|
||||
try {
|
||||
const list = await api<StreamInfo[]>(
|
||||
`/api/connections/${sourceId}/streams`
|
||||
);
|
||||
setStreamList(list);
|
||||
if (list.length === 0) toast.info("该连接没有可同步的表");
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "加载表失败");
|
||||
} finally {
|
||||
setStreamsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
function toggleStream(s: StreamInfo, checked: boolean) {
|
||||
setSelected((m) => {
|
||||
const next = { ...m };
|
||||
if (checked) next[s.name] = { name: s.name };
|
||||
else delete next[s.name];
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
function selectAll(on: boolean) {
|
||||
if (!streamList) return;
|
||||
setSelected(
|
||||
on
|
||||
? Object.fromEntries(
|
||||
streamList.map((s) => [s.name, selected[s.name] ?? { name: s.name }])
|
||||
)
|
||||
: {}
|
||||
);
|
||||
}
|
||||
|
||||
function setStreamField(
|
||||
name: string,
|
||||
field: "primary_key" | "update_key",
|
||||
value: string
|
||||
) {
|
||||
setSelected((m) => ({
|
||||
...m,
|
||||
[name]: { ...m[name], name, [field]: value || undefined },
|
||||
}));
|
||||
}
|
||||
|
||||
async function submit() {
|
||||
if (!name.trim()) return toast.error("请填写流水线名称");
|
||||
if (!sourceId || !targetId) return toast.error("请选择源连接和目标连接");
|
||||
if (sourceId === targetId) return toast.error("源连接和目标连接不能相同");
|
||||
if (selectedList.length === 0) return toast.error("请至少选择一张表");
|
||||
|
||||
setSaving(true);
|
||||
try {
|
||||
const p = await api<Pipeline>("/api/pipelines", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
name: name.trim(),
|
||||
source_conn_id: Number(sourceId),
|
||||
target_conn_id: Number(targetId),
|
||||
schema_sync: schemaSync ? 1 : 0,
|
||||
mode,
|
||||
streams: JSON.stringify(selectedList),
|
||||
}),
|
||||
});
|
||||
toast.success("流水线已创建");
|
||||
router.push(`/pipelines/${p.id}`);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "创建失败");
|
||||
setSaving(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex max-w-3xl flex-col gap-6">
|
||||
<h1 className="text-xl font-semibold">新建流水线</h1>
|
||||
|
||||
{connError ? (
|
||||
<div className="text-sm text-red-500">连接列表加载失败:{connError}</div>
|
||||
) : null}
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>基本信息</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="grid gap-4">
|
||||
<div className="grid gap-1.5">
|
||||
<Label htmlFor="p-name">名称</Label>
|
||||
<Input
|
||||
id="p-name"
|
||||
value={name}
|
||||
onChange={(e) => setName(e.target.value)}
|
||||
placeholder="例如:测试库 → 本地 dev"
|
||||
/>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="grid gap-1.5">
|
||||
<Label>源连接</Label>
|
||||
<Select
|
||||
value={sourceId}
|
||||
onValueChange={(v) => {
|
||||
setSourceId(String(v));
|
||||
setStreamList(null);
|
||||
setSelected({});
|
||||
}}
|
||||
>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder="选择源连接" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{connections.map((c) => (
|
||||
<SelectItem key={c.id} value={String(c.id)}>
|
||||
{c.name}({c.database})
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div className="grid gap-1.5">
|
||||
<Label>目标连接</Label>
|
||||
<Select value={targetId} onValueChange={(v) => setTargetId(String(v))}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder="选择目标连接" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{connections.map((c) => (
|
||||
<SelectItem key={c.id} value={String(c.id)}>
|
||||
{c.name}({c.database})
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid gap-1.5">
|
||||
<Label>同步模式</Label>
|
||||
<Select value={mode} onValueChange={(v) => setMode(String(v))}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{MODES.map((m) => (
|
||||
<SelectItem key={m.value} value={m.value}>
|
||||
{m.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div className="flex items-center justify-between rounded-md border p-3">
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<Label>Schema 同步</Label>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
先用 Atlas 同步 schema(表/索引/约束),再迁移数据
|
||||
</span>
|
||||
</div>
|
||||
<Switch checked={schemaSync} onCheckedChange={(c) => setSchemaSync(!!c)} />
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>选择要同步的表</CardTitle>
|
||||
<CardDescription>
|
||||
选择源连接后点击「加载表」,勾选需要同步的表。
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="flex flex-col gap-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={loadStreams}
|
||||
disabled={!sourceId || streamsLoading}
|
||||
>
|
||||
{streamsLoading ? "加载中…" : "加载表"}
|
||||
</Button>
|
||||
{streamList && streamList.length > 0 ? (
|
||||
<>
|
||||
<Button variant="ghost" size="sm" onClick={() => selectAll(true)}>
|
||||
全选
|
||||
</Button>
|
||||
<Button variant="ghost" size="sm" onClick={() => selectAll(false)}>
|
||||
全不选
|
||||
</Button>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
已选 {selectedList.length} / {streamList.length}
|
||||
</span>
|
||||
</>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{streamList === null ? null : streamList.length === 0 ? (
|
||||
<div className="text-sm text-muted-foreground">没有可同步的表。</div>
|
||||
) : (
|
||||
<div className="max-h-96 overflow-auto rounded-md border">
|
||||
{streamList.map((s) => {
|
||||
const checked = !!selected[s.name];
|
||||
return (
|
||||
<div
|
||||
key={s.name}
|
||||
className="flex items-center gap-3 border-b px-3 py-2 last:border-b-0"
|
||||
>
|
||||
<Checkbox
|
||||
checked={checked}
|
||||
onCheckedChange={(c) => toggleStream(s, !!c)}
|
||||
/>
|
||||
<span className="flex-1 font-mono text-sm">{s.name}</span>
|
||||
{mode === "incremental" && checked ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
className="h-7 w-36 font-mono text-xs"
|
||||
placeholder="primary_key"
|
||||
value={selected[s.name].primary_key ?? ""}
|
||||
onChange={(e) =>
|
||||
setStreamField(s.name, "primary_key", e.target.value)
|
||||
}
|
||||
/>
|
||||
<Input
|
||||
className="h-7 w-36 font-mono text-xs"
|
||||
placeholder="update_key"
|
||||
value={selected[s.name].update_key ?? ""}
|
||||
onChange={(e) =>
|
||||
setStreamField(s.name, "update_key", e.target.value)
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<div className="flex gap-2">
|
||||
<Button onClick={submit} disabled={saving}>
|
||||
{saving ? "创建中…" : "创建流水线"}
|
||||
</Button>
|
||||
<Button variant="outline" onClick={() => router.push("/pipelines")}>
|
||||
取消
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
return <PipelineForm />;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -122,6 +122,9 @@ export default function PipelinesPage() {
|
|||
<Button variant="outline" size="sm">
|
||||
<Link href={`/pipelines/${p.id}`}>详情</Link>
|
||||
</Button>
|
||||
<Button variant="outline" size="sm">
|
||||
<Link href={`/pipelines/${p.id}/edit`}>编辑</Link>
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,405 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
api,
|
||||
type Connection,
|
||||
type Pipeline,
|
||||
type StreamConfig,
|
||||
type StreamInfo,
|
||||
} from "@/components/api";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
|
||||
const MODES = [
|
||||
{ value: "full-refresh", label: "full-refresh(全量覆盖,默认)" },
|
||||
{ value: "truncate", label: "truncate(清空后写入)" },
|
||||
{ value: "incremental", label: "incremental(增量)" },
|
||||
{ value: "snapshot", label: "snapshot(快照)" },
|
||||
];
|
||||
|
||||
export function PipelineForm({ initial }: { initial?: Pipeline }) {
|
||||
const router = useRouter();
|
||||
const isEdit = !!initial;
|
||||
const [connections, setConnections] = useState<Connection[]>([]);
|
||||
const [connError, setConnError] = useState<string | null>(null);
|
||||
|
||||
const [name, setName] = useState(initial?.name ?? "");
|
||||
const [sourceId, setSourceId] = useState(
|
||||
initial ? String(initial.source_conn_id) : ""
|
||||
);
|
||||
const [targetId, setTargetId] = useState(
|
||||
initial ? String(initial.target_conn_id) : ""
|
||||
);
|
||||
const [mode, setMode] = useState(initial?.mode ?? "full-refresh");
|
||||
const [schemaSync, setSchemaSync] = useState(
|
||||
initial ? !!initial.schema_sync : true
|
||||
);
|
||||
const [schemaScope, setSchemaScope] = useState(
|
||||
initial?.schema_scope ?? "selected"
|
||||
);
|
||||
const [includeFk, setIncludeFk] = useState(initial ? !!initial.include_fk : false);
|
||||
|
||||
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
|
||||
const [streamsLoading, setStreamsLoading] = useState(false);
|
||||
const [selected, setSelected] = useState<Record<string, StreamConfig>>(() => {
|
||||
if (!initial) return {};
|
||||
try {
|
||||
const streams = JSON.parse(initial.streams || "[]") as StreamConfig[];
|
||||
return Object.fromEntries(streams.map((s) => [s.name, s]));
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
});
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
api<Connection[]>("/api/connections")
|
||||
.then(setConnections)
|
||||
.catch((e) => setConnError(e instanceof Error ? e.message : "加载失败"));
|
||||
}, []);
|
||||
|
||||
// 编辑模式:加载现有流水线后自动拉取表列表,回填勾选状态
|
||||
useEffect(() => {
|
||||
if (isEdit && sourceId) void loadStreams();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [isEdit, sourceId]);
|
||||
|
||||
const selectedList = useMemo(() => Object.values(selected), [selected]);
|
||||
|
||||
async function loadStreams() {
|
||||
if (!sourceId) return;
|
||||
setStreamsLoading(true);
|
||||
try {
|
||||
const list = await api<StreamInfo[]>(
|
||||
`/api/connections/${sourceId}/streams`
|
||||
);
|
||||
setStreamList(list);
|
||||
if (list.length === 0) toast.info("该连接没有可同步的表");
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "加载表失败");
|
||||
} finally {
|
||||
setStreamsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
function toggleStream(s: StreamInfo, checked: boolean) {
|
||||
setSelected((m) => {
|
||||
const next = { ...m };
|
||||
if (checked) next[s.name] = m[s.name] ?? { name: s.name };
|
||||
else delete next[s.name];
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
function selectAll(on: boolean) {
|
||||
if (!streamList) return;
|
||||
setSelected(
|
||||
on
|
||||
? Object.fromEntries(
|
||||
streamList.map((s) => [s.name, selected[s.name] ?? { name: s.name }])
|
||||
)
|
||||
: {}
|
||||
);
|
||||
}
|
||||
|
||||
function setStreamField(
|
||||
name: string,
|
||||
field: "primary_key" | "update_key",
|
||||
value: string
|
||||
) {
|
||||
setSelected((m) => ({
|
||||
...m,
|
||||
[name]: { ...m[name], name, [field]: value || undefined },
|
||||
}));
|
||||
}
|
||||
|
||||
async function submit() {
|
||||
if (!name.trim()) return toast.error("请填写流水线名称");
|
||||
if (!sourceId || !targetId) return toast.error("请选择源连接和目标连接");
|
||||
if (sourceId === targetId) return toast.error("源连接和目标连接不能相同");
|
||||
if (selectedList.length === 0) return toast.error("请至少选择一张表");
|
||||
|
||||
setSaving(true);
|
||||
const payload = {
|
||||
name: name.trim(),
|
||||
source_conn_id: Number(sourceId),
|
||||
target_conn_id: Number(targetId),
|
||||
schema_sync: schemaSync ? 1 : 0,
|
||||
schema_scope: schemaScope,
|
||||
include_fk: includeFk ? 1 : 0,
|
||||
mode,
|
||||
streams: JSON.stringify(selectedList),
|
||||
};
|
||||
try {
|
||||
const p = isEdit
|
||||
? await api<Pipeline>(`/api/pipelines/${initial.id}`, {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
: await api<Pipeline>("/api/pipelines", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
toast.success(isEdit ? "流水线已保存" : "流水线已创建");
|
||||
router.push(`/pipelines/${p.id}`);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : isEdit ? "保存失败" : "创建失败");
|
||||
setSaving(false);
|
||||
}
|
||||
}
|
||||
|
||||
const cancelHref = isEdit ? `/pipelines/${initial.id}` : "/pipelines";
|
||||
|
||||
return (
|
||||
<div className="flex max-w-3xl flex-col gap-6">
|
||||
<h1 className="text-xl font-semibold">
|
||||
{isEdit ? `编辑流水线:${initial.name}` : "新建流水线"}
|
||||
</h1>
|
||||
|
||||
{connError ? (
|
||||
<div className="text-sm text-red-500">连接列表加载失败:{connError}</div>
|
||||
) : null}
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>基本信息</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="grid gap-4">
|
||||
<div className="grid gap-1.5">
|
||||
<Label htmlFor="p-name">名称</Label>
|
||||
<Input
|
||||
id="p-name"
|
||||
value={name}
|
||||
onChange={(e) => setName(e.target.value)}
|
||||
placeholder="例如:测试库 → 本地 dev"
|
||||
/>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="grid gap-1.5">
|
||||
<Label>源连接</Label>
|
||||
<Select
|
||||
value={sourceId}
|
||||
onValueChange={(v) => {
|
||||
setSourceId(String(v));
|
||||
setStreamList(null);
|
||||
setSelected({});
|
||||
}}
|
||||
>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder="选择源连接" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{connections.map((c) => (
|
||||
<SelectItem key={c.id} value={String(c.id)}>
|
||||
{c.name}({c.database})
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div className="grid gap-1.5">
|
||||
<Label>目标连接</Label>
|
||||
<Select value={targetId} onValueChange={(v) => setTargetId(String(v))}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder="选择目标连接" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{connections.map((c) => (
|
||||
<SelectItem key={c.id} value={String(c.id)}>
|
||||
{c.name}({c.database})
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid gap-1.5">
|
||||
<Label>同步模式</Label>
|
||||
<Select value={mode} onValueChange={(v) => setMode(String(v))}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{MODES.map((m) => (
|
||||
<SelectItem key={m.value} value={m.value}>
|
||||
{m.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div className="flex items-center justify-between rounded-md border p-3">
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<Label>Schema 同步</Label>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
先用 Atlas 同步 schema(表/索引/约束),再迁移数据
|
||||
</span>
|
||||
</div>
|
||||
<Switch checked={schemaSync} onCheckedChange={(c) => setSchemaSync(!!c)} />
|
||||
</div>
|
||||
<div
|
||||
className={`flex items-center justify-between rounded-md border p-3 ${
|
||||
schemaSync ? "" : "opacity-50"
|
||||
}`}
|
||||
>
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<Label>Schema 同步范围</Label>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
整库模式会同步源库所有表,且目标库独有的表可能被删除
|
||||
</span>
|
||||
</div>
|
||||
<Select
|
||||
value={schemaScope}
|
||||
onValueChange={(v) => setSchemaScope(String(v))}
|
||||
disabled={!schemaSync}
|
||||
>
|
||||
<SelectTrigger className="w-36">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="selected">仅选中的表</SelectItem>
|
||||
<SelectItem value="all">整个库</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div
|
||||
className={`flex items-center justify-between rounded-md border p-3 ${
|
||||
schemaSync ? "" : "opacity-50"
|
||||
}`}
|
||||
>
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<Label>包含外键约束</Label>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
关闭可避免选中表引用了未迁移表导致的失败;排除父表时引用它的外键会自动跳过
|
||||
</span>
|
||||
</div>
|
||||
<Switch
|
||||
checked={includeFk}
|
||||
onCheckedChange={(c) => setIncludeFk(!!c)}
|
||||
disabled={!schemaSync}
|
||||
/>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>选择要同步的表</CardTitle>
|
||||
<CardDescription>
|
||||
选择源连接后点击「加载表」,勾选需要同步的表。
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="flex flex-col gap-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={loadStreams}
|
||||
disabled={!sourceId || streamsLoading}
|
||||
>
|
||||
{streamsLoading ? "加载中…" : "加载表"}
|
||||
</Button>
|
||||
{streamList && streamList.length > 0 ? (
|
||||
<>
|
||||
<Button variant="ghost" size="sm" onClick={() => selectAll(true)}>
|
||||
全选
|
||||
</Button>
|
||||
<Button variant="ghost" size="sm" onClick={() => selectAll(false)}>
|
||||
全不选
|
||||
</Button>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
已选 {selectedList.length} / {streamList.length}
|
||||
</span>
|
||||
</>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{streamList === null ? (
|
||||
selectedList.length > 0 ? (
|
||||
<div className="text-sm text-muted-foreground">
|
||||
当前已选 {selectedList.length} 张表(
|
||||
{selectedList.map((s) => s.name).join("、")}),点击「加载表」可修改。
|
||||
</div>
|
||||
) : null
|
||||
) : streamList.length === 0 ? (
|
||||
<div className="text-sm text-muted-foreground">没有可同步的表。</div>
|
||||
) : (
|
||||
<div className="max-h-96 overflow-auto rounded-md border">
|
||||
{streamList.map((s) => {
|
||||
const checked = !!selected[s.name];
|
||||
return (
|
||||
<div
|
||||
key={s.name}
|
||||
className="flex items-center gap-3 border-b px-3 py-2 last:border-b-0"
|
||||
>
|
||||
<Checkbox
|
||||
checked={checked}
|
||||
onCheckedChange={(c) => toggleStream(s, !!c)}
|
||||
/>
|
||||
<span className="flex-1 font-mono text-sm">{s.name}</span>
|
||||
{mode === "incremental" && checked ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
className="h-7 w-36 font-mono text-xs"
|
||||
placeholder="primary_key"
|
||||
value={selected[s.name].primary_key ?? ""}
|
||||
onChange={(e) =>
|
||||
setStreamField(s.name, "primary_key", e.target.value)
|
||||
}
|
||||
/>
|
||||
<Input
|
||||
className="h-7 w-36 font-mono text-xs"
|
||||
placeholder="update_key"
|
||||
value={selected[s.name].update_key ?? ""}
|
||||
onChange={(e) =>
|
||||
setStreamField(s.name, "update_key", e.target.value)
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<div className="flex gap-2">
|
||||
<Button onClick={submit} disabled={saving}>
|
||||
{saving
|
||||
? isEdit
|
||||
? "保存中…"
|
||||
: "创建中…"
|
||||
: isEdit
|
||||
? "保存修改"
|
||||
: "创建流水线"}
|
||||
</Button>
|
||||
<Button variant="outline" onClick={() => router.push(cancelHref)}>
|
||||
取消
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
102
lib/atlas.ts
102
lib/atlas.ts
|
|
@ -1,6 +1,6 @@
|
|||
import { execFile } from "child_process";
|
||||
import { Connection, getConnection } from "./db";
|
||||
import { connectionUrl, execSql } from "./sling";
|
||||
import { Connection, Pipeline, StreamConfig, getConnection } from "./db";
|
||||
import { connectionUrl, discoverStreams, execSql } from "./sling";
|
||||
import { resolveBinary } from "./binaries";
|
||||
|
||||
// Atlas needs an empty scratch database ("dev database") to plan diffs.
|
||||
|
|
@ -23,8 +23,13 @@ export function prepareDevDatabase(tgt: Connection): string {
|
|||
return devDatabaseUrl(tgt);
|
||||
}
|
||||
|
||||
function baseApplyArgs(srcUrl: string, tgtUrl: string, devUrl: string): string[] {
|
||||
return [
|
||||
function baseApplyArgs(
|
||||
srcUrl: string,
|
||||
tgtUrl: string,
|
||||
devUrl: string,
|
||||
excludes: string[]
|
||||
): string[] {
|
||||
const args = [
|
||||
"schema",
|
||||
"apply",
|
||||
"--url",
|
||||
|
|
@ -34,47 +39,112 @@ function baseApplyArgs(srcUrl: string, tgtUrl: string, devUrl: string): string[]
|
|||
"--dev-url",
|
||||
devUrl,
|
||||
];
|
||||
// Community Atlas has no --include; --exclude globs match bare resource
|
||||
// names (no schema prefix). Excluded objects are dropped from the diff
|
||||
// entirely, so e.g. unsupported column types in them never surface.
|
||||
for (const p of excludes) args.push("--exclude", p);
|
||||
return args;
|
||||
}
|
||||
|
||||
export function buildApplyArgs(
|
||||
srcUrl: string,
|
||||
tgtUrl: string,
|
||||
devUrl: string
|
||||
devUrl: string,
|
||||
excludes: string[] = []
|
||||
): string[] {
|
||||
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl), "--auto-approve"];
|
||||
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl, excludes), "--auto-approve"];
|
||||
}
|
||||
|
||||
// atlas rejects --dry-run combined with --auto-approve.
|
||||
export function buildDryRunArgs(
|
||||
srcUrl: string,
|
||||
tgtUrl: string,
|
||||
devUrl: string
|
||||
devUrl: string,
|
||||
excludes: string[] = []
|
||||
): string[] {
|
||||
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl), "--dry-run"];
|
||||
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl, excludes), "--dry-run"];
|
||||
}
|
||||
|
||||
// Compute the --exclude patterns for a pipeline's schema step. Atlas diffs
|
||||
// whole databases, and tables that exist only on the target would be planned
|
||||
// as DROPs — so for 'selected' scope we must exclude every non-selected table
|
||||
// from BOTH sides. Bare table names only: exclude globs match resource names
|
||||
// without schema prefixes.
|
||||
export async function computeSchemaExcludes(
|
||||
pipeline: Pipeline
|
||||
): Promise<string[]> {
|
||||
if ((pipeline.schema_scope || "selected") === "all") return [];
|
||||
|
||||
const streams = JSON.parse(pipeline.streams || "[]") as StreamConfig[];
|
||||
const selected = new Set(
|
||||
streams.map((s) => {
|
||||
const i = s.name.lastIndexOf(".");
|
||||
return i >= 0 ? s.name.slice(i + 1) : s.name;
|
||||
})
|
||||
);
|
||||
|
||||
const tableNames = (connId: number, side: string): Set<string> => {
|
||||
try {
|
||||
return new Set(
|
||||
discoverStreams(connId).map((s) => {
|
||||
const i = s.name.lastIndexOf(".");
|
||||
return i >= 0 ? s.name.slice(i + 1) : s.name;
|
||||
})
|
||||
);
|
||||
} catch (e) {
|
||||
// Silently falling back would risk DROPping target-only tables.
|
||||
throw new Error(
|
||||
`无法获取${side}库表列表,已中止 schema 同步:${(e as Error).message}`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const srcTables = tableNames(pipeline.source_conn_id, "源");
|
||||
const tgtTables = tableNames(pipeline.target_conn_id, "目标");
|
||||
|
||||
const excludes: string[] = [];
|
||||
for (const t of new Set([...srcTables, ...tgtTables])) {
|
||||
if (!selected.has(t)) excludes.push(t);
|
||||
}
|
||||
// FKs referencing excluded tables vanish from the desired state anyway;
|
||||
// excluding all FKs additionally avoids touching constraints on the
|
||||
// selected tables themselves.
|
||||
if (!pipeline.include_fk) excludes.push("*[type=foreign-key]");
|
||||
return excludes;
|
||||
}
|
||||
|
||||
function atlasBin(): string {
|
||||
return resolveBinary("atlas") ?? "atlas";
|
||||
}
|
||||
|
||||
export function schemaDiff(
|
||||
sourceConnId: number,
|
||||
targetConnId: number
|
||||
export async function schemaDiff(
|
||||
pipeline: Pipeline
|
||||
): Promise<{ ok: boolean; sql: string }> {
|
||||
const src = getConnection(sourceConnId);
|
||||
const tgt = getConnection(targetConnId);
|
||||
const src = getConnection(pipeline.source_conn_id);
|
||||
const tgt = getConnection(pipeline.target_conn_id);
|
||||
if (!src || !tgt) throw new Error("connection not found");
|
||||
let excludes: string[];
|
||||
try {
|
||||
excludes = await computeSchemaExcludes(pipeline);
|
||||
} catch (e) {
|
||||
return { ok: false, sql: (e as Error).message };
|
||||
}
|
||||
let devUrl: string;
|
||||
try {
|
||||
devUrl = prepareDevDatabase(tgt);
|
||||
} catch (e) {
|
||||
const err = e as { stdout?: string; stderr?: string; message?: string };
|
||||
return Promise.resolve({
|
||||
return {
|
||||
ok: false,
|
||||
sql: [err.stdout, err.stderr, err.message].filter(Boolean).join("\n"),
|
||||
});
|
||||
};
|
||||
}
|
||||
const args = buildDryRunArgs(connectionUrl(src), connectionUrl(tgt), devUrl);
|
||||
const args = buildDryRunArgs(
|
||||
connectionUrl(src),
|
||||
connectionUrl(tgt),
|
||||
devUrl,
|
||||
excludes
|
||||
);
|
||||
return new Promise((resolve) => {
|
||||
execFile(
|
||||
/* turbopackIgnore: true */ atlasBin(),
|
||||
|
|
|
|||
38
lib/db.ts
38
lib/db.ts
|
|
@ -41,6 +41,8 @@ export interface Pipeline {
|
|||
source_conn_id: number;
|
||||
target_conn_id: number;
|
||||
schema_sync: number; // 1 | 0
|
||||
schema_scope: string; // 'selected' | 'all'
|
||||
include_fk: number; // 1 | 0
|
||||
mode: string;
|
||||
streams: string; // JSON: StreamConfig[]
|
||||
created_at: string;
|
||||
|
|
@ -52,6 +54,8 @@ export interface PipelineInput {
|
|||
source_conn_id: number;
|
||||
target_conn_id: number;
|
||||
schema_sync?: number;
|
||||
schema_scope?: string;
|
||||
include_fk?: number;
|
||||
mode?: string;
|
||||
streams?: string;
|
||||
}
|
||||
|
|
@ -91,6 +95,8 @@ function createDb(): Database.Database {
|
|||
source_conn_id INTEGER NOT NULL REFERENCES connections(id),
|
||||
target_conn_id INTEGER NOT NULL REFERENCES connections(id),
|
||||
schema_sync INTEGER DEFAULT 1,
|
||||
schema_scope TEXT NOT NULL DEFAULT 'selected',
|
||||
include_fk INTEGER NOT NULL DEFAULT 0,
|
||||
mode TEXT DEFAULT 'full-refresh',
|
||||
streams TEXT NOT NULL DEFAULT '[]',
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
|
|
@ -107,6 +113,30 @@ function createDb(): Database.Database {
|
|||
pid INTEGER
|
||||
);
|
||||
`);
|
||||
// Lightweight migrations for databases created before these columns existed.
|
||||
// Several Next.js workers may open the DB concurrently during build, so a
|
||||
// racing ALTER can fail with "duplicate column name" — safe to ignore.
|
||||
const addColumnIfMissing = (sql: string, column: string) => {
|
||||
const cols = new Set(
|
||||
(db.pragma("table_info(pipelines)") as { name: string }[]).map(
|
||||
(c) => c.name
|
||||
)
|
||||
);
|
||||
if (cols.has(column)) return;
|
||||
try {
|
||||
db.exec(sql);
|
||||
} catch (e) {
|
||||
if (!(e as Error).message.includes("duplicate column name")) throw e;
|
||||
}
|
||||
};
|
||||
addColumnIfMissing(
|
||||
"ALTER TABLE pipelines ADD COLUMN schema_scope TEXT NOT NULL DEFAULT 'selected'",
|
||||
"schema_scope"
|
||||
);
|
||||
addColumnIfMissing(
|
||||
"ALTER TABLE pipelines ADD COLUMN include_fk INTEGER NOT NULL DEFAULT 0",
|
||||
"include_fk"
|
||||
);
|
||||
return db;
|
||||
}
|
||||
|
||||
|
|
@ -186,14 +216,16 @@ export function getPipeline(id: number): Pipeline | null {
|
|||
export function createPipeline(input: PipelineInput): Pipeline {
|
||||
const res = db
|
||||
.prepare(
|
||||
`INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, mode, streams)
|
||||
VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @mode, @streams)`
|
||||
`INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, schema_scope, include_fk, mode, streams)
|
||||
VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @schema_scope, @include_fk, @mode, @streams)`
|
||||
)
|
||||
.run({
|
||||
name: input.name,
|
||||
source_conn_id: input.source_conn_id,
|
||||
target_conn_id: input.target_conn_id,
|
||||
schema_sync: input.schema_sync ?? 1,
|
||||
schema_scope: input.schema_scope ?? "selected",
|
||||
include_fk: input.include_fk ?? 0,
|
||||
mode: input.mode ?? "full-refresh",
|
||||
streams: input.streams ?? "[]",
|
||||
});
|
||||
|
|
@ -209,6 +241,8 @@ export function updatePipeline(
|
|||
"source_conn_id",
|
||||
"target_conn_id",
|
||||
"schema_sync",
|
||||
"schema_scope",
|
||||
"include_fk",
|
||||
"mode",
|
||||
"streams",
|
||||
].filter((k) => input[k as keyof PipelineInput] !== undefined);
|
||||
|
|
|
|||
13
lib/jobs.ts
13
lib/jobs.ts
|
|
@ -12,7 +12,7 @@ import {
|
|||
updateRunStatus,
|
||||
} from "./db";
|
||||
import { SLING_HOME, buildReplicationYaml, connectionUrl } from "./sling";
|
||||
import { buildApplyArgs, prepareDevDatabase } from "./atlas";
|
||||
import { buildApplyArgs, computeSchemaExcludes, prepareDevDatabase } from "./atlas";
|
||||
import { resolveBinary } from "./binaries";
|
||||
|
||||
export interface JobEvent {
|
||||
|
|
@ -137,11 +137,20 @@ class JobManager {
|
|||
const src = getConnection(pipeline.source_conn_id);
|
||||
const tgt = getConnection(pipeline.target_conn_id);
|
||||
if (!src || !tgt) throw new Error("pipeline connections not found");
|
||||
const excludes = await computeSchemaExcludes(pipeline);
|
||||
const scope = pipeline.schema_scope || "selected";
|
||||
const tableExcludes = pipeline.include_fk
|
||||
? excludes.length
|
||||
: excludes.filter((p) => p !== "*[type=foreign-key]").length;
|
||||
this.emit(
|
||||
state,
|
||||
`[schema] scope=${scope}, excluding ${tableExcludes} objects, fk=${pipeline.include_fk ? "on" : "off"}`
|
||||
);
|
||||
const devUrl = prepareDevDatabase(tgt);
|
||||
const code = await this.runProcess(
|
||||
state,
|
||||
resolveBinary("atlas") ?? "atlas",
|
||||
buildApplyArgs(connectionUrl(src), connectionUrl(tgt), devUrl),
|
||||
buildApplyArgs(connectionUrl(src), connectionUrl(tgt), devUrl, excludes),
|
||||
{}
|
||||
);
|
||||
if (code !== 0) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue