feat: table-scoped schema sync, optional FK migration, pipeline edit UI
- pipelines: new schema_scope ('selected'|'all') and include_fk fields with
lightweight ALTER TABLE migration for existing DBs
- atlas: computeSchemaExcludes() limits diff/apply to selected tables via
repeated --exclude (bare-name globs); avoids unsupported-type failures on
unrelated tables (e.g. TiDB VECTOR) and prevents DROP of target-only tables
- fk toggle off appends --exclude '*[type=foreign-key]'; FKs referencing
excluded parent tables are skipped by Atlas automatically
- pipeline form extracted to shared component; new /pipelines/[id]/edit page
with full prefill; edit entries in list and detail pages
This commit is contained in:
parent
fcb0f2e299
commit
884eaa1ed8
|
|
@ -14,10 +14,7 @@ export async function GET(
|
||||||
if (!pipeline) {
|
if (!pipeline) {
|
||||||
return Response.json({ error: "pipeline not found" }, { status: 404 });
|
return Response.json({ error: "pipeline not found" }, { status: 404 });
|
||||||
}
|
}
|
||||||
const result = await schemaDiff(
|
const result = await schemaDiff(pipeline);
|
||||||
pipeline.source_conn_id,
|
|
||||||
pipeline.target_conn_id
|
|
||||||
);
|
|
||||||
return Response.json(result);
|
return Response.json(result);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return Response.json({ error: (err as Error).message }, { status: 500 });
|
return Response.json({ error: (err as Error).message }, { status: 500 });
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,8 @@ export async function PUT(request: Request, { params }: Ctx) {
|
||||||
"source_conn_id",
|
"source_conn_id",
|
||||||
"target_conn_id",
|
"target_conn_id",
|
||||||
"schema_sync",
|
"schema_sync",
|
||||||
|
"schema_scope",
|
||||||
|
"include_fk",
|
||||||
"mode",
|
"mode",
|
||||||
]) {
|
]) {
|
||||||
if (body[key] !== undefined) patch[key] = body[key];
|
if (body[key] !== undefined) patch[key] = body[key];
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,8 @@ export async function POST(request: Request) {
|
||||||
source_conn_id: Number(body.source_conn_id),
|
source_conn_id: Number(body.source_conn_id),
|
||||||
target_conn_id: Number(body.target_conn_id),
|
target_conn_id: Number(body.target_conn_id),
|
||||||
schema_sync: body.schema_sync === undefined ? 1 : Number(body.schema_sync),
|
schema_sync: body.schema_sync === undefined ? 1 : Number(body.schema_sync),
|
||||||
|
schema_scope: body.schema_scope ? String(body.schema_scope) : "selected",
|
||||||
|
include_fk: body.include_fk === undefined ? 0 : Number(body.include_fk),
|
||||||
mode: body.mode ? String(body.mode) : "full-refresh",
|
mode: body.mode ? String(body.mode) : "full-refresh",
|
||||||
streams: normalizeStreams(body.streams),
|
streams: normalizeStreams(body.streams),
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import { useParams } from "next/navigation";
|
||||||
|
import { api, type Pipeline } from "@/components/api";
|
||||||
|
import { PipelineForm } from "@/components/pipeline-form";
|
||||||
|
|
||||||
|
export default function EditPipelinePage() {
|
||||||
|
const params = useParams<{ id: string }>();
|
||||||
|
const id = Number(params.id);
|
||||||
|
|
||||||
|
const [pipeline, setPipeline] = useState<Pipeline | null>(null);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
api<Pipeline>(`/api/pipelines/${id}`)
|
||||||
|
.then(setPipeline)
|
||||||
|
.catch((e) => setError(e instanceof Error ? e.message : "加载失败"))
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
}, [id]);
|
||||||
|
|
||||||
|
if (loading) return <div className="text-sm text-muted-foreground">加载中…</div>;
|
||||||
|
if (error || !pipeline)
|
||||||
|
return (
|
||||||
|
<div className="text-sm text-red-500">
|
||||||
|
{error ? `加载失败:${error}` : "流水线不存在"}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
|
||||||
|
return <PipelineForm initial={pipeline} />;
|
||||||
|
}
|
||||||
|
|
@ -4,7 +4,7 @@ import { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
import Link from "next/link";
|
import Link from "next/link";
|
||||||
import { useParams } from "next/navigation";
|
import { useParams } from "next/navigation";
|
||||||
import { toast } from "sonner";
|
import { toast } from "sonner";
|
||||||
import { Play, GitCompareArrows } from "lucide-react";
|
import { Play, GitCompareArrows, Pencil } from "lucide-react";
|
||||||
import {
|
import {
|
||||||
api,
|
api,
|
||||||
ApiError,
|
ApiError,
|
||||||
|
|
@ -141,6 +141,12 @@ export default function PipelineDetailPage() {
|
||||||
<div className="flex items-center justify-between">
|
<div className="flex items-center justify-between">
|
||||||
<h1 className="text-xl font-semibold">{pipeline.name}</h1>
|
<h1 className="text-xl font-semibold">{pipeline.name}</h1>
|
||||||
<div className="flex gap-2">
|
<div className="flex gap-2">
|
||||||
|
<Button variant="outline">
|
||||||
|
<Link href={`/pipelines/${id}/edit`} className="flex items-center gap-1">
|
||||||
|
<Pencil className="size-4" />
|
||||||
|
编辑
|
||||||
|
</Link>
|
||||||
|
</Button>
|
||||||
<Button variant="outline" onClick={previewDiff} disabled={diffLoading}>
|
<Button variant="outline" onClick={previewDiff} disabled={diffLoading}>
|
||||||
<GitCompareArrows className="size-4" />
|
<GitCompareArrows className="size-4" />
|
||||||
{diffLoading ? "Diff 计算中…" : "Schema Diff 预览"}
|
{diffLoading ? "Diff 计算中…" : "Schema Diff 预览"}
|
||||||
|
|
@ -187,6 +193,16 @@ export default function PipelineDetailPage() {
|
||||||
)}
|
)}
|
||||||
</dd>
|
</dd>
|
||||||
</div>
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt className="text-muted-foreground">同步范围</dt>
|
||||||
|
<dd className="font-medium">
|
||||||
|
{pipeline.schema_scope === "all" ? "整个库" : "仅选中的表"}
|
||||||
|
</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt className="text-muted-foreground">包含外键</dt>
|
||||||
|
<dd className="font-medium">{pipeline.include_fk ? "是" : "否"}</dd>
|
||||||
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<dt className="text-muted-foreground">创建时间</dt>
|
<dt className="text-muted-foreground">创建时间</dt>
|
||||||
<dd>{fmtTime(pipeline.created_at)}</dd>
|
<dd>{fmtTime(pipeline.created_at)}</dd>
|
||||||
|
|
|
||||||
|
|
@ -1,314 +1,7 @@
|
||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { useEffect, useMemo, useState } from "react";
|
import { PipelineForm } from "@/components/pipeline-form";
|
||||||
import { useRouter } from "next/navigation";
|
|
||||||
import { toast } from "sonner";
|
|
||||||
import {
|
|
||||||
api,
|
|
||||||
type Connection,
|
|
||||||
type Pipeline,
|
|
||||||
type StreamConfig,
|
|
||||||
type StreamInfo,
|
|
||||||
} from "@/components/api";
|
|
||||||
import { Button } from "@/components/ui/button";
|
|
||||||
import {
|
|
||||||
Card,
|
|
||||||
CardContent,
|
|
||||||
CardDescription,
|
|
||||||
CardHeader,
|
|
||||||
CardTitle,
|
|
||||||
} from "@/components/ui/card";
|
|
||||||
import { Checkbox } from "@/components/ui/checkbox";
|
|
||||||
import { Input } from "@/components/ui/input";
|
|
||||||
import { Label } from "@/components/ui/label";
|
|
||||||
import {
|
|
||||||
Select,
|
|
||||||
SelectContent,
|
|
||||||
SelectItem,
|
|
||||||
SelectTrigger,
|
|
||||||
SelectValue,
|
|
||||||
} from "@/components/ui/select";
|
|
||||||
import { Switch } from "@/components/ui/switch";
|
|
||||||
|
|
||||||
const MODES = [
|
|
||||||
{ value: "full-refresh", label: "full-refresh(全量覆盖,默认)" },
|
|
||||||
{ value: "truncate", label: "truncate(清空后写入)" },
|
|
||||||
{ value: "incremental", label: "incremental(增量)" },
|
|
||||||
{ value: "snapshot", label: "snapshot(快照)" },
|
|
||||||
];
|
|
||||||
|
|
||||||
export default function NewPipelinePage() {
|
export default function NewPipelinePage() {
|
||||||
const router = useRouter();
|
return <PipelineForm />;
|
||||||
const [connections, setConnections] = useState<Connection[]>([]);
|
|
||||||
const [connError, setConnError] = useState<string | null>(null);
|
|
||||||
|
|
||||||
const [name, setName] = useState("");
|
|
||||||
const [sourceId, setSourceId] = useState("");
|
|
||||||
const [targetId, setTargetId] = useState("");
|
|
||||||
const [mode, setMode] = useState("full-refresh");
|
|
||||||
const [schemaSync, setSchemaSync] = useState(true);
|
|
||||||
|
|
||||||
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
|
|
||||||
const [streamsLoading, setStreamsLoading] = useState(false);
|
|
||||||
const [selected, setSelected] = useState<Record<string, StreamConfig>>({});
|
|
||||||
const [saving, setSaving] = useState(false);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
api<Connection[]>("/api/connections")
|
|
||||||
.then(setConnections)
|
|
||||||
.catch((e) => setConnError(e instanceof Error ? e.message : "加载失败"));
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// 切换源连接时同时清空已加载的表(在 onValueChange 里处理)
|
|
||||||
|
|
||||||
const selectedList = useMemo(() => Object.values(selected), [selected]);
|
|
||||||
|
|
||||||
async function loadStreams() {
|
|
||||||
if (!sourceId) return;
|
|
||||||
setStreamsLoading(true);
|
|
||||||
try {
|
|
||||||
const list = await api<StreamInfo[]>(
|
|
||||||
`/api/connections/${sourceId}/streams`
|
|
||||||
);
|
|
||||||
setStreamList(list);
|
|
||||||
if (list.length === 0) toast.info("该连接没有可同步的表");
|
|
||||||
} catch (e) {
|
|
||||||
toast.error(e instanceof Error ? e.message : "加载表失败");
|
|
||||||
} finally {
|
|
||||||
setStreamsLoading(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function toggleStream(s: StreamInfo, checked: boolean) {
|
|
||||||
setSelected((m) => {
|
|
||||||
const next = { ...m };
|
|
||||||
if (checked) next[s.name] = { name: s.name };
|
|
||||||
else delete next[s.name];
|
|
||||||
return next;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function selectAll(on: boolean) {
|
|
||||||
if (!streamList) return;
|
|
||||||
setSelected(
|
|
||||||
on
|
|
||||||
? Object.fromEntries(
|
|
||||||
streamList.map((s) => [s.name, selected[s.name] ?? { name: s.name }])
|
|
||||||
)
|
|
||||||
: {}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function setStreamField(
|
|
||||||
name: string,
|
|
||||||
field: "primary_key" | "update_key",
|
|
||||||
value: string
|
|
||||||
) {
|
|
||||||
setSelected((m) => ({
|
|
||||||
...m,
|
|
||||||
[name]: { ...m[name], name, [field]: value || undefined },
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
async function submit() {
|
|
||||||
if (!name.trim()) return toast.error("请填写流水线名称");
|
|
||||||
if (!sourceId || !targetId) return toast.error("请选择源连接和目标连接");
|
|
||||||
if (sourceId === targetId) return toast.error("源连接和目标连接不能相同");
|
|
||||||
if (selectedList.length === 0) return toast.error("请至少选择一张表");
|
|
||||||
|
|
||||||
setSaving(true);
|
|
||||||
try {
|
|
||||||
const p = await api<Pipeline>("/api/pipelines", {
|
|
||||||
method: "POST",
|
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
body: JSON.stringify({
|
|
||||||
name: name.trim(),
|
|
||||||
source_conn_id: Number(sourceId),
|
|
||||||
target_conn_id: Number(targetId),
|
|
||||||
schema_sync: schemaSync ? 1 : 0,
|
|
||||||
mode,
|
|
||||||
streams: JSON.stringify(selectedList),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
toast.success("流水线已创建");
|
|
||||||
router.push(`/pipelines/${p.id}`);
|
|
||||||
} catch (e) {
|
|
||||||
toast.error(e instanceof Error ? e.message : "创建失败");
|
|
||||||
setSaving(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="flex max-w-3xl flex-col gap-6">
|
|
||||||
<h1 className="text-xl font-semibold">新建流水线</h1>
|
|
||||||
|
|
||||||
{connError ? (
|
|
||||||
<div className="text-sm text-red-500">连接列表加载失败:{connError}</div>
|
|
||||||
) : null}
|
|
||||||
|
|
||||||
<Card>
|
|
||||||
<CardHeader>
|
|
||||||
<CardTitle>基本信息</CardTitle>
|
|
||||||
</CardHeader>
|
|
||||||
<CardContent className="grid gap-4">
|
|
||||||
<div className="grid gap-1.5">
|
|
||||||
<Label htmlFor="p-name">名称</Label>
|
|
||||||
<Input
|
|
||||||
id="p-name"
|
|
||||||
value={name}
|
|
||||||
onChange={(e) => setName(e.target.value)}
|
|
||||||
placeholder="例如:测试库 → 本地 dev"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className="grid grid-cols-2 gap-4">
|
|
||||||
<div className="grid gap-1.5">
|
|
||||||
<Label>源连接</Label>
|
|
||||||
<Select
|
|
||||||
value={sourceId}
|
|
||||||
onValueChange={(v) => {
|
|
||||||
setSourceId(String(v));
|
|
||||||
setStreamList(null);
|
|
||||||
setSelected({});
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<SelectTrigger className="w-full">
|
|
||||||
<SelectValue placeholder="选择源连接" />
|
|
||||||
</SelectTrigger>
|
|
||||||
<SelectContent>
|
|
||||||
{connections.map((c) => (
|
|
||||||
<SelectItem key={c.id} value={String(c.id)}>
|
|
||||||
{c.name}({c.database})
|
|
||||||
</SelectItem>
|
|
||||||
))}
|
|
||||||
</SelectContent>
|
|
||||||
</Select>
|
|
||||||
</div>
|
|
||||||
<div className="grid gap-1.5">
|
|
||||||
<Label>目标连接</Label>
|
|
||||||
<Select value={targetId} onValueChange={(v) => setTargetId(String(v))}>
|
|
||||||
<SelectTrigger className="w-full">
|
|
||||||
<SelectValue placeholder="选择目标连接" />
|
|
||||||
</SelectTrigger>
|
|
||||||
<SelectContent>
|
|
||||||
{connections.map((c) => (
|
|
||||||
<SelectItem key={c.id} value={String(c.id)}>
|
|
||||||
{c.name}({c.database})
|
|
||||||
</SelectItem>
|
|
||||||
))}
|
|
||||||
</SelectContent>
|
|
||||||
</Select>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div className="grid gap-1.5">
|
|
||||||
<Label>同步模式</Label>
|
|
||||||
<Select value={mode} onValueChange={(v) => setMode(String(v))}>
|
|
||||||
<SelectTrigger className="w-full">
|
|
||||||
<SelectValue />
|
|
||||||
</SelectTrigger>
|
|
||||||
<SelectContent>
|
|
||||||
{MODES.map((m) => (
|
|
||||||
<SelectItem key={m.value} value={m.value}>
|
|
||||||
{m.label}
|
|
||||||
</SelectItem>
|
|
||||||
))}
|
|
||||||
</SelectContent>
|
|
||||||
</Select>
|
|
||||||
</div>
|
|
||||||
<div className="flex items-center justify-between rounded-md border p-3">
|
|
||||||
<div className="flex flex-col gap-0.5">
|
|
||||||
<Label>Schema 同步</Label>
|
|
||||||
<span className="text-xs text-muted-foreground">
|
|
||||||
先用 Atlas 同步 schema(表/索引/约束),再迁移数据
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<Switch checked={schemaSync} onCheckedChange={(c) => setSchemaSync(!!c)} />
|
|
||||||
</div>
|
|
||||||
</CardContent>
|
|
||||||
</Card>
|
|
||||||
|
|
||||||
<Card>
|
|
||||||
<CardHeader>
|
|
||||||
<CardTitle>选择要同步的表</CardTitle>
|
|
||||||
<CardDescription>
|
|
||||||
选择源连接后点击「加载表」,勾选需要同步的表。
|
|
||||||
</CardDescription>
|
|
||||||
</CardHeader>
|
|
||||||
<CardContent className="flex flex-col gap-3">
|
|
||||||
<div className="flex items-center gap-2">
|
|
||||||
<Button
|
|
||||||
variant="outline"
|
|
||||||
onClick={loadStreams}
|
|
||||||
disabled={!sourceId || streamsLoading}
|
|
||||||
>
|
|
||||||
{streamsLoading ? "加载中…" : "加载表"}
|
|
||||||
</Button>
|
|
||||||
{streamList && streamList.length > 0 ? (
|
|
||||||
<>
|
|
||||||
<Button variant="ghost" size="sm" onClick={() => selectAll(true)}>
|
|
||||||
全选
|
|
||||||
</Button>
|
|
||||||
<Button variant="ghost" size="sm" onClick={() => selectAll(false)}>
|
|
||||||
全不选
|
|
||||||
</Button>
|
|
||||||
<span className="text-xs text-muted-foreground">
|
|
||||||
已选 {selectedList.length} / {streamList.length}
|
|
||||||
</span>
|
|
||||||
</>
|
|
||||||
) : null}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{streamList === null ? null : streamList.length === 0 ? (
|
|
||||||
<div className="text-sm text-muted-foreground">没有可同步的表。</div>
|
|
||||||
) : (
|
|
||||||
<div className="max-h-96 overflow-auto rounded-md border">
|
|
||||||
{streamList.map((s) => {
|
|
||||||
const checked = !!selected[s.name];
|
|
||||||
return (
|
|
||||||
<div
|
|
||||||
key={s.name}
|
|
||||||
className="flex items-center gap-3 border-b px-3 py-2 last:border-b-0"
|
|
||||||
>
|
|
||||||
<Checkbox
|
|
||||||
checked={checked}
|
|
||||||
onCheckedChange={(c) => toggleStream(s, !!c)}
|
|
||||||
/>
|
|
||||||
<span className="flex-1 font-mono text-sm">{s.name}</span>
|
|
||||||
{mode === "incremental" && checked ? (
|
|
||||||
<div className="flex items-center gap-2">
|
|
||||||
<Input
|
|
||||||
className="h-7 w-36 font-mono text-xs"
|
|
||||||
placeholder="primary_key"
|
|
||||||
value={selected[s.name].primary_key ?? ""}
|
|
||||||
onChange={(e) =>
|
|
||||||
setStreamField(s.name, "primary_key", e.target.value)
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
<Input
|
|
||||||
className="h-7 w-36 font-mono text-xs"
|
|
||||||
placeholder="update_key"
|
|
||||||
value={selected[s.name].update_key ?? ""}
|
|
||||||
onChange={(e) =>
|
|
||||||
setStreamField(s.name, "update_key", e.target.value)
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
) : null}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
})}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</CardContent>
|
|
||||||
</Card>
|
|
||||||
|
|
||||||
<div className="flex gap-2">
|
|
||||||
<Button onClick={submit} disabled={saving}>
|
|
||||||
{saving ? "创建中…" : "创建流水线"}
|
|
||||||
</Button>
|
|
||||||
<Button variant="outline" onClick={() => router.push("/pipelines")}>
|
|
||||||
取消
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -122,6 +122,9 @@ export default function PipelinesPage() {
|
||||||
<Button variant="outline" size="sm">
|
<Button variant="outline" size="sm">
|
||||||
<Link href={`/pipelines/${p.id}`}>详情</Link>
|
<Link href={`/pipelines/${p.id}`}>详情</Link>
|
||||||
</Button>
|
</Button>
|
||||||
|
<Button variant="outline" size="sm">
|
||||||
|
<Link href={`/pipelines/${p.id}/edit`}>编辑</Link>
|
||||||
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
variant="outline"
|
variant="outline"
|
||||||
size="sm"
|
size="sm"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,405 @@
|
||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useMemo, useState } from "react";
|
||||||
|
import { useRouter } from "next/navigation";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import {
|
||||||
|
api,
|
||||||
|
type Connection,
|
||||||
|
type Pipeline,
|
||||||
|
type StreamConfig,
|
||||||
|
type StreamInfo,
|
||||||
|
} from "@/components/api";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import {
|
||||||
|
Card,
|
||||||
|
CardContent,
|
||||||
|
CardDescription,
|
||||||
|
CardHeader,
|
||||||
|
CardTitle,
|
||||||
|
} from "@/components/ui/card";
|
||||||
|
import { Checkbox } from "@/components/ui/checkbox";
|
||||||
|
import { Input } from "@/components/ui/input";
|
||||||
|
import { Label } from "@/components/ui/label";
|
||||||
|
import {
|
||||||
|
Select,
|
||||||
|
SelectContent,
|
||||||
|
SelectItem,
|
||||||
|
SelectTrigger,
|
||||||
|
SelectValue,
|
||||||
|
} from "@/components/ui/select";
|
||||||
|
import { Switch } from "@/components/ui/switch";
|
||||||
|
|
||||||
|
const MODES = [
|
||||||
|
{ value: "full-refresh", label: "full-refresh(全量覆盖,默认)" },
|
||||||
|
{ value: "truncate", label: "truncate(清空后写入)" },
|
||||||
|
{ value: "incremental", label: "incremental(增量)" },
|
||||||
|
{ value: "snapshot", label: "snapshot(快照)" },
|
||||||
|
];
|
||||||
|
|
||||||
|
export function PipelineForm({ initial }: { initial?: Pipeline }) {
|
||||||
|
const router = useRouter();
|
||||||
|
const isEdit = !!initial;
|
||||||
|
const [connections, setConnections] = useState<Connection[]>([]);
|
||||||
|
const [connError, setConnError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const [name, setName] = useState(initial?.name ?? "");
|
||||||
|
const [sourceId, setSourceId] = useState(
|
||||||
|
initial ? String(initial.source_conn_id) : ""
|
||||||
|
);
|
||||||
|
const [targetId, setTargetId] = useState(
|
||||||
|
initial ? String(initial.target_conn_id) : ""
|
||||||
|
);
|
||||||
|
const [mode, setMode] = useState(initial?.mode ?? "full-refresh");
|
||||||
|
const [schemaSync, setSchemaSync] = useState(
|
||||||
|
initial ? !!initial.schema_sync : true
|
||||||
|
);
|
||||||
|
const [schemaScope, setSchemaScope] = useState(
|
||||||
|
initial?.schema_scope ?? "selected"
|
||||||
|
);
|
||||||
|
const [includeFk, setIncludeFk] = useState(initial ? !!initial.include_fk : false);
|
||||||
|
|
||||||
|
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
|
||||||
|
const [streamsLoading, setStreamsLoading] = useState(false);
|
||||||
|
const [selected, setSelected] = useState<Record<string, StreamConfig>>(() => {
|
||||||
|
if (!initial) return {};
|
||||||
|
try {
|
||||||
|
const streams = JSON.parse(initial.streams || "[]") as StreamConfig[];
|
||||||
|
return Object.fromEntries(streams.map((s) => [s.name, s]));
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
api<Connection[]>("/api/connections")
|
||||||
|
.then(setConnections)
|
||||||
|
.catch((e) => setConnError(e instanceof Error ? e.message : "加载失败"));
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// 编辑模式:加载现有流水线后自动拉取表列表,回填勾选状态
|
||||||
|
useEffect(() => {
|
||||||
|
if (isEdit && sourceId) void loadStreams();
|
||||||
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
|
}, [isEdit, sourceId]);
|
||||||
|
|
||||||
|
const selectedList = useMemo(() => Object.values(selected), [selected]);
|
||||||
|
|
||||||
|
async function loadStreams() {
|
||||||
|
if (!sourceId) return;
|
||||||
|
setStreamsLoading(true);
|
||||||
|
try {
|
||||||
|
const list = await api<StreamInfo[]>(
|
||||||
|
`/api/connections/${sourceId}/streams`
|
||||||
|
);
|
||||||
|
setStreamList(list);
|
||||||
|
if (list.length === 0) toast.info("该连接没有可同步的表");
|
||||||
|
} catch (e) {
|
||||||
|
toast.error(e instanceof Error ? e.message : "加载表失败");
|
||||||
|
} finally {
|
||||||
|
setStreamsLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function toggleStream(s: StreamInfo, checked: boolean) {
|
||||||
|
setSelected((m) => {
|
||||||
|
const next = { ...m };
|
||||||
|
if (checked) next[s.name] = m[s.name] ?? { name: s.name };
|
||||||
|
else delete next[s.name];
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectAll(on: boolean) {
|
||||||
|
if (!streamList) return;
|
||||||
|
setSelected(
|
||||||
|
on
|
||||||
|
? Object.fromEntries(
|
||||||
|
streamList.map((s) => [s.name, selected[s.name] ?? { name: s.name }])
|
||||||
|
)
|
||||||
|
: {}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function setStreamField(
|
||||||
|
name: string,
|
||||||
|
field: "primary_key" | "update_key",
|
||||||
|
value: string
|
||||||
|
) {
|
||||||
|
setSelected((m) => ({
|
||||||
|
...m,
|
||||||
|
[name]: { ...m[name], name, [field]: value || undefined },
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function submit() {
|
||||||
|
if (!name.trim()) return toast.error("请填写流水线名称");
|
||||||
|
if (!sourceId || !targetId) return toast.error("请选择源连接和目标连接");
|
||||||
|
if (sourceId === targetId) return toast.error("源连接和目标连接不能相同");
|
||||||
|
if (selectedList.length === 0) return toast.error("请至少选择一张表");
|
||||||
|
|
||||||
|
setSaving(true);
|
||||||
|
const payload = {
|
||||||
|
name: name.trim(),
|
||||||
|
source_conn_id: Number(sourceId),
|
||||||
|
target_conn_id: Number(targetId),
|
||||||
|
schema_sync: schemaSync ? 1 : 0,
|
||||||
|
schema_scope: schemaScope,
|
||||||
|
include_fk: includeFk ? 1 : 0,
|
||||||
|
mode,
|
||||||
|
streams: JSON.stringify(selectedList),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const p = isEdit
|
||||||
|
? await api<Pipeline>(`/api/pipelines/${initial.id}`, {
|
||||||
|
method: "PUT",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
})
|
||||||
|
: await api<Pipeline>("/api/pipelines", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
toast.success(isEdit ? "流水线已保存" : "流水线已创建");
|
||||||
|
router.push(`/pipelines/${p.id}`);
|
||||||
|
} catch (e) {
|
||||||
|
toast.error(e instanceof Error ? e.message : isEdit ? "保存失败" : "创建失败");
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const cancelHref = isEdit ? `/pipelines/${initial.id}` : "/pipelines";
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex max-w-3xl flex-col gap-6">
|
||||||
|
<h1 className="text-xl font-semibold">
|
||||||
|
{isEdit ? `编辑流水线:${initial.name}` : "新建流水线"}
|
||||||
|
</h1>
|
||||||
|
|
||||||
|
{connError ? (
|
||||||
|
<div className="text-sm text-red-500">连接列表加载失败:{connError}</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>基本信息</CardTitle>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="grid gap-4">
|
||||||
|
<div className="grid gap-1.5">
|
||||||
|
<Label htmlFor="p-name">名称</Label>
|
||||||
|
<Input
|
||||||
|
id="p-name"
|
||||||
|
value={name}
|
||||||
|
onChange={(e) => setName(e.target.value)}
|
||||||
|
placeholder="例如:测试库 → 本地 dev"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="grid grid-cols-2 gap-4">
|
||||||
|
<div className="grid gap-1.5">
|
||||||
|
<Label>源连接</Label>
|
||||||
|
<Select
|
||||||
|
value={sourceId}
|
||||||
|
onValueChange={(v) => {
|
||||||
|
setSourceId(String(v));
|
||||||
|
setStreamList(null);
|
||||||
|
setSelected({});
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<SelectTrigger className="w-full">
|
||||||
|
<SelectValue placeholder="选择源连接" />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
{connections.map((c) => (
|
||||||
|
<SelectItem key={c.id} value={String(c.id)}>
|
||||||
|
{c.name}({c.database})
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
<div className="grid gap-1.5">
|
||||||
|
<Label>目标连接</Label>
|
||||||
|
<Select value={targetId} onValueChange={(v) => setTargetId(String(v))}>
|
||||||
|
<SelectTrigger className="w-full">
|
||||||
|
<SelectValue placeholder="选择目标连接" />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
{connections.map((c) => (
|
||||||
|
<SelectItem key={c.id} value={String(c.id)}>
|
||||||
|
{c.name}({c.database})
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="grid gap-1.5">
|
||||||
|
<Label>同步模式</Label>
|
||||||
|
<Select value={mode} onValueChange={(v) => setMode(String(v))}>
|
||||||
|
<SelectTrigger className="w-full">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
{MODES.map((m) => (
|
||||||
|
<SelectItem key={m.value} value={m.value}>
|
||||||
|
{m.label}
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center justify-between rounded-md border p-3">
|
||||||
|
<div className="flex flex-col gap-0.5">
|
||||||
|
<Label>Schema 同步</Label>
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
先用 Atlas 同步 schema(表/索引/约束),再迁移数据
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<Switch checked={schemaSync} onCheckedChange={(c) => setSchemaSync(!!c)} />
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
className={`flex items-center justify-between rounded-md border p-3 ${
|
||||||
|
schemaSync ? "" : "opacity-50"
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<div className="flex flex-col gap-0.5">
|
||||||
|
<Label>Schema 同步范围</Label>
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
整库模式会同步源库所有表,且目标库独有的表可能被删除
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<Select
|
||||||
|
value={schemaScope}
|
||||||
|
onValueChange={(v) => setSchemaScope(String(v))}
|
||||||
|
disabled={!schemaSync}
|
||||||
|
>
|
||||||
|
<SelectTrigger className="w-36">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="selected">仅选中的表</SelectItem>
|
||||||
|
<SelectItem value="all">整个库</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
className={`flex items-center justify-between rounded-md border p-3 ${
|
||||||
|
schemaSync ? "" : "opacity-50"
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<div className="flex flex-col gap-0.5">
|
||||||
|
<Label>包含外键约束</Label>
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
关闭可避免选中表引用了未迁移表导致的失败;排除父表时引用它的外键会自动跳过
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<Switch
|
||||||
|
checked={includeFk}
|
||||||
|
onCheckedChange={(c) => setIncludeFk(!!c)}
|
||||||
|
disabled={!schemaSync}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card>
|
||||||
|
<CardHeader>
|
||||||
|
<CardTitle>选择要同步的表</CardTitle>
|
||||||
|
<CardDescription>
|
||||||
|
选择源连接后点击「加载表」,勾选需要同步的表。
|
||||||
|
</CardDescription>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className="flex flex-col gap-3">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
onClick={loadStreams}
|
||||||
|
disabled={!sourceId || streamsLoading}
|
||||||
|
>
|
||||||
|
{streamsLoading ? "加载中…" : "加载表"}
|
||||||
|
</Button>
|
||||||
|
{streamList && streamList.length > 0 ? (
|
||||||
|
<>
|
||||||
|
<Button variant="ghost" size="sm" onClick={() => selectAll(true)}>
|
||||||
|
全选
|
||||||
|
</Button>
|
||||||
|
<Button variant="ghost" size="sm" onClick={() => selectAll(false)}>
|
||||||
|
全不选
|
||||||
|
</Button>
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
已选 {selectedList.length} / {streamList.length}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{streamList === null ? (
|
||||||
|
selectedList.length > 0 ? (
|
||||||
|
<div className="text-sm text-muted-foreground">
|
||||||
|
当前已选 {selectedList.length} 张表(
|
||||||
|
{selectedList.map((s) => s.name).join("、")}),点击「加载表」可修改。
|
||||||
|
</div>
|
||||||
|
) : null
|
||||||
|
) : streamList.length === 0 ? (
|
||||||
|
<div className="text-sm text-muted-foreground">没有可同步的表。</div>
|
||||||
|
) : (
|
||||||
|
<div className="max-h-96 overflow-auto rounded-md border">
|
||||||
|
{streamList.map((s) => {
|
||||||
|
const checked = !!selected[s.name];
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={s.name}
|
||||||
|
className="flex items-center gap-3 border-b px-3 py-2 last:border-b-0"
|
||||||
|
>
|
||||||
|
<Checkbox
|
||||||
|
checked={checked}
|
||||||
|
onCheckedChange={(c) => toggleStream(s, !!c)}
|
||||||
|
/>
|
||||||
|
<span className="flex-1 font-mono text-sm">{s.name}</span>
|
||||||
|
{mode === "incremental" && checked ? (
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Input
|
||||||
|
className="h-7 w-36 font-mono text-xs"
|
||||||
|
placeholder="primary_key"
|
||||||
|
value={selected[s.name].primary_key ?? ""}
|
||||||
|
onChange={(e) =>
|
||||||
|
setStreamField(s.name, "primary_key", e.target.value)
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<Input
|
||||||
|
className="h-7 w-36 font-mono text-xs"
|
||||||
|
placeholder="update_key"
|
||||||
|
value={selected[s.name].update_key ?? ""}
|
||||||
|
onChange={(e) =>
|
||||||
|
setStreamField(s.name, "update_key", e.target.value)
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Button onClick={submit} disabled={saving}>
|
||||||
|
{saving
|
||||||
|
? isEdit
|
||||||
|
? "保存中…"
|
||||||
|
: "创建中…"
|
||||||
|
: isEdit
|
||||||
|
? "保存修改"
|
||||||
|
: "创建流水线"}
|
||||||
|
</Button>
|
||||||
|
<Button variant="outline" onClick={() => router.push(cancelHref)}>
|
||||||
|
取消
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
102
lib/atlas.ts
102
lib/atlas.ts
|
|
@ -1,6 +1,6 @@
|
||||||
import { execFile } from "child_process";
|
import { execFile } from "child_process";
|
||||||
import { Connection, getConnection } from "./db";
|
import { Connection, Pipeline, StreamConfig, getConnection } from "./db";
|
||||||
import { connectionUrl, execSql } from "./sling";
|
import { connectionUrl, discoverStreams, execSql } from "./sling";
|
||||||
import { resolveBinary } from "./binaries";
|
import { resolveBinary } from "./binaries";
|
||||||
|
|
||||||
// Atlas needs an empty scratch database ("dev database") to plan diffs.
|
// Atlas needs an empty scratch database ("dev database") to plan diffs.
|
||||||
|
|
@ -23,8 +23,13 @@ export function prepareDevDatabase(tgt: Connection): string {
|
||||||
return devDatabaseUrl(tgt);
|
return devDatabaseUrl(tgt);
|
||||||
}
|
}
|
||||||
|
|
||||||
function baseApplyArgs(srcUrl: string, tgtUrl: string, devUrl: string): string[] {
|
function baseApplyArgs(
|
||||||
return [
|
srcUrl: string,
|
||||||
|
tgtUrl: string,
|
||||||
|
devUrl: string,
|
||||||
|
excludes: string[]
|
||||||
|
): string[] {
|
||||||
|
const args = [
|
||||||
"schema",
|
"schema",
|
||||||
"apply",
|
"apply",
|
||||||
"--url",
|
"--url",
|
||||||
|
|
@ -34,47 +39,112 @@ function baseApplyArgs(srcUrl: string, tgtUrl: string, devUrl: string): string[]
|
||||||
"--dev-url",
|
"--dev-url",
|
||||||
devUrl,
|
devUrl,
|
||||||
];
|
];
|
||||||
|
// Community Atlas has no --include; --exclude globs match bare resource
|
||||||
|
// names (no schema prefix). Excluded objects are dropped from the diff
|
||||||
|
// entirely, so e.g. unsupported column types in them never surface.
|
||||||
|
for (const p of excludes) args.push("--exclude", p);
|
||||||
|
return args;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildApplyArgs(
|
export function buildApplyArgs(
|
||||||
srcUrl: string,
|
srcUrl: string,
|
||||||
tgtUrl: string,
|
tgtUrl: string,
|
||||||
devUrl: string
|
devUrl: string,
|
||||||
|
excludes: string[] = []
|
||||||
): string[] {
|
): string[] {
|
||||||
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl), "--auto-approve"];
|
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl, excludes), "--auto-approve"];
|
||||||
}
|
}
|
||||||
|
|
||||||
// atlas rejects --dry-run combined with --auto-approve.
|
// atlas rejects --dry-run combined with --auto-approve.
|
||||||
export function buildDryRunArgs(
|
export function buildDryRunArgs(
|
||||||
srcUrl: string,
|
srcUrl: string,
|
||||||
tgtUrl: string,
|
tgtUrl: string,
|
||||||
devUrl: string
|
devUrl: string,
|
||||||
|
excludes: string[] = []
|
||||||
): string[] {
|
): string[] {
|
||||||
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl), "--dry-run"];
|
return [...baseApplyArgs(srcUrl, tgtUrl, devUrl, excludes), "--dry-run"];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute the --exclude patterns for a pipeline's schema step. Atlas diffs
|
||||||
|
// whole databases, and tables that exist only on the target would be planned
|
||||||
|
// as DROPs — so for 'selected' scope we must exclude every non-selected table
|
||||||
|
// from BOTH sides. Bare table names only: exclude globs match resource names
|
||||||
|
// without schema prefixes.
|
||||||
|
export async function computeSchemaExcludes(
|
||||||
|
pipeline: Pipeline
|
||||||
|
): Promise<string[]> {
|
||||||
|
if ((pipeline.schema_scope || "selected") === "all") return [];
|
||||||
|
|
||||||
|
const streams = JSON.parse(pipeline.streams || "[]") as StreamConfig[];
|
||||||
|
const selected = new Set(
|
||||||
|
streams.map((s) => {
|
||||||
|
const i = s.name.lastIndexOf(".");
|
||||||
|
return i >= 0 ? s.name.slice(i + 1) : s.name;
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
const tableNames = (connId: number, side: string): Set<string> => {
|
||||||
|
try {
|
||||||
|
return new Set(
|
||||||
|
discoverStreams(connId).map((s) => {
|
||||||
|
const i = s.name.lastIndexOf(".");
|
||||||
|
return i >= 0 ? s.name.slice(i + 1) : s.name;
|
||||||
|
})
|
||||||
|
);
|
||||||
|
} catch (e) {
|
||||||
|
// Silently falling back would risk DROPping target-only tables.
|
||||||
|
throw new Error(
|
||||||
|
`无法获取${side}库表列表,已中止 schema 同步:${(e as Error).message}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const srcTables = tableNames(pipeline.source_conn_id, "源");
|
||||||
|
const tgtTables = tableNames(pipeline.target_conn_id, "目标");
|
||||||
|
|
||||||
|
const excludes: string[] = [];
|
||||||
|
for (const t of new Set([...srcTables, ...tgtTables])) {
|
||||||
|
if (!selected.has(t)) excludes.push(t);
|
||||||
|
}
|
||||||
|
// FKs referencing excluded tables vanish from the desired state anyway;
|
||||||
|
// excluding all FKs additionally avoids touching constraints on the
|
||||||
|
// selected tables themselves.
|
||||||
|
if (!pipeline.include_fk) excludes.push("*[type=foreign-key]");
|
||||||
|
return excludes;
|
||||||
}
|
}
|
||||||
|
|
||||||
function atlasBin(): string {
|
function atlasBin(): string {
|
||||||
return resolveBinary("atlas") ?? "atlas";
|
return resolveBinary("atlas") ?? "atlas";
|
||||||
}
|
}
|
||||||
|
|
||||||
export function schemaDiff(
|
export async function schemaDiff(
|
||||||
sourceConnId: number,
|
pipeline: Pipeline
|
||||||
targetConnId: number
|
|
||||||
): Promise<{ ok: boolean; sql: string }> {
|
): Promise<{ ok: boolean; sql: string }> {
|
||||||
const src = getConnection(sourceConnId);
|
const src = getConnection(pipeline.source_conn_id);
|
||||||
const tgt = getConnection(targetConnId);
|
const tgt = getConnection(pipeline.target_conn_id);
|
||||||
if (!src || !tgt) throw new Error("connection not found");
|
if (!src || !tgt) throw new Error("connection not found");
|
||||||
|
let excludes: string[];
|
||||||
|
try {
|
||||||
|
excludes = await computeSchemaExcludes(pipeline);
|
||||||
|
} catch (e) {
|
||||||
|
return { ok: false, sql: (e as Error).message };
|
||||||
|
}
|
||||||
let devUrl: string;
|
let devUrl: string;
|
||||||
try {
|
try {
|
||||||
devUrl = prepareDevDatabase(tgt);
|
devUrl = prepareDevDatabase(tgt);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
const err = e as { stdout?: string; stderr?: string; message?: string };
|
const err = e as { stdout?: string; stderr?: string; message?: string };
|
||||||
return Promise.resolve({
|
return {
|
||||||
ok: false,
|
ok: false,
|
||||||
sql: [err.stdout, err.stderr, err.message].filter(Boolean).join("\n"),
|
sql: [err.stdout, err.stderr, err.message].filter(Boolean).join("\n"),
|
||||||
});
|
};
|
||||||
}
|
}
|
||||||
const args = buildDryRunArgs(connectionUrl(src), connectionUrl(tgt), devUrl);
|
const args = buildDryRunArgs(
|
||||||
|
connectionUrl(src),
|
||||||
|
connectionUrl(tgt),
|
||||||
|
devUrl,
|
||||||
|
excludes
|
||||||
|
);
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
execFile(
|
execFile(
|
||||||
/* turbopackIgnore: true */ atlasBin(),
|
/* turbopackIgnore: true */ atlasBin(),
|
||||||
|
|
|
||||||
38
lib/db.ts
38
lib/db.ts
|
|
@ -41,6 +41,8 @@ export interface Pipeline {
|
||||||
source_conn_id: number;
|
source_conn_id: number;
|
||||||
target_conn_id: number;
|
target_conn_id: number;
|
||||||
schema_sync: number; // 1 | 0
|
schema_sync: number; // 1 | 0
|
||||||
|
schema_scope: string; // 'selected' | 'all'
|
||||||
|
include_fk: number; // 1 | 0
|
||||||
mode: string;
|
mode: string;
|
||||||
streams: string; // JSON: StreamConfig[]
|
streams: string; // JSON: StreamConfig[]
|
||||||
created_at: string;
|
created_at: string;
|
||||||
|
|
@ -52,6 +54,8 @@ export interface PipelineInput {
|
||||||
source_conn_id: number;
|
source_conn_id: number;
|
||||||
target_conn_id: number;
|
target_conn_id: number;
|
||||||
schema_sync?: number;
|
schema_sync?: number;
|
||||||
|
schema_scope?: string;
|
||||||
|
include_fk?: number;
|
||||||
mode?: string;
|
mode?: string;
|
||||||
streams?: string;
|
streams?: string;
|
||||||
}
|
}
|
||||||
|
|
@ -91,6 +95,8 @@ function createDb(): Database.Database {
|
||||||
source_conn_id INTEGER NOT NULL REFERENCES connections(id),
|
source_conn_id INTEGER NOT NULL REFERENCES connections(id),
|
||||||
target_conn_id INTEGER NOT NULL REFERENCES connections(id),
|
target_conn_id INTEGER NOT NULL REFERENCES connections(id),
|
||||||
schema_sync INTEGER DEFAULT 1,
|
schema_sync INTEGER DEFAULT 1,
|
||||||
|
schema_scope TEXT NOT NULL DEFAULT 'selected',
|
||||||
|
include_fk INTEGER NOT NULL DEFAULT 0,
|
||||||
mode TEXT DEFAULT 'full-refresh',
|
mode TEXT DEFAULT 'full-refresh',
|
||||||
streams TEXT NOT NULL DEFAULT '[]',
|
streams TEXT NOT NULL DEFAULT '[]',
|
||||||
created_at TEXT DEFAULT (datetime('now')),
|
created_at TEXT DEFAULT (datetime('now')),
|
||||||
|
|
@ -107,6 +113,30 @@ function createDb(): Database.Database {
|
||||||
pid INTEGER
|
pid INTEGER
|
||||||
);
|
);
|
||||||
`);
|
`);
|
||||||
|
// Lightweight migrations for databases created before these columns existed.
|
||||||
|
// Several Next.js workers may open the DB concurrently during build, so a
|
||||||
|
// racing ALTER can fail with "duplicate column name" — safe to ignore.
|
||||||
|
const addColumnIfMissing = (sql: string, column: string) => {
|
||||||
|
const cols = new Set(
|
||||||
|
(db.pragma("table_info(pipelines)") as { name: string }[]).map(
|
||||||
|
(c) => c.name
|
||||||
|
)
|
||||||
|
);
|
||||||
|
if (cols.has(column)) return;
|
||||||
|
try {
|
||||||
|
db.exec(sql);
|
||||||
|
} catch (e) {
|
||||||
|
if (!(e as Error).message.includes("duplicate column name")) throw e;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
addColumnIfMissing(
|
||||||
|
"ALTER TABLE pipelines ADD COLUMN schema_scope TEXT NOT NULL DEFAULT 'selected'",
|
||||||
|
"schema_scope"
|
||||||
|
);
|
||||||
|
addColumnIfMissing(
|
||||||
|
"ALTER TABLE pipelines ADD COLUMN include_fk INTEGER NOT NULL DEFAULT 0",
|
||||||
|
"include_fk"
|
||||||
|
);
|
||||||
return db;
|
return db;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -186,14 +216,16 @@ export function getPipeline(id: number): Pipeline | null {
|
||||||
export function createPipeline(input: PipelineInput): Pipeline {
|
export function createPipeline(input: PipelineInput): Pipeline {
|
||||||
const res = db
|
const res = db
|
||||||
.prepare(
|
.prepare(
|
||||||
`INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, mode, streams)
|
`INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, schema_scope, include_fk, mode, streams)
|
||||||
VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @mode, @streams)`
|
VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @schema_scope, @include_fk, @mode, @streams)`
|
||||||
)
|
)
|
||||||
.run({
|
.run({
|
||||||
name: input.name,
|
name: input.name,
|
||||||
source_conn_id: input.source_conn_id,
|
source_conn_id: input.source_conn_id,
|
||||||
target_conn_id: input.target_conn_id,
|
target_conn_id: input.target_conn_id,
|
||||||
schema_sync: input.schema_sync ?? 1,
|
schema_sync: input.schema_sync ?? 1,
|
||||||
|
schema_scope: input.schema_scope ?? "selected",
|
||||||
|
include_fk: input.include_fk ?? 0,
|
||||||
mode: input.mode ?? "full-refresh",
|
mode: input.mode ?? "full-refresh",
|
||||||
streams: input.streams ?? "[]",
|
streams: input.streams ?? "[]",
|
||||||
});
|
});
|
||||||
|
|
@ -209,6 +241,8 @@ export function updatePipeline(
|
||||||
"source_conn_id",
|
"source_conn_id",
|
||||||
"target_conn_id",
|
"target_conn_id",
|
||||||
"schema_sync",
|
"schema_sync",
|
||||||
|
"schema_scope",
|
||||||
|
"include_fk",
|
||||||
"mode",
|
"mode",
|
||||||
"streams",
|
"streams",
|
||||||
].filter((k) => input[k as keyof PipelineInput] !== undefined);
|
].filter((k) => input[k as keyof PipelineInput] !== undefined);
|
||||||
|
|
|
||||||
13
lib/jobs.ts
13
lib/jobs.ts
|
|
@ -12,7 +12,7 @@ import {
|
||||||
updateRunStatus,
|
updateRunStatus,
|
||||||
} from "./db";
|
} from "./db";
|
||||||
import { SLING_HOME, buildReplicationYaml, connectionUrl } from "./sling";
|
import { SLING_HOME, buildReplicationYaml, connectionUrl } from "./sling";
|
||||||
import { buildApplyArgs, prepareDevDatabase } from "./atlas";
|
import { buildApplyArgs, computeSchemaExcludes, prepareDevDatabase } from "./atlas";
|
||||||
import { resolveBinary } from "./binaries";
|
import { resolveBinary } from "./binaries";
|
||||||
|
|
||||||
export interface JobEvent {
|
export interface JobEvent {
|
||||||
|
|
@ -137,11 +137,20 @@ class JobManager {
|
||||||
const src = getConnection(pipeline.source_conn_id);
|
const src = getConnection(pipeline.source_conn_id);
|
||||||
const tgt = getConnection(pipeline.target_conn_id);
|
const tgt = getConnection(pipeline.target_conn_id);
|
||||||
if (!src || !tgt) throw new Error("pipeline connections not found");
|
if (!src || !tgt) throw new Error("pipeline connections not found");
|
||||||
|
const excludes = await computeSchemaExcludes(pipeline);
|
||||||
|
const scope = pipeline.schema_scope || "selected";
|
||||||
|
const tableExcludes = pipeline.include_fk
|
||||||
|
? excludes.length
|
||||||
|
: excludes.filter((p) => p !== "*[type=foreign-key]").length;
|
||||||
|
this.emit(
|
||||||
|
state,
|
||||||
|
`[schema] scope=${scope}, excluding ${tableExcludes} objects, fk=${pipeline.include_fk ? "on" : "off"}`
|
||||||
|
);
|
||||||
const devUrl = prepareDevDatabase(tgt);
|
const devUrl = prepareDevDatabase(tgt);
|
||||||
const code = await this.runProcess(
|
const code = await this.runProcess(
|
||||||
state,
|
state,
|
||||||
resolveBinary("atlas") ?? "atlas",
|
resolveBinary("atlas") ?? "atlas",
|
||||||
buildApplyArgs(connectionUrl(src), connectionUrl(tgt), devUrl),
|
buildApplyArgs(connectionUrl(src), connectionUrl(tgt), devUrl, excludes),
|
||||||
{}
|
{}
|
||||||
);
|
);
|
||||||
if (code !== 0) {
|
if (code !== 0) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue