feat: per-connection URL params and per-pipeline env vars
- connections: params (JSON) merged into connection URL query string, applied to both sling env.yaml and atlas URLs; user's foreign_key_checks overrides the default =0 - pipelines: env map written to replication yaml top-level env: - UI: key=value textarea in connection dialog and pipeline form (new/edit) - docs: TiDB Error 8048 (SERIALIZABLE) troubleshooting — set tidb_skip_isolation_level_check=1 as a connection param
This commit is contained in:
parent
884eaa1ed8
commit
8779ad4867
|
|
@ -38,6 +38,11 @@ pnpm dev # 或 pnpm build && pnpm start
|
||||||
- `schema_sync` 开启时,Atlas 负责目标端表结构;此时即使选择 `full-refresh`,实际执行会翻译为 `truncate`(保留 Atlas 建的表结构重灌全量数据,避免 sling drop+重建丢掉索引/约束)。
|
- `schema_sync` 开启时,Atlas 负责目标端表结构;此时即使选择 `full-refresh`,实际执行会翻译为 `truncate`(保留 Atlas 建的表结构重灌全量数据,避免 sling drop+重建丢掉索引/约束)。
|
||||||
- `incremental` 需要为每张表配置 `primary_key` 和 `update_key`。
|
- `incremental` 需要为每张表配置 `primary_key` 和 `update_key`。
|
||||||
- sling 连接 URL 自动带 `foreign_key_checks=0`(会话级),避免外键顺序问题;约束定义本身保留。
|
- sling 连接 URL 自动带 `foreign_key_checks=0`(会话级),避免外键顺序问题;约束定义本身保留。
|
||||||
|
- 连接支持自定义「连接参数」(URL query string,sling/Atlas 均生效),流水线支持自定义「环境变量」(写入 replication 的 `env:`)。
|
||||||
|
|
||||||
|
## 已知问题
|
||||||
|
|
||||||
|
- TiDB 目标库报 `Error 8048: isolation level 'SERIALIZABLE' is not supported`:见 [docs/tidb-serializable-isolation.md](docs/tidb-serializable-isolation.md),给连接加参数 `tidb_skip_isolation_level_check=1` 即可。
|
||||||
|
|
||||||
## 数据与安全说明
|
## 数据与安全说明
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { deleteConnection, updateConnection } from "@/lib/db";
|
import { deleteConnection, updateConnection, normalizeJsonColumn } from "@/lib/db";
|
||||||
|
|
||||||
export const runtime = "nodejs";
|
export const runtime = "nodejs";
|
||||||
export const dynamic = "force-dynamic";
|
export const dynamic = "force-dynamic";
|
||||||
|
|
@ -9,6 +9,9 @@ export async function PUT(request: Request, { params }: Ctx) {
|
||||||
try {
|
try {
|
||||||
const { id } = await params;
|
const { id } = await params;
|
||||||
const body = await request.json();
|
const body = await request.json();
|
||||||
|
if (body.params !== undefined) {
|
||||||
|
body.params = normalizeJsonColumn(body.params);
|
||||||
|
}
|
||||||
const conn = updateConnection(Number(id), body);
|
const conn = updateConnection(Number(id), body);
|
||||||
if (!conn) {
|
if (!conn) {
|
||||||
return Response.json({ error: "connection not found" }, { status: 404 });
|
return Response.json({ error: "connection not found" }, { status: 404 });
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { createConnection, listConnections } from "@/lib/db";
|
import { createConnection, listConnections, normalizeJsonColumn } from "@/lib/db";
|
||||||
|
|
||||||
export const runtime = "nodejs";
|
export const runtime = "nodejs";
|
||||||
export const dynamic = "force-dynamic";
|
export const dynamic = "force-dynamic";
|
||||||
|
|
@ -28,6 +28,7 @@ export async function POST(request: Request) {
|
||||||
user: String(body.user ?? ""),
|
user: String(body.user ?? ""),
|
||||||
password: String(body.password ?? ""),
|
password: String(body.password ?? ""),
|
||||||
database: String(body.database ?? ""),
|
database: String(body.database ?? ""),
|
||||||
|
params: normalizeJsonColumn(body.params),
|
||||||
});
|
});
|
||||||
return Response.json(conn, { status: 201 });
|
return Response.json(conn, { status: 201 });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { deletePipeline, getPipeline, updatePipeline } from "@/lib/db";
|
import { deletePipeline, getPipeline, updatePipeline, normalizeJsonColumn } from "@/lib/db";
|
||||||
|
|
||||||
export const runtime = "nodejs";
|
export const runtime = "nodejs";
|
||||||
export const dynamic = "force-dynamic";
|
export const dynamic = "force-dynamic";
|
||||||
|
|
@ -42,6 +42,9 @@ export async function PUT(request: Request, { params }: Ctx) {
|
||||||
if (body.streams !== undefined) {
|
if (body.streams !== undefined) {
|
||||||
patch.streams = normalizeStreams(body.streams);
|
patch.streams = normalizeStreams(body.streams);
|
||||||
}
|
}
|
||||||
|
if (body.env !== undefined) {
|
||||||
|
patch.env = normalizeJsonColumn(body.env);
|
||||||
|
}
|
||||||
const pipeline = updatePipeline(Number(id), patch);
|
const pipeline = updatePipeline(Number(id), patch);
|
||||||
if (!pipeline) {
|
if (!pipeline) {
|
||||||
return Response.json({ error: "pipeline not found" }, { status: 404 });
|
return Response.json({ error: "pipeline not found" }, { status: 404 });
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { createPipeline, listPipelines } from "@/lib/db";
|
import { createPipeline, listPipelines, normalizeJsonColumn } from "@/lib/db";
|
||||||
|
|
||||||
export const runtime = "nodejs";
|
export const runtime = "nodejs";
|
||||||
export const dynamic = "force-dynamic";
|
export const dynamic = "force-dynamic";
|
||||||
|
|
@ -34,6 +34,7 @@ export async function POST(request: Request) {
|
||||||
include_fk: body.include_fk === undefined ? 0 : Number(body.include_fk),
|
include_fk: body.include_fk === undefined ? 0 : Number(body.include_fk),
|
||||||
mode: body.mode ? String(body.mode) : "full-refresh",
|
mode: body.mode ? String(body.mode) : "full-refresh",
|
||||||
streams: normalizeStreams(body.streams),
|
streams: normalizeStreams(body.streams),
|
||||||
|
env: normalizeJsonColumn(body.env),
|
||||||
});
|
});
|
||||||
return Response.json(pipeline, { status: 201 });
|
return Response.json(pipeline, { status: 201 });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
|
||||||
|
|
@ -117,7 +117,18 @@ export default function ConnectionsPage() {
|
||||||
{list.map((c) => (
|
{list.map((c) => (
|
||||||
<Fragment key={c.id}>
|
<Fragment key={c.id}>
|
||||||
<TableRow>
|
<TableRow>
|
||||||
<TableCell className="font-medium">{c.name}</TableCell>
|
<TableCell className="font-medium">
|
||||||
|
{c.name}
|
||||||
|
{c.params && c.params !== "{}" ? (
|
||||||
|
<div className="mt-0.5 font-mono text-xs text-muted-foreground">
|
||||||
|
{Object.entries(
|
||||||
|
JSON.parse(c.params) as Record<string, string>
|
||||||
|
)
|
||||||
|
.map(([k, v]) => `${k}=${v}`)
|
||||||
|
.join(" ")}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</TableCell>
|
||||||
<TableCell className="font-mono text-xs">{c.type}</TableCell>
|
<TableCell className="font-mono text-xs">{c.type}</TableCell>
|
||||||
<TableCell className="font-mono text-xs">
|
<TableCell className="font-mono text-xs">
|
||||||
{c.host}:{c.port}
|
{c.host}:{c.port}
|
||||||
|
|
|
||||||
|
|
@ -91,6 +91,22 @@ export default function PipelineDetailPage() {
|
||||||
}
|
}
|
||||||
}, [pipeline]);
|
}, [pipeline]);
|
||||||
|
|
||||||
|
const envEntries = useMemo<[string, string][]>(() => {
|
||||||
|
if (!pipeline) return [];
|
||||||
|
try {
|
||||||
|
const v = JSON.parse(pipeline.env || "{}") as unknown;
|
||||||
|
if (v && typeof v === "object" && !Array.isArray(v)) {
|
||||||
|
return Object.entries(v as Record<string, unknown>).map(([k, val]) => [
|
||||||
|
k,
|
||||||
|
String(val),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// fall through
|
||||||
|
}
|
||||||
|
return [];
|
||||||
|
}, [pipeline]);
|
||||||
|
|
||||||
const latestRun = runs[0] ?? null;
|
const latestRun = runs[0] ?? null;
|
||||||
|
|
||||||
async function runNow() {
|
async function runNow() {
|
||||||
|
|
@ -203,6 +219,14 @@ export default function PipelineDetailPage() {
|
||||||
<dt className="text-muted-foreground">包含外键</dt>
|
<dt className="text-muted-foreground">包含外键</dt>
|
||||||
<dd className="font-medium">{pipeline.include_fk ? "是" : "否"}</dd>
|
<dd className="font-medium">{pipeline.include_fk ? "是" : "否"}</dd>
|
||||||
</div>
|
</div>
|
||||||
|
{envEntries.length > 0 ? (
|
||||||
|
<div>
|
||||||
|
<dt className="text-muted-foreground">环境变量</dt>
|
||||||
|
<dd className="font-mono text-xs">
|
||||||
|
{envEntries.map(([k, v]) => `${k}=${v}`).join(" ")}
|
||||||
|
</dd>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
<div>
|
<div>
|
||||||
<dt className="text-muted-foreground">创建时间</dt>
|
<dt className="text-muted-foreground">创建时间</dt>
|
||||||
<dd>{fmtTime(pipeline.created_at)}</dd>
|
<dd>{fmtTime(pipeline.created_at)}</dd>
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
import { useState } from "react";
|
import { useState } from "react";
|
||||||
import { toast } from "sonner";
|
import { toast } from "sonner";
|
||||||
import { api, type Connection } from "@/components/api";
|
import { api, type Connection } from "@/components/api";
|
||||||
|
import { keyValueLinesFromJson, parseKeyValueLines } from "@/components/kv";
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
import {
|
import {
|
||||||
Dialog,
|
Dialog,
|
||||||
|
|
@ -20,6 +21,7 @@ import {
|
||||||
SelectTrigger,
|
SelectTrigger,
|
||||||
SelectValue,
|
SelectValue,
|
||||||
} from "@/components/ui/select";
|
} from "@/components/ui/select";
|
||||||
|
import { Textarea } from "@/components/ui/textarea";
|
||||||
|
|
||||||
interface ConnectionDialogProps {
|
interface ConnectionDialogProps {
|
||||||
open: boolean;
|
open: boolean;
|
||||||
|
|
@ -59,6 +61,9 @@ export function ConnectionDialog({
|
||||||
: EMPTY
|
: EMPTY
|
||||||
);
|
);
|
||||||
const [saving, setSaving] = useState(false);
|
const [saving, setSaving] = useState(false);
|
||||||
|
const [paramsText, setParamsText] = useState(() =>
|
||||||
|
initial ? keyValueLinesFromJson(initial.params) : ""
|
||||||
|
);
|
||||||
|
|
||||||
const set = (k: keyof typeof EMPTY) => (e: React.ChangeEvent<HTMLInputElement>) =>
|
const set = (k: keyof typeof EMPTY) => (e: React.ChangeEvent<HTMLInputElement>) =>
|
||||||
setForm((f) => ({ ...f, [k]: e.target.value }));
|
setForm((f) => ({ ...f, [k]: e.target.value }));
|
||||||
|
|
@ -79,6 +84,7 @@ export function ConnectionDialog({
|
||||||
user: form.user.trim(),
|
user: form.user.trim(),
|
||||||
password: form.password,
|
password: form.password,
|
||||||
database: form.database.trim(),
|
database: form.database.trim(),
|
||||||
|
params: parseKeyValueLines(paramsText),
|
||||||
});
|
});
|
||||||
if (initial) {
|
if (initial) {
|
||||||
await api(`/api/connections/${initial.id}`, {
|
await api(`/api/connections/${initial.id}`, {
|
||||||
|
|
@ -174,6 +180,21 @@ export function ConnectionDialog({
|
||||||
onChange={set("database")}
|
onChange={set("database")}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
<div className="grid gap-1.5">
|
||||||
|
<Label htmlFor="conn-params">连接参数</Label>
|
||||||
|
<Textarea
|
||||||
|
id="conn-params"
|
||||||
|
className="font-mono text-xs"
|
||||||
|
rows={3}
|
||||||
|
value={paramsText}
|
||||||
|
onChange={(e) => setParamsText(e.target.value)}
|
||||||
|
placeholder={"tidb_skip_isolation_level_check=1"}
|
||||||
|
/>
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
每行一个 key=value,作为 URL 查询参数传递,sling / atlas 均生效;TiDB
|
||||||
|
目标库建议设置 tidb_skip_isolation_level_check=1。
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<DialogFooter>
|
<DialogFooter>
|
||||||
<Button variant="outline" onClick={() => onOpenChange(false)} disabled={saving}>
|
<Button variant="outline" onClick={() => onOpenChange(false)} disabled={saving}>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
// Helpers for the "one key=value per line" textareas used by the connection
|
||||||
|
// dialog (URL params) and the pipeline form (replication env).
|
||||||
|
|
||||||
|
// Parse textarea lines into an object. Blank lines, lines starting with '#'
|
||||||
|
// and lines without a key or '=' are ignored. Values may contain '='.
|
||||||
|
export function parseKeyValueLines(text: string): Record<string, string> {
|
||||||
|
const out: Record<string, string> = {};
|
||||||
|
for (const raw of text.split("\n")) {
|
||||||
|
const line = raw.trim();
|
||||||
|
if (!line || line.startsWith("#")) continue;
|
||||||
|
const i = line.indexOf("=");
|
||||||
|
if (i <= 0) continue;
|
||||||
|
const key = line.slice(0, i).trim();
|
||||||
|
const value = line.slice(i + 1).trim();
|
||||||
|
if (key) out[key] = value;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serialize a JSON-object column back into key=value lines for editing.
|
||||||
|
export function keyValueLinesFromJson(json: string | undefined | null): string {
|
||||||
|
if (!json) return "";
|
||||||
|
try {
|
||||||
|
const v = JSON.parse(json) as unknown;
|
||||||
|
if (v && typeof v === "object" && !Array.isArray(v)) {
|
||||||
|
return Object.entries(v as Record<string, unknown>)
|
||||||
|
.map(([k, val]) => `${k}=${String(val)}`)
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// fall through
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
@ -10,6 +10,7 @@ import {
|
||||||
type StreamConfig,
|
type StreamConfig,
|
||||||
type StreamInfo,
|
type StreamInfo,
|
||||||
} from "@/components/api";
|
} from "@/components/api";
|
||||||
|
import { keyValueLinesFromJson, parseKeyValueLines } from "@/components/kv";
|
||||||
import { Button } from "@/components/ui/button";
|
import { Button } from "@/components/ui/button";
|
||||||
import {
|
import {
|
||||||
Card,
|
Card,
|
||||||
|
|
@ -29,6 +30,7 @@ import {
|
||||||
SelectValue,
|
SelectValue,
|
||||||
} from "@/components/ui/select";
|
} from "@/components/ui/select";
|
||||||
import { Switch } from "@/components/ui/switch";
|
import { Switch } from "@/components/ui/switch";
|
||||||
|
import { Textarea } from "@/components/ui/textarea";
|
||||||
|
|
||||||
const MODES = [
|
const MODES = [
|
||||||
{ value: "full-refresh", label: "full-refresh(全量覆盖,默认)" },
|
{ value: "full-refresh", label: "full-refresh(全量覆盖,默认)" },
|
||||||
|
|
@ -58,6 +60,9 @@ export function PipelineForm({ initial }: { initial?: Pipeline }) {
|
||||||
initial?.schema_scope ?? "selected"
|
initial?.schema_scope ?? "selected"
|
||||||
);
|
);
|
||||||
const [includeFk, setIncludeFk] = useState(initial ? !!initial.include_fk : false);
|
const [includeFk, setIncludeFk] = useState(initial ? !!initial.include_fk : false);
|
||||||
|
const [envText, setEnvText] = useState(() =>
|
||||||
|
initial ? keyValueLinesFromJson(initial.env) : ""
|
||||||
|
);
|
||||||
|
|
||||||
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
|
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
|
||||||
const [streamsLoading, setStreamsLoading] = useState(false);
|
const [streamsLoading, setStreamsLoading] = useState(false);
|
||||||
|
|
@ -149,6 +154,7 @@ export function PipelineForm({ initial }: { initial?: Pipeline }) {
|
||||||
include_fk: includeFk ? 1 : 0,
|
include_fk: includeFk ? 1 : 0,
|
||||||
mode,
|
mode,
|
||||||
streams: JSON.stringify(selectedList),
|
streams: JSON.stringify(selectedList),
|
||||||
|
env: parseKeyValueLines(envText),
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
const p = isEdit
|
const p = isEdit
|
||||||
|
|
@ -301,6 +307,21 @@ export function PipelineForm({ initial }: { initial?: Pipeline }) {
|
||||||
disabled={!schemaSync}
|
disabled={!schemaSync}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
<div className="grid gap-1.5">
|
||||||
|
<Label htmlFor="p-env">环境变量</Label>
|
||||||
|
<Textarea
|
||||||
|
id="p-env"
|
||||||
|
className="font-mono text-xs"
|
||||||
|
rows={3}
|
||||||
|
value={envText}
|
||||||
|
onChange={(e) => setEnvText(e.target.value)}
|
||||||
|
placeholder={"SLING_THREADS=4"}
|
||||||
|
/>
|
||||||
|
<p className="text-xs text-muted-foreground">
|
||||||
|
每行一个 key=value,写入 sling replication 的顶层 env(sling 会导出为进程环境变量),如
|
||||||
|
SLING_THREADS=4。
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
</CardContent>
|
</CardContent>
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,53 @@
|
||||||
|
# TiDB 目标库:Error 8048 isolation level 'SERIALIZABLE' is not supported
|
||||||
|
|
||||||
|
## 现象
|
||||||
|
|
||||||
|
目标库为 TiDB 时,sling 数据迁移在每个 stream 的写入阶段失败:
|
||||||
|
|
||||||
|
```
|
||||||
|
writing to target database [mode: truncate]
|
||||||
|
created table `test_demo`.`crm_orders_tmp`
|
||||||
|
execution failed
|
||||||
|
Error 8048 (HY000): The isolation level 'SERIALIZABLE' is not supported.
|
||||||
|
Set tidb_skip_isolation_level_check=1 to skip this error
|
||||||
|
```
|
||||||
|
|
||||||
|
## 原因
|
||||||
|
|
||||||
|
TiDB 不支持 `SERIALIZABLE` 隔离级别(TiDB 默认使用乐观锁/快照隔离)。sling
|
||||||
|
在目标端开启事务时请求了 `SERIALIZABLE`,TiDB 默认会直接报错拒绝。
|
||||||
|
|
||||||
|
TiDB 提供了兼容开关 `tidb_skip_isolation_level_check=1`:设置后跳过隔离级别
|
||||||
|
检查,按 TiDB 实际支持的级别执行。对"测试环境 → 本地 dev 库"的全量数据灌入
|
||||||
|
场景,这是安全的。
|
||||||
|
|
||||||
|
## 解决:连接参数
|
||||||
|
|
||||||
|
在 sling-ui 中编辑 TiDB 目标连接,在「连接参数」中加一行:
|
||||||
|
|
||||||
|
```
|
||||||
|
tidb_skip_isolation_level_check=1
|
||||||
|
```
|
||||||
|
|
||||||
|
### 工作原理
|
||||||
|
|
||||||
|
连接参数会被拼进连接 URL 的 query string,例如:
|
||||||
|
|
||||||
|
```
|
||||||
|
mysql://user:pass@tidb-host:4000/dbname?tidb_skip_isolation_level_check=1
|
||||||
|
```
|
||||||
|
|
||||||
|
sling 底层使用 go-sql-driver/mysql,会把未识别的 DSN 参数作为 session 变量
|
||||||
|
在连接建立时执行 `SET <key>=<value>`(本项目内置的 `foreign_key_checks=0`
|
||||||
|
也是同一机制)。Atlas 的连接 URL 同样会带上这些参数。
|
||||||
|
|
||||||
|
已实测验证:对普通 MySQL 设置该参数会报 `Error 1193: Unknown system
|
||||||
|
variable`,证明参数确实被 SET 为 session 变量;TiDB 上该变量存在,即达到
|
||||||
|
预期效果。
|
||||||
|
|
||||||
|
## 备注
|
||||||
|
|
||||||
|
- 该参数只对 TiDB 有效,不要给普通 MySQL/MariaDB 连接设置。
|
||||||
|
- 若希望对整个 TiDB 实例生效(所有客户端),也可以在 TiDB 上执行
|
||||||
|
`SET GLOBAL tidb_skip_isolation_level_check=1;`,但用连接参数的方式
|
||||||
|
影响面最小,推荐。
|
||||||
13
lib/atlas.ts
13
lib/atlas.ts
|
|
@ -1,6 +1,6 @@
|
||||||
import { execFile } from "child_process";
|
import { execFile } from "child_process";
|
||||||
import { Connection, Pipeline, StreamConfig, getConnection } from "./db";
|
import { Connection, Pipeline, StreamConfig, getConnection } from "./db";
|
||||||
import { connectionUrl, discoverStreams, execSql } from "./sling";
|
import { appendParams, connectionUrl, discoverStreams, execSql } from "./sling";
|
||||||
import { resolveBinary } from "./binaries";
|
import { resolveBinary } from "./binaries";
|
||||||
|
|
||||||
// Atlas needs an empty scratch database ("dev database") to plan diffs.
|
// Atlas needs an empty scratch database ("dev database") to plan diffs.
|
||||||
|
|
@ -10,10 +10,19 @@ import { resolveBinary } from "./binaries";
|
||||||
// scratch schema on the target server instead — no Docker needed.
|
// scratch schema on the target server instead — no Docker needed.
|
||||||
const DEV_SCHEMA = "sling_ui_atlas_dev";
|
const DEV_SCHEMA = "sling_ui_atlas_dev";
|
||||||
|
|
||||||
|
// The scratch schema lives on the target server, so it inherits the target
|
||||||
|
// connection's URL params (e.g. tidb_skip_isolation_level_check for TiDB).
|
||||||
function devDatabaseUrl(tgt: Connection): string {
|
function devDatabaseUrl(tgt: Connection): string {
|
||||||
const user = encodeURIComponent(tgt.user);
|
const user = encodeURIComponent(tgt.user);
|
||||||
const pass = encodeURIComponent(tgt.password);
|
const pass = encodeURIComponent(tgt.password);
|
||||||
return `mysql://${user}:${pass}@${tgt.host}:${tgt.port}/${DEV_SCHEMA}`;
|
const base = `mysql://${user}:${pass}@${tgt.host}:${tgt.port}/${DEV_SCHEMA}`;
|
||||||
|
let params: Record<string, string> = {};
|
||||||
|
try {
|
||||||
|
params = JSON.parse(tgt.params || "{}") as Record<string, string>;
|
||||||
|
} catch {
|
||||||
|
// ignore malformed params
|
||||||
|
}
|
||||||
|
return appendParams(base, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create the scratch schema if missing and return its URL. Atlas keeps it
|
// Create the scratch schema if missing and return its URL. Atlas keeps it
|
||||||
|
|
|
||||||
42
lib/db.ts
42
lib/db.ts
|
|
@ -14,6 +14,7 @@ export interface Connection {
|
||||||
user: string;
|
user: string;
|
||||||
password: string;
|
password: string;
|
||||||
database: string;
|
database: string;
|
||||||
|
params: string; // JSON object: URL query params, e.g. {"tidb_skip_isolation_level_check":"1"}
|
||||||
created_at: string;
|
created_at: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -25,6 +26,7 @@ export interface ConnectionInput {
|
||||||
user: string;
|
user: string;
|
||||||
password: string;
|
password: string;
|
||||||
database: string;
|
database: string;
|
||||||
|
params?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StreamConfig {
|
export interface StreamConfig {
|
||||||
|
|
@ -45,6 +47,7 @@ export interface Pipeline {
|
||||||
include_fk: number; // 1 | 0
|
include_fk: number; // 1 | 0
|
||||||
mode: string;
|
mode: string;
|
||||||
streams: string; // JSON: StreamConfig[]
|
streams: string; // JSON: StreamConfig[]
|
||||||
|
env: string; // JSON object: sling replication env, e.g. {"SLING_THREADS":"4"}
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|
@ -58,10 +61,18 @@ export interface PipelineInput {
|
||||||
include_fk?: number;
|
include_fk?: number;
|
||||||
mode?: string;
|
mode?: string;
|
||||||
streams?: string;
|
streams?: string;
|
||||||
|
env?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type RunStatus = "queued" | "running" | "success" | "failed";
|
export type RunStatus = "queued" | "running" | "success" | "failed";
|
||||||
|
|
||||||
|
// Normalize a JSON-object column value from an API body: accept either a
|
||||||
|
// plain object or a pre-serialized JSON string.
|
||||||
|
export function normalizeJsonColumn(raw: unknown): string {
|
||||||
|
if (typeof raw === "string") return raw;
|
||||||
|
return JSON.stringify(raw ?? {});
|
||||||
|
}
|
||||||
|
|
||||||
export interface Run {
|
export interface Run {
|
||||||
id: number;
|
id: number;
|
||||||
pipeline_id: number;
|
pipeline_id: number;
|
||||||
|
|
@ -87,6 +98,7 @@ function createDb(): Database.Database {
|
||||||
user TEXT NOT NULL,
|
user TEXT NOT NULL,
|
||||||
password TEXT NOT NULL DEFAULT '',
|
password TEXT NOT NULL DEFAULT '',
|
||||||
database TEXT NOT NULL,
|
database TEXT NOT NULL,
|
||||||
|
params TEXT NOT NULL DEFAULT '{}',
|
||||||
created_at TEXT DEFAULT (datetime('now'))
|
created_at TEXT DEFAULT (datetime('now'))
|
||||||
);
|
);
|
||||||
CREATE TABLE IF NOT EXISTS pipelines (
|
CREATE TABLE IF NOT EXISTS pipelines (
|
||||||
|
|
@ -99,6 +111,7 @@ function createDb(): Database.Database {
|
||||||
include_fk INTEGER NOT NULL DEFAULT 0,
|
include_fk INTEGER NOT NULL DEFAULT 0,
|
||||||
mode TEXT DEFAULT 'full-refresh',
|
mode TEXT DEFAULT 'full-refresh',
|
||||||
streams TEXT NOT NULL DEFAULT '[]',
|
streams TEXT NOT NULL DEFAULT '[]',
|
||||||
|
env TEXT NOT NULL DEFAULT '{}',
|
||||||
created_at TEXT DEFAULT (datetime('now')),
|
created_at TEXT DEFAULT (datetime('now')),
|
||||||
updated_at TEXT DEFAULT (datetime('now'))
|
updated_at TEXT DEFAULT (datetime('now'))
|
||||||
);
|
);
|
||||||
|
|
@ -116,9 +129,9 @@ function createDb(): Database.Database {
|
||||||
// Lightweight migrations for databases created before these columns existed.
|
// Lightweight migrations for databases created before these columns existed.
|
||||||
// Several Next.js workers may open the DB concurrently during build, so a
|
// Several Next.js workers may open the DB concurrently during build, so a
|
||||||
// racing ALTER can fail with "duplicate column name" — safe to ignore.
|
// racing ALTER can fail with "duplicate column name" — safe to ignore.
|
||||||
const addColumnIfMissing = (sql: string, column: string) => {
|
const addColumnIfMissing = (table: string, sql: string, column: string) => {
|
||||||
const cols = new Set(
|
const cols = new Set(
|
||||||
(db.pragma("table_info(pipelines)") as { name: string }[]).map(
|
(db.pragma(`table_info(${table})`) as { name: string }[]).map(
|
||||||
(c) => c.name
|
(c) => c.name
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
@ -130,13 +143,25 @@ function createDb(): Database.Database {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
addColumnIfMissing(
|
addColumnIfMissing(
|
||||||
|
"pipelines",
|
||||||
"ALTER TABLE pipelines ADD COLUMN schema_scope TEXT NOT NULL DEFAULT 'selected'",
|
"ALTER TABLE pipelines ADD COLUMN schema_scope TEXT NOT NULL DEFAULT 'selected'",
|
||||||
"schema_scope"
|
"schema_scope"
|
||||||
);
|
);
|
||||||
addColumnIfMissing(
|
addColumnIfMissing(
|
||||||
|
"pipelines",
|
||||||
"ALTER TABLE pipelines ADD COLUMN include_fk INTEGER NOT NULL DEFAULT 0",
|
"ALTER TABLE pipelines ADD COLUMN include_fk INTEGER NOT NULL DEFAULT 0",
|
||||||
"include_fk"
|
"include_fk"
|
||||||
);
|
);
|
||||||
|
addColumnIfMissing(
|
||||||
|
"connections",
|
||||||
|
"ALTER TABLE connections ADD COLUMN params TEXT NOT NULL DEFAULT '{}'",
|
||||||
|
"params"
|
||||||
|
);
|
||||||
|
addColumnIfMissing(
|
||||||
|
"pipelines",
|
||||||
|
"ALTER TABLE pipelines ADD COLUMN env TEXT NOT NULL DEFAULT '{}'",
|
||||||
|
"env"
|
||||||
|
);
|
||||||
return db;
|
return db;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -164,8 +189,8 @@ export function getConnection(id: number): Connection | null {
|
||||||
export function createConnection(input: ConnectionInput): Connection {
|
export function createConnection(input: ConnectionInput): Connection {
|
||||||
const res = db
|
const res = db
|
||||||
.prepare(
|
.prepare(
|
||||||
`INSERT INTO connections (name, type, host, port, user, password, database)
|
`INSERT INTO connections (name, type, host, port, user, password, database, params)
|
||||||
VALUES (@name, @type, @host, @port, @user, @password, @database)`
|
VALUES (@name, @type, @host, @port, @user, @password, @database, @params)`
|
||||||
)
|
)
|
||||||
.run({
|
.run({
|
||||||
name: input.name,
|
name: input.name,
|
||||||
|
|
@ -175,6 +200,7 @@ export function createConnection(input: ConnectionInput): Connection {
|
||||||
user: input.user,
|
user: input.user,
|
||||||
password: input.password ?? "",
|
password: input.password ?? "",
|
||||||
database: input.database,
|
database: input.database,
|
||||||
|
params: input.params ?? "{}",
|
||||||
});
|
});
|
||||||
return getConnection(Number(res.lastInsertRowid))!;
|
return getConnection(Number(res.lastInsertRowid))!;
|
||||||
}
|
}
|
||||||
|
|
@ -183,7 +209,7 @@ export function updateConnection(
|
||||||
id: number,
|
id: number,
|
||||||
input: Partial<ConnectionInput>
|
input: Partial<ConnectionInput>
|
||||||
): Connection | null {
|
): Connection | null {
|
||||||
const fields = ["name", "type", "host", "port", "user", "password", "database"].filter(
|
const fields = ["name", "type", "host", "port", "user", "password", "database", "params"].filter(
|
||||||
(k) => input[k as keyof ConnectionInput] !== undefined
|
(k) => input[k as keyof ConnectionInput] !== undefined
|
||||||
);
|
);
|
||||||
if (fields.length === 0) return getConnection(id);
|
if (fields.length === 0) return getConnection(id);
|
||||||
|
|
@ -216,8 +242,8 @@ export function getPipeline(id: number): Pipeline | null {
|
||||||
export function createPipeline(input: PipelineInput): Pipeline {
|
export function createPipeline(input: PipelineInput): Pipeline {
|
||||||
const res = db
|
const res = db
|
||||||
.prepare(
|
.prepare(
|
||||||
`INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, schema_scope, include_fk, mode, streams)
|
`INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, schema_scope, include_fk, mode, streams, env)
|
||||||
VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @schema_scope, @include_fk, @mode, @streams)`
|
VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @schema_scope, @include_fk, @mode, @streams, @env)`
|
||||||
)
|
)
|
||||||
.run({
|
.run({
|
||||||
name: input.name,
|
name: input.name,
|
||||||
|
|
@ -228,6 +254,7 @@ export function createPipeline(input: PipelineInput): Pipeline {
|
||||||
include_fk: input.include_fk ?? 0,
|
include_fk: input.include_fk ?? 0,
|
||||||
mode: input.mode ?? "full-refresh",
|
mode: input.mode ?? "full-refresh",
|
||||||
streams: input.streams ?? "[]",
|
streams: input.streams ?? "[]",
|
||||||
|
env: input.env ?? "{}",
|
||||||
});
|
});
|
||||||
return getPipeline(Number(res.lastInsertRowid))!;
|
return getPipeline(Number(res.lastInsertRowid))!;
|
||||||
}
|
}
|
||||||
|
|
@ -245,6 +272,7 @@ export function updatePipeline(
|
||||||
"include_fk",
|
"include_fk",
|
||||||
"mode",
|
"mode",
|
||||||
"streams",
|
"streams",
|
||||||
|
"env",
|
||||||
].filter((k) => input[k as keyof PipelineInput] !== undefined);
|
].filter((k) => input[k as keyof PipelineInput] !== undefined);
|
||||||
if (fields.length === 0) return getPipeline(id);
|
if (fields.length === 0) return getPipeline(id);
|
||||||
const sets = fields.map((f) => `${f} = @${f}`).join(", ");
|
const sets = fields.map((f) => `${f} = @${f}`).join(", ");
|
||||||
|
|
|
||||||
54
lib/sling.ts
54
lib/sling.ts
|
|
@ -19,24 +19,62 @@ export interface DiscoveredStream {
|
||||||
columns?: string[];
|
columns?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build a mysql:// URL from a connection record, escaping credentials.
|
// Parse a connection/pipeline JSON-object column; tolerate bad data.
|
||||||
|
function parseJsonObject(json: string | undefined | null): Record<string, string> {
|
||||||
|
if (!json) return {};
|
||||||
|
try {
|
||||||
|
const v = JSON.parse(json) as unknown;
|
||||||
|
if (v && typeof v === "object" && !Array.isArray(v)) {
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(v as Record<string, unknown>).map(([k, val]) => [
|
||||||
|
k,
|
||||||
|
String(val),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// fall through
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Append query params to a URL, percent-encoding keys and values.
|
||||||
|
export function appendParams(url: string, params: Record<string, string>): string {
|
||||||
|
const entries = Object.entries(params);
|
||||||
|
if (entries.length === 0) return url;
|
||||||
|
const qs = entries
|
||||||
|
.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
|
||||||
|
.join("&");
|
||||||
|
return `${url}${url.includes("?") ? "&" : "?"}${qs}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build a mysql:// URL from a connection record, escaping credentials and
|
||||||
|
// merging the connection's extra params into the query string. These params
|
||||||
|
// reach both sling (env.yaml) and atlas (--url/--to/--dev-url); the
|
||||||
|
// go-sql-driver SETs unknown DSN params as session variables, which is how
|
||||||
|
// e.g. tidb_skip_isolation_level_check=1 takes effect.
|
||||||
export function connectionUrl(conn: Connection): string {
|
export function connectionUrl(conn: Connection): string {
|
||||||
const user = encodeURIComponent(conn.user);
|
const user = encodeURIComponent(conn.user);
|
||||||
const pass = encodeURIComponent(conn.password);
|
const pass = encodeURIComponent(conn.password);
|
||||||
return `mysql://${user}:${pass}@${conn.host}:${conn.port}/${conn.database}`;
|
const base = `mysql://${user}:${pass}@${conn.host}:${conn.port}/${conn.database}`;
|
||||||
|
return appendParams(base, parseJsonObject(conn.params));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Regenerate env.yaml from the connections table. Names are uppercased
|
// Regenerate env.yaml from the connections table. Names are uppercased
|
||||||
// because sling treats connection names case-insensitively. Session-level
|
// because sling treats connection names case-insensitively. Session-level
|
||||||
// foreign_key_checks=0 lets sling truncate/drop FK-referenced parent tables
|
// foreign_key_checks=0 lets sling truncate/drop FK-referenced parent tables
|
||||||
// in any order (same approach as mysqldump); constraints stay defined.
|
// in any order (same approach as mysqldump); constraints stay defined. It is
|
||||||
|
// appended only here (atlas doesn't need it) and never overrides a value the
|
||||||
|
// user set explicitly in connection params.
|
||||||
export function syncEnvYaml(): string {
|
export function syncEnvYaml(): string {
|
||||||
fs.mkdirSync(SLING_HOME, { recursive: true });
|
fs.mkdirSync(SLING_HOME, { recursive: true });
|
||||||
const connections: Record<string, { url: string }> = {};
|
const connections: Record<string, { url: string }> = {};
|
||||||
for (const c of listConnections()) {
|
for (const c of listConnections()) {
|
||||||
connections[c.name.toUpperCase()] = {
|
let url = connectionUrl(c);
|
||||||
url: `${connectionUrl(c)}?foreign_key_checks=0`,
|
if (!parseJsonObject(c.params)["foreign_key_checks"]) {
|
||||||
};
|
url = appendParams(url, { foreign_key_checks: "0" });
|
||||||
|
}
|
||||||
|
connections[c.name.toUpperCase()] = { url };
|
||||||
}
|
}
|
||||||
const file = path.join(SLING_HOME, "env.yaml");
|
const file = path.join(SLING_HOME, "env.yaml");
|
||||||
fs.writeFileSync(file, dump({ connections }));
|
fs.writeFileSync(file, dump({ connections }));
|
||||||
|
|
@ -236,7 +274,9 @@ export function buildReplicationYaml(
|
||||||
object: `${tgt.database}.{stream_table}`,
|
object: `${tgt.database}.{stream_table}`,
|
||||||
},
|
},
|
||||||
streams: streamsYaml,
|
streams: streamsYaml,
|
||||||
env: { SLING_THREADS: 4 },
|
// Pipeline-level env is exported as process env vars by sling. User
|
||||||
|
// values override the built-in defaults (e.g. SLING_THREADS).
|
||||||
|
env: { SLING_THREADS: "4", ...parseJsonObject(pipeline.env) },
|
||||||
};
|
};
|
||||||
|
|
||||||
const dir = path.join(DATA_DIR, "replications");
|
const dir = path.join(DATA_DIR, "replications");
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue