feat: per-connection URL params and per-pipeline env vars

- connections: params (JSON) merged into connection URL query string, applied
  to both sling env.yaml and atlas URLs; user's foreign_key_checks overrides
  the default =0
- pipelines: env map written to replication yaml top-level env:
- UI: key=value textarea in connection dialog and pipeline form (new/edit)
- docs: TiDB Error 8048 (SERIALIZABLE) troubleshooting — set
  tidb_skip_isolation_level_check=1 as a connection param
This commit is contained in:
tigerenwork 2026-08-09 18:28:20 +08:00
parent 884eaa1ed8
commit 8779ad4867
14 changed files with 275 additions and 21 deletions

View File

@ -38,6 +38,11 @@ pnpm dev # 或 pnpm build && pnpm start
- `schema_sync` 开启时Atlas 负责目标端表结构;此时即使选择 `full-refresh`,实际执行会翻译为 `truncate`(保留 Atlas 建的表结构重灌全量数据,避免 sling drop+重建丢掉索引/约束)。 - `schema_sync` 开启时Atlas 负责目标端表结构;此时即使选择 `full-refresh`,实际执行会翻译为 `truncate`(保留 Atlas 建的表结构重灌全量数据,避免 sling drop+重建丢掉索引/约束)。
- `incremental` 需要为每张表配置 `primary_key``update_key` - `incremental` 需要为每张表配置 `primary_key``update_key`
- sling 连接 URL 自动带 `foreign_key_checks=0`(会话级),避免外键顺序问题;约束定义本身保留。 - sling 连接 URL 自动带 `foreign_key_checks=0`(会话级),避免外键顺序问题;约束定义本身保留。
- 连接支持自定义「连接参数」URL query stringsling/Atlas 均生效),流水线支持自定义「环境变量」(写入 replication 的 `env:`)。
## 已知问题
- TiDB 目标库报 `Error 8048: isolation level 'SERIALIZABLE' is not supported`:见 [docs/tidb-serializable-isolation.md](docs/tidb-serializable-isolation.md),给连接加参数 `tidb_skip_isolation_level_check=1` 即可。
## 数据与安全说明 ## 数据与安全说明

View File

@ -1,4 +1,4 @@
import { deleteConnection, updateConnection } from "@/lib/db"; import { deleteConnection, updateConnection, normalizeJsonColumn } from "@/lib/db";
export const runtime = "nodejs"; export const runtime = "nodejs";
export const dynamic = "force-dynamic"; export const dynamic = "force-dynamic";
@ -9,6 +9,9 @@ export async function PUT(request: Request, { params }: Ctx) {
try { try {
const { id } = await params; const { id } = await params;
const body = await request.json(); const body = await request.json();
if (body.params !== undefined) {
body.params = normalizeJsonColumn(body.params);
}
const conn = updateConnection(Number(id), body); const conn = updateConnection(Number(id), body);
if (!conn) { if (!conn) {
return Response.json({ error: "connection not found" }, { status: 404 }); return Response.json({ error: "connection not found" }, { status: 404 });

View File

@ -1,4 +1,4 @@
import { createConnection, listConnections } from "@/lib/db"; import { createConnection, listConnections, normalizeJsonColumn } from "@/lib/db";
export const runtime = "nodejs"; export const runtime = "nodejs";
export const dynamic = "force-dynamic"; export const dynamic = "force-dynamic";
@ -28,6 +28,7 @@ export async function POST(request: Request) {
user: String(body.user ?? ""), user: String(body.user ?? ""),
password: String(body.password ?? ""), password: String(body.password ?? ""),
database: String(body.database ?? ""), database: String(body.database ?? ""),
params: normalizeJsonColumn(body.params),
}); });
return Response.json(conn, { status: 201 }); return Response.json(conn, { status: 201 });
} catch (err) { } catch (err) {

View File

@ -1,4 +1,4 @@
import { deletePipeline, getPipeline, updatePipeline } from "@/lib/db"; import { deletePipeline, getPipeline, updatePipeline, normalizeJsonColumn } from "@/lib/db";
export const runtime = "nodejs"; export const runtime = "nodejs";
export const dynamic = "force-dynamic"; export const dynamic = "force-dynamic";
@ -42,6 +42,9 @@ export async function PUT(request: Request, { params }: Ctx) {
if (body.streams !== undefined) { if (body.streams !== undefined) {
patch.streams = normalizeStreams(body.streams); patch.streams = normalizeStreams(body.streams);
} }
if (body.env !== undefined) {
patch.env = normalizeJsonColumn(body.env);
}
const pipeline = updatePipeline(Number(id), patch); const pipeline = updatePipeline(Number(id), patch);
if (!pipeline) { if (!pipeline) {
return Response.json({ error: "pipeline not found" }, { status: 404 }); return Response.json({ error: "pipeline not found" }, { status: 404 });

View File

@ -1,4 +1,4 @@
import { createPipeline, listPipelines } from "@/lib/db"; import { createPipeline, listPipelines, normalizeJsonColumn } from "@/lib/db";
export const runtime = "nodejs"; export const runtime = "nodejs";
export const dynamic = "force-dynamic"; export const dynamic = "force-dynamic";
@ -34,6 +34,7 @@ export async function POST(request: Request) {
include_fk: body.include_fk === undefined ? 0 : Number(body.include_fk), include_fk: body.include_fk === undefined ? 0 : Number(body.include_fk),
mode: body.mode ? String(body.mode) : "full-refresh", mode: body.mode ? String(body.mode) : "full-refresh",
streams: normalizeStreams(body.streams), streams: normalizeStreams(body.streams),
env: normalizeJsonColumn(body.env),
}); });
return Response.json(pipeline, { status: 201 }); return Response.json(pipeline, { status: 201 });
} catch (err) { } catch (err) {

View File

@ -117,7 +117,18 @@ export default function ConnectionsPage() {
{list.map((c) => ( {list.map((c) => (
<Fragment key={c.id}> <Fragment key={c.id}>
<TableRow> <TableRow>
<TableCell className="font-medium">{c.name}</TableCell> <TableCell className="font-medium">
{c.name}
{c.params && c.params !== "{}" ? (
<div className="mt-0.5 font-mono text-xs text-muted-foreground">
{Object.entries(
JSON.parse(c.params) as Record<string, string>
)
.map(([k, v]) => `${k}=${v}`)
.join(" ")}
</div>
) : null}
</TableCell>
<TableCell className="font-mono text-xs">{c.type}</TableCell> <TableCell className="font-mono text-xs">{c.type}</TableCell>
<TableCell className="font-mono text-xs"> <TableCell className="font-mono text-xs">
{c.host}:{c.port} {c.host}:{c.port}

View File

@ -91,6 +91,22 @@ export default function PipelineDetailPage() {
} }
}, [pipeline]); }, [pipeline]);
const envEntries = useMemo<[string, string][]>(() => {
if (!pipeline) return [];
try {
const v = JSON.parse(pipeline.env || "{}") as unknown;
if (v && typeof v === "object" && !Array.isArray(v)) {
return Object.entries(v as Record<string, unknown>).map(([k, val]) => [
k,
String(val),
]);
}
} catch {
// fall through
}
return [];
}, [pipeline]);
const latestRun = runs[0] ?? null; const latestRun = runs[0] ?? null;
async function runNow() { async function runNow() {
@ -203,6 +219,14 @@ export default function PipelineDetailPage() {
<dt className="text-muted-foreground"></dt> <dt className="text-muted-foreground"></dt>
<dd className="font-medium">{pipeline.include_fk ? "是" : "否"}</dd> <dd className="font-medium">{pipeline.include_fk ? "是" : "否"}</dd>
</div> </div>
{envEntries.length > 0 ? (
<div>
<dt className="text-muted-foreground"></dt>
<dd className="font-mono text-xs">
{envEntries.map(([k, v]) => `${k}=${v}`).join(" ")}
</dd>
</div>
) : null}
<div> <div>
<dt className="text-muted-foreground"></dt> <dt className="text-muted-foreground"></dt>
<dd>{fmtTime(pipeline.created_at)}</dd> <dd>{fmtTime(pipeline.created_at)}</dd>

View File

@ -3,6 +3,7 @@
import { useState } from "react"; import { useState } from "react";
import { toast } from "sonner"; import { toast } from "sonner";
import { api, type Connection } from "@/components/api"; import { api, type Connection } from "@/components/api";
import { keyValueLinesFromJson, parseKeyValueLines } from "@/components/kv";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { import {
Dialog, Dialog,
@ -20,6 +21,7 @@ import {
SelectTrigger, SelectTrigger,
SelectValue, SelectValue,
} from "@/components/ui/select"; } from "@/components/ui/select";
import { Textarea } from "@/components/ui/textarea";
interface ConnectionDialogProps { interface ConnectionDialogProps {
open: boolean; open: boolean;
@ -59,6 +61,9 @@ export function ConnectionDialog({
: EMPTY : EMPTY
); );
const [saving, setSaving] = useState(false); const [saving, setSaving] = useState(false);
const [paramsText, setParamsText] = useState(() =>
initial ? keyValueLinesFromJson(initial.params) : ""
);
const set = (k: keyof typeof EMPTY) => (e: React.ChangeEvent<HTMLInputElement>) => const set = (k: keyof typeof EMPTY) => (e: React.ChangeEvent<HTMLInputElement>) =>
setForm((f) => ({ ...f, [k]: e.target.value })); setForm((f) => ({ ...f, [k]: e.target.value }));
@ -79,6 +84,7 @@ export function ConnectionDialog({
user: form.user.trim(), user: form.user.trim(),
password: form.password, password: form.password,
database: form.database.trim(), database: form.database.trim(),
params: parseKeyValueLines(paramsText),
}); });
if (initial) { if (initial) {
await api(`/api/connections/${initial.id}`, { await api(`/api/connections/${initial.id}`, {
@ -174,6 +180,21 @@ export function ConnectionDialog({
onChange={set("database")} onChange={set("database")}
/> />
</div> </div>
<div className="grid gap-1.5">
<Label htmlFor="conn-params"></Label>
<Textarea
id="conn-params"
className="font-mono text-xs"
rows={3}
value={paramsText}
onChange={(e) => setParamsText(e.target.value)}
placeholder={"tidb_skip_isolation_level_check=1"}
/>
<p className="text-xs text-muted-foreground">
key=value URL sling / atlas TiDB
tidb_skip_isolation_level_check=1
</p>
</div>
</div> </div>
<DialogFooter> <DialogFooter>
<Button variant="outline" onClick={() => onOpenChange(false)} disabled={saving}> <Button variant="outline" onClick={() => onOpenChange(false)} disabled={saving}>

34
components/kv.ts Normal file
View File

@ -0,0 +1,34 @@
// Helpers for the "one key=value per line" textareas used by the connection
// dialog (URL params) and the pipeline form (replication env).
// Parse textarea lines into an object. Blank lines, lines starting with '#'
// and lines without a key or '=' are ignored. Values may contain '='.
export function parseKeyValueLines(text: string): Record<string, string> {
const out: Record<string, string> = {};
for (const raw of text.split("\n")) {
const line = raw.trim();
if (!line || line.startsWith("#")) continue;
const i = line.indexOf("=");
if (i <= 0) continue;
const key = line.slice(0, i).trim();
const value = line.slice(i + 1).trim();
if (key) out[key] = value;
}
return out;
}
// Serialize a JSON-object column back into key=value lines for editing.
export function keyValueLinesFromJson(json: string | undefined | null): string {
if (!json) return "";
try {
const v = JSON.parse(json) as unknown;
if (v && typeof v === "object" && !Array.isArray(v)) {
return Object.entries(v as Record<string, unknown>)
.map(([k, val]) => `${k}=${String(val)}`)
.join("\n");
}
} catch {
// fall through
}
return "";
}

View File

@ -10,6 +10,7 @@ import {
type StreamConfig, type StreamConfig,
type StreamInfo, type StreamInfo,
} from "@/components/api"; } from "@/components/api";
import { keyValueLinesFromJson, parseKeyValueLines } from "@/components/kv";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { import {
Card, Card,
@ -29,6 +30,7 @@ import {
SelectValue, SelectValue,
} from "@/components/ui/select"; } from "@/components/ui/select";
import { Switch } from "@/components/ui/switch"; import { Switch } from "@/components/ui/switch";
import { Textarea } from "@/components/ui/textarea";
const MODES = [ const MODES = [
{ value: "full-refresh", label: "full-refresh全量覆盖默认" }, { value: "full-refresh", label: "full-refresh全量覆盖默认" },
@ -58,6 +60,9 @@ export function PipelineForm({ initial }: { initial?: Pipeline }) {
initial?.schema_scope ?? "selected" initial?.schema_scope ?? "selected"
); );
const [includeFk, setIncludeFk] = useState(initial ? !!initial.include_fk : false); const [includeFk, setIncludeFk] = useState(initial ? !!initial.include_fk : false);
const [envText, setEnvText] = useState(() =>
initial ? keyValueLinesFromJson(initial.env) : ""
);
const [streamList, setStreamList] = useState<StreamInfo[] | null>(null); const [streamList, setStreamList] = useState<StreamInfo[] | null>(null);
const [streamsLoading, setStreamsLoading] = useState(false); const [streamsLoading, setStreamsLoading] = useState(false);
@ -149,6 +154,7 @@ export function PipelineForm({ initial }: { initial?: Pipeline }) {
include_fk: includeFk ? 1 : 0, include_fk: includeFk ? 1 : 0,
mode, mode,
streams: JSON.stringify(selectedList), streams: JSON.stringify(selectedList),
env: parseKeyValueLines(envText),
}; };
try { try {
const p = isEdit const p = isEdit
@ -301,6 +307,21 @@ export function PipelineForm({ initial }: { initial?: Pipeline }) {
disabled={!schemaSync} disabled={!schemaSync}
/> />
</div> </div>
<div className="grid gap-1.5">
<Label htmlFor="p-env"></Label>
<Textarea
id="p-env"
className="font-mono text-xs"
rows={3}
value={envText}
onChange={(e) => setEnvText(e.target.value)}
placeholder={"SLING_THREADS=4"}
/>
<p className="text-xs text-muted-foreground">
key=value sling replication envsling
SLING_THREADS=4
</p>
</div>
</CardContent> </CardContent>
</Card> </Card>

View File

@ -0,0 +1,53 @@
# TiDB 目标库Error 8048 isolation level 'SERIALIZABLE' is not supported
## 现象
目标库为 TiDB 时sling 数据迁移在每个 stream 的写入阶段失败:
```
writing to target database [mode: truncate]
created table `test_demo`.`crm_orders_tmp`
execution failed
Error 8048 (HY000): The isolation level 'SERIALIZABLE' is not supported.
Set tidb_skip_isolation_level_check=1 to skip this error
```
## 原因
TiDB 不支持 `SERIALIZABLE` 隔离级别TiDB 默认使用乐观锁/快照隔离。sling
在目标端开启事务时请求了 `SERIALIZABLE`TiDB 默认会直接报错拒绝。
TiDB 提供了兼容开关 `tidb_skip_isolation_level_check=1`:设置后跳过隔离级别
检查,按 TiDB 实际支持的级别执行。对"测试环境 → 本地 dev 库"的全量数据灌入
场景,这是安全的。
## 解决:连接参数
在 sling-ui 中编辑 TiDB 目标连接,在「连接参数」中加一行:
```
tidb_skip_isolation_level_check=1
```
### 工作原理
连接参数会被拼进连接 URL 的 query string例如
```
mysql://user:pass@tidb-host:4000/dbname?tidb_skip_isolation_level_check=1
```
sling 底层使用 go-sql-driver/mysql会把未识别的 DSN 参数作为 session 变量
在连接建立时执行 `SET <key>=<value>`(本项目内置的 `foreign_key_checks=0`
也是同一机制。Atlas 的连接 URL 同样会带上这些参数。
已实测验证:对普通 MySQL 设置该参数会报 `Error 1193: Unknown system
variable`,证明参数确实被 SET 为 session 变量TiDB 上该变量存在,即达到
预期效果。
## 备注
- 该参数只对 TiDB 有效,不要给普通 MySQL/MariaDB 连接设置。
- 若希望对整个 TiDB 实例生效(所有客户端),也可以在 TiDB 上执行
`SET GLOBAL tidb_skip_isolation_level_check=1;`,但用连接参数的方式
影响面最小,推荐。

View File

@ -1,6 +1,6 @@
import { execFile } from "child_process"; import { execFile } from "child_process";
import { Connection, Pipeline, StreamConfig, getConnection } from "./db"; import { Connection, Pipeline, StreamConfig, getConnection } from "./db";
import { connectionUrl, discoverStreams, execSql } from "./sling"; import { appendParams, connectionUrl, discoverStreams, execSql } from "./sling";
import { resolveBinary } from "./binaries"; import { resolveBinary } from "./binaries";
// Atlas needs an empty scratch database ("dev database") to plan diffs. // Atlas needs an empty scratch database ("dev database") to plan diffs.
@ -10,10 +10,19 @@ import { resolveBinary } from "./binaries";
// scratch schema on the target server instead — no Docker needed. // scratch schema on the target server instead — no Docker needed.
const DEV_SCHEMA = "sling_ui_atlas_dev"; const DEV_SCHEMA = "sling_ui_atlas_dev";
// The scratch schema lives on the target server, so it inherits the target
// connection's URL params (e.g. tidb_skip_isolation_level_check for TiDB).
function devDatabaseUrl(tgt: Connection): string { function devDatabaseUrl(tgt: Connection): string {
const user = encodeURIComponent(tgt.user); const user = encodeURIComponent(tgt.user);
const pass = encodeURIComponent(tgt.password); const pass = encodeURIComponent(tgt.password);
return `mysql://${user}:${pass}@${tgt.host}:${tgt.port}/${DEV_SCHEMA}`; const base = `mysql://${user}:${pass}@${tgt.host}:${tgt.port}/${DEV_SCHEMA}`;
let params: Record<string, string> = {};
try {
params = JSON.parse(tgt.params || "{}") as Record<string, string>;
} catch {
// ignore malformed params
}
return appendParams(base, params);
} }
// Create the scratch schema if missing and return its URL. Atlas keeps it // Create the scratch schema if missing and return its URL. Atlas keeps it

View File

@ -14,6 +14,7 @@ export interface Connection {
user: string; user: string;
password: string; password: string;
database: string; database: string;
params: string; // JSON object: URL query params, e.g. {"tidb_skip_isolation_level_check":"1"}
created_at: string; created_at: string;
} }
@ -25,6 +26,7 @@ export interface ConnectionInput {
user: string; user: string;
password: string; password: string;
database: string; database: string;
params?: string;
} }
export interface StreamConfig { export interface StreamConfig {
@ -45,6 +47,7 @@ export interface Pipeline {
include_fk: number; // 1 | 0 include_fk: number; // 1 | 0
mode: string; mode: string;
streams: string; // JSON: StreamConfig[] streams: string; // JSON: StreamConfig[]
env: string; // JSON object: sling replication env, e.g. {"SLING_THREADS":"4"}
created_at: string; created_at: string;
updated_at: string; updated_at: string;
} }
@ -58,10 +61,18 @@ export interface PipelineInput {
include_fk?: number; include_fk?: number;
mode?: string; mode?: string;
streams?: string; streams?: string;
env?: string;
} }
export type RunStatus = "queued" | "running" | "success" | "failed"; export type RunStatus = "queued" | "running" | "success" | "failed";
// Normalize a JSON-object column value from an API body: accept either a
// plain object or a pre-serialized JSON string.
export function normalizeJsonColumn(raw: unknown): string {
if (typeof raw === "string") return raw;
return JSON.stringify(raw ?? {});
}
export interface Run { export interface Run {
id: number; id: number;
pipeline_id: number; pipeline_id: number;
@ -87,6 +98,7 @@ function createDb(): Database.Database {
user TEXT NOT NULL, user TEXT NOT NULL,
password TEXT NOT NULL DEFAULT '', password TEXT NOT NULL DEFAULT '',
database TEXT NOT NULL, database TEXT NOT NULL,
params TEXT NOT NULL DEFAULT '{}',
created_at TEXT DEFAULT (datetime('now')) created_at TEXT DEFAULT (datetime('now'))
); );
CREATE TABLE IF NOT EXISTS pipelines ( CREATE TABLE IF NOT EXISTS pipelines (
@ -99,6 +111,7 @@ function createDb(): Database.Database {
include_fk INTEGER NOT NULL DEFAULT 0, include_fk INTEGER NOT NULL DEFAULT 0,
mode TEXT DEFAULT 'full-refresh', mode TEXT DEFAULT 'full-refresh',
streams TEXT NOT NULL DEFAULT '[]', streams TEXT NOT NULL DEFAULT '[]',
env TEXT NOT NULL DEFAULT '{}',
created_at TEXT DEFAULT (datetime('now')), created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')) updated_at TEXT DEFAULT (datetime('now'))
); );
@ -116,9 +129,9 @@ function createDb(): Database.Database {
// Lightweight migrations for databases created before these columns existed. // Lightweight migrations for databases created before these columns existed.
// Several Next.js workers may open the DB concurrently during build, so a // Several Next.js workers may open the DB concurrently during build, so a
// racing ALTER can fail with "duplicate column name" — safe to ignore. // racing ALTER can fail with "duplicate column name" — safe to ignore.
const addColumnIfMissing = (sql: string, column: string) => { const addColumnIfMissing = (table: string, sql: string, column: string) => {
const cols = new Set( const cols = new Set(
(db.pragma("table_info(pipelines)") as { name: string }[]).map( (db.pragma(`table_info(${table})`) as { name: string }[]).map(
(c) => c.name (c) => c.name
) )
); );
@ -130,13 +143,25 @@ function createDb(): Database.Database {
} }
}; };
addColumnIfMissing( addColumnIfMissing(
"pipelines",
"ALTER TABLE pipelines ADD COLUMN schema_scope TEXT NOT NULL DEFAULT 'selected'", "ALTER TABLE pipelines ADD COLUMN schema_scope TEXT NOT NULL DEFAULT 'selected'",
"schema_scope" "schema_scope"
); );
addColumnIfMissing( addColumnIfMissing(
"pipelines",
"ALTER TABLE pipelines ADD COLUMN include_fk INTEGER NOT NULL DEFAULT 0", "ALTER TABLE pipelines ADD COLUMN include_fk INTEGER NOT NULL DEFAULT 0",
"include_fk" "include_fk"
); );
addColumnIfMissing(
"connections",
"ALTER TABLE connections ADD COLUMN params TEXT NOT NULL DEFAULT '{}'",
"params"
);
addColumnIfMissing(
"pipelines",
"ALTER TABLE pipelines ADD COLUMN env TEXT NOT NULL DEFAULT '{}'",
"env"
);
return db; return db;
} }
@ -164,8 +189,8 @@ export function getConnection(id: number): Connection | null {
export function createConnection(input: ConnectionInput): Connection { export function createConnection(input: ConnectionInput): Connection {
const res = db const res = db
.prepare( .prepare(
`INSERT INTO connections (name, type, host, port, user, password, database) `INSERT INTO connections (name, type, host, port, user, password, database, params)
VALUES (@name, @type, @host, @port, @user, @password, @database)` VALUES (@name, @type, @host, @port, @user, @password, @database, @params)`
) )
.run({ .run({
name: input.name, name: input.name,
@ -175,6 +200,7 @@ export function createConnection(input: ConnectionInput): Connection {
user: input.user, user: input.user,
password: input.password ?? "", password: input.password ?? "",
database: input.database, database: input.database,
params: input.params ?? "{}",
}); });
return getConnection(Number(res.lastInsertRowid))!; return getConnection(Number(res.lastInsertRowid))!;
} }
@ -183,7 +209,7 @@ export function updateConnection(
id: number, id: number,
input: Partial<ConnectionInput> input: Partial<ConnectionInput>
): Connection | null { ): Connection | null {
const fields = ["name", "type", "host", "port", "user", "password", "database"].filter( const fields = ["name", "type", "host", "port", "user", "password", "database", "params"].filter(
(k) => input[k as keyof ConnectionInput] !== undefined (k) => input[k as keyof ConnectionInput] !== undefined
); );
if (fields.length === 0) return getConnection(id); if (fields.length === 0) return getConnection(id);
@ -216,8 +242,8 @@ export function getPipeline(id: number): Pipeline | null {
export function createPipeline(input: PipelineInput): Pipeline { export function createPipeline(input: PipelineInput): Pipeline {
const res = db const res = db
.prepare( .prepare(
`INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, schema_scope, include_fk, mode, streams) `INSERT INTO pipelines (name, source_conn_id, target_conn_id, schema_sync, schema_scope, include_fk, mode, streams, env)
VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @schema_scope, @include_fk, @mode, @streams)` VALUES (@name, @source_conn_id, @target_conn_id, @schema_sync, @schema_scope, @include_fk, @mode, @streams, @env)`
) )
.run({ .run({
name: input.name, name: input.name,
@ -228,6 +254,7 @@ export function createPipeline(input: PipelineInput): Pipeline {
include_fk: input.include_fk ?? 0, include_fk: input.include_fk ?? 0,
mode: input.mode ?? "full-refresh", mode: input.mode ?? "full-refresh",
streams: input.streams ?? "[]", streams: input.streams ?? "[]",
env: input.env ?? "{}",
}); });
return getPipeline(Number(res.lastInsertRowid))!; return getPipeline(Number(res.lastInsertRowid))!;
} }
@ -245,6 +272,7 @@ export function updatePipeline(
"include_fk", "include_fk",
"mode", "mode",
"streams", "streams",
"env",
].filter((k) => input[k as keyof PipelineInput] !== undefined); ].filter((k) => input[k as keyof PipelineInput] !== undefined);
if (fields.length === 0) return getPipeline(id); if (fields.length === 0) return getPipeline(id);
const sets = fields.map((f) => `${f} = @${f}`).join(", "); const sets = fields.map((f) => `${f} = @${f}`).join(", ");

View File

@ -19,24 +19,62 @@ export interface DiscoveredStream {
columns?: string[]; columns?: string[];
} }
// Build a mysql:// URL from a connection record, escaping credentials. // Parse a connection/pipeline JSON-object column; tolerate bad data.
function parseJsonObject(json: string | undefined | null): Record<string, string> {
if (!json) return {};
try {
const v = JSON.parse(json) as unknown;
if (v && typeof v === "object" && !Array.isArray(v)) {
return Object.fromEntries(
Object.entries(v as Record<string, unknown>).map(([k, val]) => [
k,
String(val),
])
);
}
} catch {
// fall through
}
return {};
}
// Append query params to a URL, percent-encoding keys and values.
export function appendParams(url: string, params: Record<string, string>): string {
const entries = Object.entries(params);
if (entries.length === 0) return url;
const qs = entries
.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
.join("&");
return `${url}${url.includes("?") ? "&" : "?"}${qs}`;
}
// Build a mysql:// URL from a connection record, escaping credentials and
// merging the connection's extra params into the query string. These params
// reach both sling (env.yaml) and atlas (--url/--to/--dev-url); the
// go-sql-driver SETs unknown DSN params as session variables, which is how
// e.g. tidb_skip_isolation_level_check=1 takes effect.
export function connectionUrl(conn: Connection): string { export function connectionUrl(conn: Connection): string {
const user = encodeURIComponent(conn.user); const user = encodeURIComponent(conn.user);
const pass = encodeURIComponent(conn.password); const pass = encodeURIComponent(conn.password);
return `mysql://${user}:${pass}@${conn.host}:${conn.port}/${conn.database}`; const base = `mysql://${user}:${pass}@${conn.host}:${conn.port}/${conn.database}`;
return appendParams(base, parseJsonObject(conn.params));
} }
// Regenerate env.yaml from the connections table. Names are uppercased // Regenerate env.yaml from the connections table. Names are uppercased
// because sling treats connection names case-insensitively. Session-level // because sling treats connection names case-insensitively. Session-level
// foreign_key_checks=0 lets sling truncate/drop FK-referenced parent tables // foreign_key_checks=0 lets sling truncate/drop FK-referenced parent tables
// in any order (same approach as mysqldump); constraints stay defined. // in any order (same approach as mysqldump); constraints stay defined. It is
// appended only here (atlas doesn't need it) and never overrides a value the
// user set explicitly in connection params.
export function syncEnvYaml(): string { export function syncEnvYaml(): string {
fs.mkdirSync(SLING_HOME, { recursive: true }); fs.mkdirSync(SLING_HOME, { recursive: true });
const connections: Record<string, { url: string }> = {}; const connections: Record<string, { url: string }> = {};
for (const c of listConnections()) { for (const c of listConnections()) {
connections[c.name.toUpperCase()] = { let url = connectionUrl(c);
url: `${connectionUrl(c)}?foreign_key_checks=0`, if (!parseJsonObject(c.params)["foreign_key_checks"]) {
}; url = appendParams(url, { foreign_key_checks: "0" });
}
connections[c.name.toUpperCase()] = { url };
} }
const file = path.join(SLING_HOME, "env.yaml"); const file = path.join(SLING_HOME, "env.yaml");
fs.writeFileSync(file, dump({ connections })); fs.writeFileSync(file, dump({ connections }));
@ -236,7 +274,9 @@ export function buildReplicationYaml(
object: `${tgt.database}.{stream_table}`, object: `${tgt.database}.{stream_table}`,
}, },
streams: streamsYaml, streams: streamsYaml,
env: { SLING_THREADS: 4 }, // Pipeline-level env is exported as process env vars by sling. User
// values override the built-in defaults (e.g. SLING_THREADS).
env: { SLING_THREADS: "4", ...parseJsonObject(pipeline.env) },
}; };
const dir = path.join(DATA_DIR, "replications"); const dir = path.join(DATA_DIR, "replications");