mirror of
https://github.com/wahyd4/heygo.git
synced 2026-08-09 05:15:52 +10:00
436 lines
15 KiB
TypeScript
436 lines
15 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/**
|
|
* D1 import script for Heygo shortlinks.
|
|
*
|
|
* Reads the JSONL produced by scripts/export-django-links.py (Task 13) and
|
|
* imports each link into the Heygo D1 `links` table as a **private** link
|
|
* owned by the given admin user. Reuses the Worker's own alias and URL
|
|
* template validation so imported data matches the API's rules exactly.
|
|
*
|
|
* Usage:
|
|
* npm run import:links -- --file ./exports/shortlinks.jsonl \
|
|
* --owner-email admin@example.com [--local] [--report path]
|
|
*
|
|
* Flags:
|
|
* --file PATH Path to the JSONL export file (required).
|
|
* --owner-email EMAIL Admin user email to own the imported links (required).
|
|
* --local Target the local miniflare D1 database.
|
|
* Omit to target the remote Cloudflare D1 database (default).
|
|
* --report PATH Path to write import-report.json (default: ./import-report.json).
|
|
* --dry-run Validate and build the report without writing to D1.
|
|
*
|
|
* Behavior:
|
|
* - Resolves the admin user by email; fails if not found or not an admin.
|
|
* - Queries existing active private aliases for that user up front.
|
|
* - For each JSONL record:
|
|
* * validate alias via worker/lib/aliases.validateAlias
|
|
* * validate redirect target URLs are http(s) and that URL template
|
|
* parameters parse via worker/lib/templates.parseTemplateParameters
|
|
* * custom links require non-empty content_markdown
|
|
* * if the alias already exists as an active private link for this
|
|
* owner, skip it and log to the report (count: skipped)
|
|
* * otherwise insert it as scope='private', status='active',
|
|
* owner_user_id = admin user id (count: imported)
|
|
* * validation failures are counted as conflicts and logged
|
|
* - Writes import-report.json with imported/skipped/conflicts/total counts
|
|
* and per-record details.
|
|
*
|
|
* This script is intentionally not included in tsconfig.json (it uses Node
|
|
* APIs unsuitable for the Workers runtime), so it is not type-checked by
|
|
* `npm run build`. Run it with tsx.
|
|
*/
|
|
|
|
import { execFileSync } from 'node:child_process';
|
|
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
import { dirname, resolve as pathResolve, join } from 'node:path';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { tmpdir } from 'node:os';
|
|
import { pathToFileURL } from 'node:url';
|
|
|
|
import { validateAlias } from '../worker/lib/aliases';
|
|
import { parseTemplateParameters, TemplateResolutionError } from '../worker/lib/templates';
|
|
|
|
type LinkType = 'redirect' | 'custom';
|
|
|
|
type ExportRecord = {
|
|
alias: string;
|
|
link_type: string;
|
|
target_url: string | null;
|
|
content_markdown: string | null;
|
|
description: string | null;
|
|
tags?: string[];
|
|
created_at?: string | null;
|
|
updated_at?: string | null;
|
|
};
|
|
|
|
type OwnerRow = { id: string; email: string | null; role: 'user' | 'admin' };
|
|
|
|
type Report = {
|
|
file: string;
|
|
owner_email: string;
|
|
mode: 'local' | 'remote';
|
|
dry_run: boolean;
|
|
owner_user_id: string | null;
|
|
total: number;
|
|
imported: number;
|
|
skipped: number;
|
|
conflicts: number;
|
|
skipped_aliases: string[];
|
|
conflict_details: Array<{ alias: string; reason: string }>;
|
|
imported_aliases: string[];
|
|
generated_at: string;
|
|
};
|
|
|
|
const DEFAULT_REPORT_PATH = 'import-report.json';
|
|
|
|
function stderr(msg: string): void {
|
|
process.stderr.write(`${msg}\n`);
|
|
}
|
|
|
|
function parseArgs(argv: string[]): {
|
|
file: string;
|
|
ownerEmail: string;
|
|
local: boolean;
|
|
reportPath: string;
|
|
dryRun: boolean;
|
|
} {
|
|
const opts = {
|
|
file: '',
|
|
ownerEmail: '',
|
|
local: false,
|
|
reportPath: DEFAULT_REPORT_PATH,
|
|
dryRun: false,
|
|
};
|
|
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
switch (a) {
|
|
case '--file':
|
|
opts.file = argv[++i] ?? '';
|
|
break;
|
|
case '--owner-email':
|
|
opts.ownerEmail = argv[++i] ?? '';
|
|
break;
|
|
case '--local':
|
|
opts.local = true;
|
|
break;
|
|
case '--report':
|
|
opts.reportPath = argv[++i] ?? DEFAULT_REPORT_PATH;
|
|
break;
|
|
case '--dry-run':
|
|
opts.dryRun = true;
|
|
break;
|
|
case '--help':
|
|
case '-h':
|
|
printHelp();
|
|
process.exit(0);
|
|
default:
|
|
stderr(`error: unknown argument ${a}`);
|
|
printHelp();
|
|
process.exit(2);
|
|
}
|
|
}
|
|
|
|
if (!opts.file) {
|
|
stderr('error: --file is required');
|
|
printHelp();
|
|
process.exit(2);
|
|
}
|
|
if (!opts.ownerEmail) {
|
|
stderr('error: --owner-email is required');
|
|
printHelp();
|
|
process.exit(2);
|
|
}
|
|
|
|
return opts;
|
|
}
|
|
|
|
function printHelp(): void {
|
|
const help = [
|
|
'Usage: import-links-to-d1 --file <jsonl> --owner-email <email> [options]',
|
|
'',
|
|
'Options:',
|
|
' --file PATH Path to the JSONL export file (required).',
|
|
' --owner-email EMAIL Admin user email that will own the imported links (required).',
|
|
' --local Target the local miniflare D1 database.',
|
|
' Omit to target the remote Cloudflare D1 database (default).',
|
|
' --report PATH Path to write import-report.json (default: ./import-report.json).',
|
|
' --dry-run Validate and report without writing to D1.',
|
|
' -h, --help Show this help and exit.',
|
|
'',
|
|
'Environment:',
|
|
' HEYGO_WRANGLER_BIN Override the wrangler binary path (default: ./node_modules/.bin/wrangler).',
|
|
].join('\n');
|
|
process.stdout.write(`${help}\n`);
|
|
}
|
|
|
|
function wranglerBin(): string {
|
|
const override = process.env.HEYGO_WRANGLER_BIN;
|
|
if (override) return override;
|
|
return pathResolve(process.cwd(), 'node_modules/.bin/wrangler');
|
|
}
|
|
|
|
type WranglerD1Result<T> = { results: T[]; success: boolean; meta: Record<string, unknown> };
|
|
|
|
function runD1Command<T>(sql: string, local: boolean, dryRun: boolean): WranglerD1Result<T> {
|
|
if (dryRun) {
|
|
return { results: [] as T[], success: true, meta: {} };
|
|
}
|
|
const flag = local ? '--local' : '--remote';
|
|
const out = execFileSync(wranglerBin(), ['d1', 'execute', 'DB', flag, '--json', '--command', sql], {
|
|
encoding: 'utf8',
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
cwd: process.cwd(),
|
|
});
|
|
const parsed = JSON.parse(out) as WranglerD1Result<T>[];
|
|
if (!parsed || parsed.length === 0) {
|
|
return { results: [] as T[], success: false, meta: {} };
|
|
}
|
|
return parsed[0];
|
|
}
|
|
|
|
function runD1File(local: boolean, dryRun: boolean, sqlFile: string): WranglerD1Result<unknown> {
|
|
if (dryRun) {
|
|
return { results: [] as unknown[], success: true, meta: {} };
|
|
}
|
|
const flag = local ? '--local' : '--remote';
|
|
const out = execFileSync(
|
|
wranglerBin(),
|
|
['d1', 'execute', 'DB', flag, '--file', sqlFile],
|
|
{
|
|
encoding: 'utf8',
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
cwd: process.cwd(),
|
|
},
|
|
);
|
|
// wrangler d1 execute --file does not emit JSON by default; return a stub.
|
|
return { results: [] as unknown[], success: !out.toLowerCase().includes('error'), meta: {} };
|
|
}
|
|
|
|
function sqlStringLiteral(value: string | null): string {
|
|
if (value === null) return 'NULL';
|
|
// Escape single quotes for a SQLite string literal.
|
|
return `'${value.replace(/'/g, "''")}'`;
|
|
}
|
|
|
|
function isHttpUrl(value: string): boolean {
|
|
try {
|
|
const url = new URL(value);
|
|
return url.protocol === 'http:' || url.protocol === 'https:';
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function resolveOwner(email: string, local: boolean, dryRun: boolean): OwnerRow {
|
|
const escaped = email.replace(/'/g, "''");
|
|
const sql = `SELECT id, email, role FROM users WHERE email='${escaped.toLowerCase()}' LIMIT 1`;
|
|
const result = runD1Command<OwnerRow>(sql, local, dryRun);
|
|
const row = result.results?.[0] ?? null;
|
|
if (!row) {
|
|
stderr(`error: no user found with email ${email}. Create the admin user in D1 first.`);
|
|
process.exit(1);
|
|
}
|
|
if (row.role !== 'admin') {
|
|
stderr(`error: user ${email} has role '${row.role}', but --owner-email requires an admin user.`);
|
|
process.exit(1);
|
|
}
|
|
return row;
|
|
}
|
|
|
|
function existingActiveAliases(ownerUserId: string, local: boolean, dryRun: boolean): Set<string> {
|
|
const escaped = ownerUserId.replace(/'/g, "''");
|
|
const sql = `SELECT alias FROM links WHERE scope='private' AND status='active' AND owner_user_id='${escaped}'`;
|
|
const result = runD1Command<{ alias: string }>(sql, local, dryRun);
|
|
return new Set<string>(result.results?.map((r) => r.alias) ?? []);
|
|
}
|
|
|
|
function validateRecord(rec: ExportRecord): { ok: true; alias: string; linkType: LinkType; targetUrl: string | null; contentMarkdown: string | null; description: string | null } | { ok: false; reason: string } {
|
|
if (rec.alias == null) {
|
|
return { ok: false, reason: 'record has no alias' };
|
|
}
|
|
const aliasValidation = validateAlias(rec.alias);
|
|
if (!aliasValidation.ok) {
|
|
return { ok: false, reason: aliasValidation.error };
|
|
}
|
|
const alias = aliasValidation.value;
|
|
|
|
const rawType = String(rec.link_type ?? '').toLowerCase();
|
|
if (rawType !== 'redirect' && rawType !== 'custom') {
|
|
return { ok: false, reason: `unknown link_type '${rec.link_type}'` };
|
|
}
|
|
|
|
const description = rec.description ?? null;
|
|
|
|
if (rawType === 'custom') {
|
|
const content = rec.content_markdown ?? '';
|
|
if (content.trim().length === 0) {
|
|
return { ok: false, reason: 'custom link has empty content_markdown' };
|
|
}
|
|
return { ok: true, alias, linkType: 'custom', targetUrl: null, contentMarkdown: content, description };
|
|
}
|
|
|
|
// redirect
|
|
const targetUrl = rec.target_url ?? '';
|
|
if (targetUrl.length === 0) {
|
|
return { ok: false, reason: 'redirect link has no target_url' };
|
|
}
|
|
if (!isHttpUrl(targetUrl)) {
|
|
return { ok: false, reason: `target_url is not an http(s) URL: ${targetUrl}` };
|
|
}
|
|
try {
|
|
parseTemplateParameters(targetUrl);
|
|
} catch (error) {
|
|
const reason = error instanceof TemplateResolutionError
|
|
? error.message
|
|
: `template parameter validation failed for ${targetUrl}`;
|
|
return { ok: false, reason };
|
|
}
|
|
|
|
return { ok: true, alias, linkType: 'redirect', targetUrl, contentMarkdown: null, description };
|
|
}
|
|
|
|
function readJsonl(file: string): ExportRecord[] {
|
|
if (!existsSync(file)) {
|
|
stderr(`error: --file not found: ${file}`);
|
|
process.exit(2);
|
|
}
|
|
const text = readFileSync(file, 'utf8');
|
|
const records: ExportRecord[] = [];
|
|
const lines = text.split(/\r?\n/);
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const line = lines[i];
|
|
if (line.trim().length === 0) continue;
|
|
let obj: unknown;
|
|
try {
|
|
obj = JSON.parse(line);
|
|
} catch (e) {
|
|
const msg = e instanceof Error ? e.message : String(e);
|
|
stderr(`error: line ${i + 1} is not valid JSON: ${msg}`);
|
|
process.exit(2);
|
|
}
|
|
records.push(obj as ExportRecord);
|
|
}
|
|
return records;
|
|
}
|
|
|
|
function buildInsertSqlFile(imports: Array<{ id: string; alias: string; ownerUserId: string; linkType: LinkType; targetUrl: string | null; contentMarkdown: string | null; description: string | null }>): string {
|
|
const lines: string[] = [];
|
|
for (const imp of imports) {
|
|
// Inline literals because wrangler d1 execute --file has no bind params.
|
|
lines.push(`INSERT INTO links (id, scope, owner_user_id, alias, link_type, target_url, content_markdown, description, status, click_count) VALUES (${sqlStringLiteral(imp.id)}, 'private', ${sqlStringLiteral(imp.ownerUserId)}, ${sqlStringLiteral(imp.alias)}, ${sqlStringLiteral(imp.linkType)}, ${sqlStringLiteral(imp.targetUrl)}, ${sqlStringLiteral(imp.contentMarkdown)}, ${sqlStringLiteral(imp.description)}, 'active', 0);`);
|
|
}
|
|
lines.push('COMMIT;');
|
|
return lines.join('\n') + '\n';
|
|
}
|
|
|
|
function writeReport(reportPath: string, report: Report): void {
|
|
const abs = pathResolve(process.cwd(), reportPath);
|
|
mkdirSync(dirname(abs), { recursive: true });
|
|
writeFileSync(abs, JSON.stringify(report, null, 2) + '\n', 'utf8');
|
|
stderr(`wrote report to ${abs}`);
|
|
}
|
|
|
|
function main(): number {
|
|
const opts = parseArgs(process.argv.slice(2));
|
|
const file = pathResolve(process.cwd(), opts.file);
|
|
const mode: 'local' | 'remote' = opts.local ? 'local' : 'remote';
|
|
stderr(`import mode: ${mode}${opts.dryRun ? ' (dry-run)' : ''}`);
|
|
|
|
const owner = resolveOwner(opts.ownerEmail, opts.local, opts.dryRun);
|
|
stderr(`owner: ${owner.email} (id=${owner.id}, role=${owner.role})`);
|
|
|
|
const existing = existingActiveAliases(owner.id, opts.local, opts.dryRun);
|
|
stderr(`existing active private aliases for owner: ${existing.size}`);
|
|
|
|
const records = readJsonl(file);
|
|
stderr(`read ${records.length} record(s) from ${file}`);
|
|
|
|
const report: Report = {
|
|
file,
|
|
owner_email: opts.ownerEmail,
|
|
mode,
|
|
dry_run: opts.dryRun,
|
|
owner_user_id: owner.id,
|
|
total: records.length,
|
|
imported: 0,
|
|
skipped: 0,
|
|
conflicts: 0,
|
|
skipped_aliases: [],
|
|
conflict_details: [],
|
|
imported_aliases: [],
|
|
generated_at: new Date().toISOString(),
|
|
};
|
|
|
|
const imports: Array<{ id: string; alias: string; ownerUserId: string; linkType: LinkType; targetUrl: string | null; contentMarkdown: string | null; description: string | null }> = [];
|
|
|
|
for (const rec of records) {
|
|
const validation = validateRecord(rec);
|
|
if (!validation.ok) {
|
|
const aliasName = (rec.alias ?? '<none>').toString();
|
|
report.conflicts += 1;
|
|
report.conflict_details.push({ alias: aliasName, reason: validation.reason });
|
|
stderr(`conflict: '${aliasName}' skipped: ${validation.reason}`);
|
|
continue;
|
|
}
|
|
|
|
if (existing.has(validation.alias)) {
|
|
report.skipped += 1;
|
|
report.skipped_aliases.push(validation.alias);
|
|
stderr(`skip: '${validation.alias}' already exists as active private link for owner`);
|
|
continue;
|
|
}
|
|
|
|
const id = randomUUID();
|
|
imports.push({
|
|
id,
|
|
alias: validation.alias,
|
|
ownerUserId: owner.id,
|
|
linkType: validation.linkType,
|
|
targetUrl: validation.targetUrl,
|
|
contentMarkdown: validation.contentMarkdown,
|
|
description: validation.description,
|
|
});
|
|
// Track within this run so duplicate aliases in the JSONL become skipped.
|
|
existing.add(validation.alias);
|
|
report.imported += 1;
|
|
report.imported_aliases.push(validation.alias);
|
|
}
|
|
|
|
if (imports.length > 0 && !opts.dryRun) {
|
|
const sql = buildInsertSqlFile(imports);
|
|
const sqlFile = join(tmpdir(), `heygo-import-${Date.now()}.sql`);
|
|
writeFileSync(sqlFile, sql, 'utf8');
|
|
stderr(`wrote ${imports.length} INSERT statement(s) to ${sqlFile}`);
|
|
const result = runD1File(opts.local, opts.dryRun, sqlFile);
|
|
if (!result.success) {
|
|
stderr('error: D1 insert batch failed. See wrangler output above.');
|
|
report.imported = 0;
|
|
}
|
|
}
|
|
|
|
writeReport(opts.reportPath, report);
|
|
|
|
stderr(
|
|
`done: imported=${report.imported} skipped=${report.skipped} conflicts=${report.conflicts} total=${report.total}`,
|
|
);
|
|
|
|
return 0;
|
|
}
|
|
|
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
const code = main();
|
|
process.exit(code);
|
|
}
|
|
|
|
export {
|
|
parseArgs,
|
|
validateRecord,
|
|
isHttpUrl,
|
|
sqlStringLiteral,
|
|
buildInsertSqlFile,
|
|
type ExportRecord,
|
|
type Report,
|
|
type LinkType,
|
|
};
|