From f4bf43189fb04b4e83924c6067ebfc776cfe832e Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sat, 20 Jun 2026 13:56:40 +1000 Subject: [PATCH] feat: add d1 import script --- .gitignore | 1 + package.json | 3 +- scripts/import-links-to-d1.ts | 435 ++++++++++++++++++++++++++++++++++ 3 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 scripts/import-links-to-d1.ts diff --git a/.gitignore b/.gitignore index 6bd5324..a326a3f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ dist/ !.env.example .DS_Store npm-debug.log* +/exports diff --git a/package.json b/package.json index 6003a5b..c787444 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "dev:worker": "wrangler dev", "build": "tsc --noEmit && vite build", "preview": "vite preview", - "test": "vitest run" + "test": "vitest run", + "import:links": "npx tsx scripts/import-links-to-d1.ts" }, "dependencies": { "react": "^19.2.7", diff --git a/scripts/import-links-to-d1.ts b/scripts/import-links-to-d1.ts new file mode 100644 index 0000000..dd9c1c7 --- /dev/null +++ b/scripts/import-links-to-d1.ts @@ -0,0 +1,435 @@ +#!/usr/bin/env tsx +/** + * D1 import script for Heygo shortlinks. + * + * Reads the JSONL produced by scripts/export-django-links.py (Task 13) and + * imports each link into the Heygo D1 `links` table as a **private** link + * owned by the given admin user. Reuses the Worker's own alias and URL + * template validation so imported data matches the API's rules exactly. + * + * Usage: + * npm run import:links -- --file ./exports/shortlinks.jsonl \ + * --owner-email admin@example.com [--local] [--report path] + * + * Flags: + * --file PATH Path to the JSONL export file (required). + * --owner-email EMAIL Admin user email to own the imported links (required). + * --local Target the local miniflare D1 database. + * Omit to target the remote Cloudflare D1 database (default). + * --report PATH Path to write import-report.json (default: ./import-report.json). + * --dry-run Validate and build the report without writing to D1. + * + * Behavior: + * - Resolves the admin user by email; fails if not found or not an admin. + * - Queries existing active private aliases for that user up front. + * - For each JSONL record: + * * validate alias via worker/lib/aliases.validateAlias + * * validate redirect target URLs are http(s) and that URL template + * parameters parse via worker/lib/templates.parseTemplateParameters + * * custom links require non-empty content_markdown + * * if the alias already exists as an active private link for this + * owner, skip it and log to the report (count: skipped) + * * otherwise insert it as scope='private', status='active', + * owner_user_id = admin user id (count: imported) + * * validation failures are counted as conflicts and logged + * - Writes import-report.json with imported/skipped/conflicts/total counts + * and per-record details. + * + * This script is intentionally not included in tsconfig.json (it uses Node + * APIs unsuitable for the Workers runtime), so it is not type-checked by + * `npm run build`. Run it with tsx. + */ + +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { dirname, resolve as pathResolve, join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { tmpdir } from 'node:os'; +import { pathToFileURL } from 'node:url'; + +import { validateAlias } from '../worker/lib/aliases'; +import { parseTemplateParameters, TemplateResolutionError } from '../worker/lib/templates'; + +type LinkType = 'redirect' | 'custom'; + +type ExportRecord = { + alias: string; + link_type: string; + target_url: string | null; + content_markdown: string | null; + description: string | null; + tags?: string[]; + created_at?: string | null; + updated_at?: string | null; +}; + +type OwnerRow = { id: string; email: string | null; role: 'user' | 'admin' }; + +type Report = { + file: string; + owner_email: string; + mode: 'local' | 'remote'; + dry_run: boolean; + owner_user_id: string | null; + total: number; + imported: number; + skipped: number; + conflicts: number; + skipped_aliases: string[]; + conflict_details: Array<{ alias: string; reason: string }>; + imported_aliases: string[]; + generated_at: string; +}; + +const DEFAULT_REPORT_PATH = 'import-report.json'; + +function stderr(msg: string): void { + process.stderr.write(`${msg}\n`); +} + +function parseArgs(argv: string[]): { + file: string; + ownerEmail: string; + local: boolean; + reportPath: string; + dryRun: boolean; +} { + const opts = { + file: '', + ownerEmail: '', + local: false, + reportPath: DEFAULT_REPORT_PATH, + dryRun: false, + }; + + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + switch (a) { + case '--file': + opts.file = argv[++i] ?? ''; + break; + case '--owner-email': + opts.ownerEmail = argv[++i] ?? ''; + break; + case '--local': + opts.local = true; + break; + case '--report': + opts.reportPath = argv[++i] ?? DEFAULT_REPORT_PATH; + break; + case '--dry-run': + opts.dryRun = true; + break; + case '--help': + case '-h': + printHelp(); + process.exit(0); + default: + stderr(`error: unknown argument ${a}`); + printHelp(); + process.exit(2); + } + } + + if (!opts.file) { + stderr('error: --file is required'); + printHelp(); + process.exit(2); + } + if (!opts.ownerEmail) { + stderr('error: --owner-email is required'); + printHelp(); + process.exit(2); + } + + return opts; +} + +function printHelp(): void { + const help = [ + 'Usage: import-links-to-d1 --file --owner-email [options]', + '', + 'Options:', + ' --file PATH Path to the JSONL export file (required).', + ' --owner-email EMAIL Admin user email that will own the imported links (required).', + ' --local Target the local miniflare D1 database.', + ' Omit to target the remote Cloudflare D1 database (default).', + ' --report PATH Path to write import-report.json (default: ./import-report.json).', + ' --dry-run Validate and report without writing to D1.', + ' -h, --help Show this help and exit.', + '', + 'Environment:', + ' HEYGO_WRANGLER_BIN Override the wrangler binary path (default: ./node_modules/.bin/wrangler).', + ].join('\n'); + process.stdout.write(`${help}\n`); +} + +function wranglerBin(): string { + const override = process.env.HEYGO_WRANGLER_BIN; + if (override) return override; + return pathResolve(process.cwd(), 'node_modules/.bin/wrangler'); +} + +type WranglerD1Result = { results: T[]; success: boolean; meta: Record }; + +function runD1Command(sql: string, local: boolean, dryRun: boolean): WranglerD1Result { + if (dryRun) { + return { results: [] as T[], success: true, meta: {} }; + } + const flag = local ? '--local' : '--remote'; + const out = execFileSync(wranglerBin(), ['d1', 'execute', 'DB', flag, '--json', '--command', sql], { + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + cwd: process.cwd(), + }); + const parsed = JSON.parse(out) as WranglerD1Result[]; + if (!parsed || parsed.length === 0) { + return { results: [] as T[], success: false, meta: {} }; + } + return parsed[0]; +} + +function runD1File(local: boolean, dryRun: boolean, sqlFile: string): WranglerD1Result { + if (dryRun) { + return { results: [] as unknown[], success: true, meta: {} }; + } + const flag = local ? '--local' : '--remote'; + const out = execFileSync( + wranglerBin(), + ['d1', 'execute', 'DB', flag, '--file', sqlFile], + { + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + cwd: process.cwd(), + }, + ); + // wrangler d1 execute --file does not emit JSON by default; return a stub. + return { results: [] as unknown[], success: !out.toLowerCase().includes('error'), meta: {} }; +} + +function sqlStringLiteral(value: string | null): string { + if (value === null) return 'NULL'; + // Escape single quotes for a SQLite string literal. + return `'${value.replace(/'/g, "''")}'`; +} + +function isHttpUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === 'http:' || url.protocol === 'https:'; + } catch { + return false; + } +} + +function resolveOwner(email: string, local: boolean, dryRun: boolean): OwnerRow { + const escaped = email.replace(/'/g, "''"); + const sql = `SELECT id, email, role FROM users WHERE email='${escaped.toLowerCase()}' LIMIT 1`; + const result = runD1Command(sql, local, dryRun); + const row = result.results?.[0] ?? null; + if (!row) { + stderr(`error: no user found with email ${email}. Create the admin user in D1 first.`); + process.exit(1); + } + if (row.role !== 'admin') { + stderr(`error: user ${email} has role '${row.role}', but --owner-email requires an admin user.`); + process.exit(1); + } + return row; +} + +function existingActiveAliases(ownerUserId: string, local: boolean, dryRun: boolean): Set { + const escaped = ownerUserId.replace(/'/g, "''"); + const sql = `SELECT alias FROM links WHERE scope='private' AND status='active' AND owner_user_id='${escaped}'`; + const result = runD1Command<{ alias: string }>(sql, local, dryRun); + return new Set(result.results?.map((r) => r.alias) ?? []); +} + +function validateRecord(rec: ExportRecord): { ok: true; alias: string; linkType: LinkType; targetUrl: string | null; contentMarkdown: string | null; description: string | null } | { ok: false; reason: string } { + if (rec.alias == null) { + return { ok: false, reason: 'record has no alias' }; + } + const aliasValidation = validateAlias(rec.alias); + if (!aliasValidation.ok) { + return { ok: false, reason: aliasValidation.error }; + } + const alias = aliasValidation.value; + + const rawType = String(rec.link_type ?? '').toLowerCase(); + if (rawType !== 'redirect' && rawType !== 'custom') { + return { ok: false, reason: `unknown link_type '${rec.link_type}'` }; + } + + const description = rec.description ?? null; + + if (rawType === 'custom') { + const content = rec.content_markdown ?? ''; + if (content.trim().length === 0) { + return { ok: false, reason: 'custom link has empty content_markdown' }; + } + return { ok: true, alias, linkType: 'custom', targetUrl: null, contentMarkdown: content, description }; + } + + // redirect + const targetUrl = rec.target_url ?? ''; + if (targetUrl.length === 0) { + return { ok: false, reason: 'redirect link has no target_url' }; + } + if (!isHttpUrl(targetUrl)) { + return { ok: false, reason: `target_url is not an http(s) URL: ${targetUrl}` }; + } + try { + parseTemplateParameters(targetUrl); + } catch (error) { + const reason = error instanceof TemplateResolutionError + ? error.message + : `template parameter validation failed for ${targetUrl}`; + return { ok: false, reason }; + } + + return { ok: true, alias, linkType: 'redirect', targetUrl, contentMarkdown: null, description }; +} + +function readJsonl(file: string): ExportRecord[] { + if (!existsSync(file)) { + stderr(`error: --file not found: ${file}`); + process.exit(2); + } + const text = readFileSync(file, 'utf8'); + const records: ExportRecord[] = []; + const lines = text.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (line.trim().length === 0) continue; + let obj: unknown; + try { + obj = JSON.parse(line); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + stderr(`error: line ${i + 1} is not valid JSON: ${msg}`); + process.exit(2); + } + records.push(obj as ExportRecord); + } + return records; +} + +function buildInsertSqlFile(imports: Array<{ id: string; alias: string; ownerUserId: string; linkType: LinkType; targetUrl: string | null; contentMarkdown: string | null; description: string | null }>): string { + const lines: string[] = []; + for (const imp of imports) { + // Inline literals because wrangler d1 execute --file has no bind params. + lines.push(`INSERT INTO links (id, scope, owner_user_id, alias, link_type, target_url, content_markdown, description, status, click_count) VALUES (${sqlStringLiteral(imp.id)}, 'private', ${sqlStringLiteral(imp.ownerUserId)}, ${sqlStringLiteral(imp.alias)}, ${sqlStringLiteral(imp.linkType)}, ${sqlStringLiteral(imp.targetUrl)}, ${sqlStringLiteral(imp.contentMarkdown)}, ${sqlStringLiteral(imp.description)}, 'active', 0);`); + } + lines.push('COMMIT;'); + return lines.join('\n') + '\n'; +} + +function writeReport(reportPath: string, report: Report): void { + const abs = pathResolve(process.cwd(), reportPath); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, JSON.stringify(report, null, 2) + '\n', 'utf8'); + stderr(`wrote report to ${abs}`); +} + +function main(): number { + const opts = parseArgs(process.argv.slice(2)); + const file = pathResolve(process.cwd(), opts.file); + const mode: 'local' | 'remote' = opts.local ? 'local' : 'remote'; + stderr(`import mode: ${mode}${opts.dryRun ? ' (dry-run)' : ''}`); + + const owner = resolveOwner(opts.ownerEmail, opts.local, opts.dryRun); + stderr(`owner: ${owner.email} (id=${owner.id}, role=${owner.role})`); + + const existing = existingActiveAliases(owner.id, opts.local, opts.dryRun); + stderr(`existing active private aliases for owner: ${existing.size}`); + + const records = readJsonl(file); + stderr(`read ${records.length} record(s) from ${file}`); + + const report: Report = { + file, + owner_email: opts.ownerEmail, + mode, + dry_run: opts.dryRun, + owner_user_id: owner.id, + total: records.length, + imported: 0, + skipped: 0, + conflicts: 0, + skipped_aliases: [], + conflict_details: [], + imported_aliases: [], + generated_at: new Date().toISOString(), + }; + + const imports: Array<{ id: string; alias: string; ownerUserId: string; linkType: LinkType; targetUrl: string | null; contentMarkdown: string | null; description: string | null }> = []; + + for (const rec of records) { + const validation = validateRecord(rec); + if (!validation.ok) { + const aliasName = (rec.alias ?? '').toString(); + report.conflicts += 1; + report.conflict_details.push({ alias: aliasName, reason: validation.reason }); + stderr(`conflict: '${aliasName}' skipped: ${validation.reason}`); + continue; + } + + if (existing.has(validation.alias)) { + report.skipped += 1; + report.skipped_aliases.push(validation.alias); + stderr(`skip: '${validation.alias}' already exists as active private link for owner`); + continue; + } + + const id = randomUUID(); + imports.push({ + id, + alias: validation.alias, + ownerUserId: owner.id, + linkType: validation.linkType, + targetUrl: validation.targetUrl, + contentMarkdown: validation.contentMarkdown, + description: validation.description, + }); + // Track within this run so duplicate aliases in the JSONL become skipped. + existing.add(validation.alias); + report.imported += 1; + report.imported_aliases.push(validation.alias); + } + + if (imports.length > 0 && !opts.dryRun) { + const sql = buildInsertSqlFile(imports); + const sqlFile = join(tmpdir(), `heygo-import-${Date.now()}.sql`); + writeFileSync(sqlFile, sql, 'utf8'); + stderr(`wrote ${imports.length} INSERT statement(s) to ${sqlFile}`); + const result = runD1File(opts.local, opts.dryRun, sqlFile); + if (!result.success) { + stderr('error: D1 insert batch failed. See wrangler output above.'); + report.imported = 0; + } + } + + writeReport(opts.reportPath, report); + + stderr( + `done: imported=${report.imported} skipped=${report.skipped} conflicts=${report.conflicts} total=${report.total}`, + ); + + return 0; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + const code = main(); + process.exit(code); +} + +export { + parseArgs, + validateRecord, + isHttpUrl, + sqlStringLiteral, + buildInsertSqlFile, + type ExportRecord, + type Report, + type LinkType, +};