From 3864219f0a7f72ba7fe8b3e376d3c9264af23234 Mon Sep 17 00:00:00 2001 From: Randall Stillwell Date: Sat, 23 May 2026 15:09:08 -0500 Subject: [PATCH] Backfill: prefer fieldData over top-level + auto-load .env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While running the backfill against production we found that many cards had user-corrected values trapped in fieldData with stale OCR originals in the top-level column (e.g. LaGay: top-level email "lagayferters@yahoo.com" vs fieldData "lagayfenters@yahoo.com"). The previous "only promote when top-level is empty" rule skipped these — so the list view, search, and CSV export still showed the stale OCR data even though the detail view showed the correction. New rule: - When fieldData[canonical] is non-empty, promote it to the top-level column (regardless of whether the top-level column already has a value). Reasoning: pre-fix, the UI saved non-core edits only to fieldData, so any non-empty fieldData[canonical] is the user's most recent value (or matches the OCR original — harmless either way). Verified against the prod DB that fieldData never contains firstName/lastName/name, so there is no risk of reverting user-edited names. Also: - Adds `import "dotenv/config"` so `npx tsx` picks up .env. - Logs fill-empty vs overwrite counts and a sample of conflicts (top-level → fieldData) so the dry-run is easy to audit. Applied against prod: 23 form fields flipped, ~3127 values promoted across 901 cards, 282 names recomputed. Re-run dry-run reports 0 remaining changes. Co-authored-by: Cursor --- scripts/backfill-core-fields.ts | 70 +++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/scripts/backfill-core-fields.ts b/scripts/backfill-core-fields.ts index 67e0269..ec79014 100644 --- a/scripts/backfill-core-fields.ts +++ b/scripts/backfill-core-fields.ts @@ -20,17 +20,25 @@ * `key` corresponds to a real top-level `ResponseCard` column. * 2. Walks every `ResponseCard` and: * a. promotes any matching keys from `fieldData` up to the top-level - * columns (only when the column is currently empty — explicit - * column values win over stale-or-newer fieldData values); + * columns whenever `fieldData[key]` is non-empty. `fieldData` + * wins over the top-level column because pre-fix the UI saved + * non-core edits *only* to `fieldData`, so a non-empty + * `fieldData[canonical]` is the user's most recent value (or + * matches the OCR original anyway — harmless either way). * b. recomputes `name` from `firstName` / `lastName` when it's stale * or missing. * + * Conflicts (where the top-level column already has a value but + * `fieldData` has a *different* value) are logged so you can audit + * the diff before committing with `--apply`. + * * Usage * ───── * npx tsx scripts/backfill-core-fields.ts # dry-run (default) * npx tsx scripts/backfill-core-fields.ts --apply # actually write */ +import "dotenv/config"; import { PrismaClient } from "../src/generated/prisma/client.js"; import { PrismaPg } from "@prisma/adapter-pg"; import pg from "pg"; @@ -81,6 +89,17 @@ function isEmpty(v: unknown): boolean { return false; } +function valuesEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (a == null || b == null) return false; + if (a instanceof Date && b instanceof Date) return a.getTime() === b.getTime(); + if (Array.isArray(a) && Array.isArray(b)) { + if (a.length !== b.length) return false; + return a.every((v, i) => v === b[i]); + } + return String(a) === String(b); +} + async function flipIsCoreOnTemplateFields(): Promise { console.log("\n1. Promoting FormField rows to isCore = true…"); const candidates = await prisma.formField.findMany({ @@ -113,9 +132,11 @@ async function backfillResponseCards(): Promise { let cardsWithPromotion = 0; let cardsWithNameFix = 0; let totalFieldsPromoted = 0; + let totalFillEmpty = 0; + let totalOverwrite = 0; - // Sample logging const sample: string[] = []; + const conflicts: string[] = []; while (true) { const cards = await prisma.responseCard.findMany({ @@ -128,21 +149,34 @@ async function backfillResponseCards(): Promise { for (const card of cards) { scanned++; const updates: Record = {}; + const cardOverwrites: Array<{ key: string; oldVal: unknown; newVal: unknown }> = []; - // (a) Promote fieldData keys → matching top-level columns when the - // top-level column is currently empty. + // (a) Promote fieldData keys → matching top-level columns. fieldData + // wins (see file header for reasoning). const fd = (card.fieldData as Record | null) ?? null; if (fd && typeof fd === "object") { const cardAsAny = card as unknown as Record; for (const [k, v] of Object.entries(fd)) { if (!ALL_PROMOTABLE_KEYS.has(k)) continue; - if (!isEmpty(cardAsAny[k])) continue; // don't clobber existing values if (isEmpty(v)) continue; - if (PROMOTABLE_STRING.has(k)) updates[k] = String(v); - else if (PROMOTABLE_BOOL.has(k)) updates[k] = Boolean(v); - else if (PROMOTABLE_ARRAY.has(k)) updates[k] = Array.isArray(v) ? v : null; - else if (PROMOTABLE_DATE.has(k)) updates[k] = new Date(String(v)); + let coerced: unknown; + if (PROMOTABLE_STRING.has(k)) coerced = String(v); + else if (PROMOTABLE_BOOL.has(k)) coerced = Boolean(v); + else if (PROMOTABLE_ARRAY.has(k)) coerced = Array.isArray(v) ? v : null; + else if (PROMOTABLE_DATE.has(k)) coerced = new Date(String(v)); + else continue; + + const currentVal = cardAsAny[k]; + if (valuesEqual(currentVal, coerced)) continue; // already in sync, skip + + updates[k] = coerced; + if (isEmpty(currentVal)) { + totalFillEmpty++; + } else { + totalOverwrite++; + cardOverwrites.push({ key: k, oldVal: currentVal, newVal: coerced }); + } } } @@ -153,8 +187,6 @@ async function backfillResponseCards(): Promise { ("lastName" in updates ? (updates.lastName as string | null) : card.lastName) ?? null; const combined = [nextFirst, nextLast].filter(Boolean).join(" ").trim(); const desiredName = combined || null; - // Only touch `name` if we have first/last to derive from AND the current - // stored value differs. This avoids wiping a legacy name with null. if (desiredName && desiredName !== card.name) { updates.name = desiredName; } @@ -173,6 +205,14 @@ async function backfillResponseCards(): Promise { ` - ${card.id}: ${Object.keys(updates).map((k) => `${k}=${JSON.stringify(updates[k])}`).join(", ")}` ); } + if (cardOverwrites.length > 0 && conflicts.length < 10) { + for (const o of cardOverwrites) { + if (conflicts.length >= 10) break; + conflicts.push( + ` - ${card.id} ${o.key}: ${JSON.stringify(o.oldVal)} → ${JSON.stringify(o.newVal)}` + ); + } + } if (APPLY) { await prisma.responseCard.update({ @@ -188,11 +228,17 @@ async function backfillResponseCards(): Promise { console.log(` Scanned ${scanned} card(s)`); console.log(` Promotion candidates: ${cardsWithPromotion} card(s), ${totalFieldsPromoted} field(s) total`); + console.log(` · fill-empty (column was blank): ${totalFillEmpty} field(s)`); + console.log(` · overwrite (column had a different value): ${totalOverwrite} field(s)`); console.log(` Name-recompute candidates: ${cardsWithNameFix} card(s)`); if (sample.length) { console.log(" Sample changes:"); for (const line of sample) console.log(line); } + if (conflicts.length) { + console.log(` Sample conflicts (top-level → fieldData):`); + for (const line of conflicts) console.log(line); + } } async function main(): Promise {