Backfill: prefer fieldData over top-level + auto-load .env

While running the backfill against production we found that many
cards had user-corrected values trapped in fieldData with stale
OCR originals in the top-level column (e.g. LaGay: top-level
email "lagayferters@yahoo.com" vs fieldData "lagayfenters@yahoo.com").

The previous "only promote when top-level is empty" rule skipped
these — so the list view, search, and CSV export still showed the
stale OCR data even though the detail view showed the correction.

New rule:
  - When fieldData[canonical] is non-empty, promote it to the
    top-level column (regardless of whether the top-level column
    already has a value). Reasoning: pre-fix, the UI saved
    non-core edits only to fieldData, so any non-empty
    fieldData[canonical] is the user's most recent value (or
    matches the OCR original — harmless either way). Verified
    against the prod DB that fieldData never contains
    firstName/lastName/name, so there is no risk of reverting
    user-edited names.

Also:
  - Adds `import "dotenv/config"` so `npx tsx` picks up .env.
  - Logs fill-empty vs overwrite counts and a sample of conflicts
    (top-level → fieldData) so the dry-run is easy to audit.

Applied against prod: 23 form fields flipped, ~3127 values
promoted across 901 cards, 282 names recomputed. Re-run dry-run
reports 0 remaining changes.

Co-authored-by: Cursor <cursoragent@cursor.com>

scripts/backfill-core-fields.ts

@@ -20,17 +20,25 @@
  *    `key` corresponds to a real top-level `ResponseCard` column.
  * 2. Walks every `ResponseCard` and:
  *    a. promotes any matching keys from `fieldData` up to the top-level
- *       columns (only when the column is currently empty — explicit
+ *       columns whenever `fieldData[key]` is non-empty. `fieldData`
- *       column values win over stale-or-newer fieldData values);
+ *       wins over the top-level column because pre-fix the UI saved
+ *       non-core edits *only* to `fieldData`, so a non-empty
+ *       `fieldData[canonical]` is the user's most recent value (or
+ *       matches the OCR original anyway — harmless either way).
  *    b. recomputes `name` from `firstName` / `lastName` when it's stale
  *       or missing.
  *
+ *    Conflicts (where the top-level column already has a value but
+ *    `fieldData` has a *different* value) are logged so you can audit
+ *    the diff before committing with `--apply`.
+ *
  * Usage
  * ─────
  *   npx tsx scripts/backfill-core-fields.ts            # dry-run (default)
  *   npx tsx scripts/backfill-core-fields.ts --apply    # actually write
  */
 
+import "dotenv/config";
 import { PrismaClient } from "../src/generated/prisma/client.js";
 import { PrismaPg } from "@prisma/adapter-pg";
 import pg from "pg";
@@ -81,6 +89,17 @@ function isEmpty(v: unknown): boolean {
   return false;
 }
 
+function valuesEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (a == null || b == null) return false;
+  if (a instanceof Date && b instanceof Date) return a.getTime() === b.getTime();
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    return a.every((v, i) => v === b[i]);
+  }
+  return String(a) === String(b);
+}
+
 async function flipIsCoreOnTemplateFields(): Promise<void> {
   console.log("\n1. Promoting FormField rows to isCore = true…");
   const candidates = await prisma.formField.findMany({
@@ -113,9 +132,11 @@ async function backfillResponseCards(): Promise<void> {
   let cardsWithPromotion = 0;
   let cardsWithNameFix = 0;
   let totalFieldsPromoted = 0;
+  let totalFillEmpty = 0;
+  let totalOverwrite = 0;
 
-  // Sample logging
   const sample: string[] = [];
+  const conflicts: string[] = [];
 
   while (true) {
     const cards = await prisma.responseCard.findMany({
@@ -128,21 +149,34 @@ async function backfillResponseCards(): Promise<void> {
     for (const card of cards) {
       scanned++;
       const updates: Record<string, unknown> = {};
+      const cardOverwrites: Array<{ key: string; oldVal: unknown; newVal: unknown }> = [];
 
-      // (a) Promote fieldData keys → matching top-level columns when the
+      // (a) Promote fieldData keys → matching top-level columns. fieldData
-      //     top-level column is currently empty.
+      //     wins (see file header for reasoning).
       const fd = (card.fieldData as Record<string, unknown> | null) ?? null;
       if (fd && typeof fd === "object") {
         const cardAsAny = card as unknown as Record<string, unknown>;
         for (const [k, v] of Object.entries(fd)) {
           if (!ALL_PROMOTABLE_KEYS.has(k)) continue;
-          if (!isEmpty(cardAsAny[k])) continue; // don't clobber existing values
           if (isEmpty(v)) continue;
 
-          if (PROMOTABLE_STRING.has(k)) updates[k] = String(v);
+          let coerced: unknown;
-          else if (PROMOTABLE_BOOL.has(k)) updates[k] = Boolean(v);
+          if (PROMOTABLE_STRING.has(k)) coerced = String(v);
-          else if (PROMOTABLE_ARRAY.has(k)) updates[k] = Array.isArray(v) ? v : null;
+          else if (PROMOTABLE_BOOL.has(k)) coerced = Boolean(v);
-          else if (PROMOTABLE_DATE.has(k)) updates[k] = new Date(String(v));
+          else if (PROMOTABLE_ARRAY.has(k)) coerced = Array.isArray(v) ? v : null;
+          else if (PROMOTABLE_DATE.has(k)) coerced = new Date(String(v));
+          else continue;
+
+          const currentVal = cardAsAny[k];
+          if (valuesEqual(currentVal, coerced)) continue; // already in sync, skip
+
+          updates[k] = coerced;
+          if (isEmpty(currentVal)) {
+            totalFillEmpty++;
+          } else {
+            totalOverwrite++;
+            cardOverwrites.push({ key: k, oldVal: currentVal, newVal: coerced });
+          }
         }
       }
 
@@ -153,8 +187,6 @@ async function backfillResponseCards(): Promise<void> {
         ("lastName" in updates ? (updates.lastName as string | null) : card.lastName) ?? null;
       const combined = [nextFirst, nextLast].filter(Boolean).join(" ").trim();
       const desiredName = combined || null;
-      // Only touch `name` if we have first/last to derive from AND the current
-      // stored value differs. This avoids wiping a legacy name with null.
       if (desiredName && desiredName !== card.name) {
         updates.name = desiredName;
       }
@@ -173,6 +205,14 @@ async function backfillResponseCards(): Promise<void> {
           `     - ${card.id}: ${Object.keys(updates).map((k) => `${k}=${JSON.stringify(updates[k])}`).join(", ")}`
         );
       }
+      if (cardOverwrites.length > 0 && conflicts.length < 10) {
+        for (const o of cardOverwrites) {
+          if (conflicts.length >= 10) break;
+          conflicts.push(
+            `     - ${card.id}  ${o.key}:  ${JSON.stringify(o.oldVal)}  →  ${JSON.stringify(o.newVal)}`
+          );
+        }
+      }
 
       if (APPLY) {
         await prisma.responseCard.update({
@@ -188,11 +228,17 @@ async function backfillResponseCards(): Promise<void> {
 
   console.log(`   Scanned ${scanned} card(s)`);
   console.log(`   Promotion candidates: ${cardsWithPromotion} card(s), ${totalFieldsPromoted} field(s) total`);
+  console.log(`     · fill-empty (column was blank): ${totalFillEmpty} field(s)`);
+  console.log(`     · overwrite (column had a different value): ${totalOverwrite} field(s)`);
   console.log(`   Name-recompute candidates: ${cardsWithNameFix} card(s)`);
   if (sample.length) {
     console.log("   Sample changes:");
     for (const line of sample) console.log(line);
   }
+  if (conflicts.length) {
+    console.log(`   Sample conflicts (top-level → fieldData):`);
+    for (const line of conflicts) console.log(line);
+  }
 }
 
 async function main(): Promise<void> {