echos-ocr/scripts/backfill-core-fields.ts

/**
 * Backfill script for the "core fields" bug fix.
 *
 * Background
 * ──────────
 * The default form template marked only `firstName` / `lastName` as
 * `isCore: true`. Every other field (email, cellPhone, address, …) was
 * treated as non-core, so user edits in the dynamic-field form landed
 * inside `ResponseCard.fieldData` (JSON) instead of the matching top-level
 * column. The list view, search, integrations, and CSV export all read the
 * top-level columns, so they showed stale data after edits.
 *
 * Likewise, `ResponseCard.name` is a denormalized display string that was
 * never recomputed when `firstName` / `lastName` changed, so the table's
 * Name column and any integration that reads `name` stayed stale.
 *
 * What this script does
 * ─────────────────────
 * 1. Flips `isCore = true` on every existing `FormTemplateField` whose
 *    `key` corresponds to a real top-level `ResponseCard` column.
 * 2. Walks every `ResponseCard` and:
 *    a. promotes any matching keys from `fieldData` up to the top-level
 *       columns whenever `fieldData[key]` is non-empty. `fieldData`
 *       wins over the top-level column because pre-fix the UI saved
 *       non-core edits *only* to `fieldData`, so a non-empty
 *       `fieldData[canonical]` is the user's most recent value (or
 *       matches the OCR original anyway — harmless either way).
 *    b. recomputes `name` from `firstName` / `lastName` when it's stale
 *       or missing.
 *
 *    Conflicts (where the top-level column already has a value but
 *    `fieldData` has a *different* value) are logged so you can audit
 *    the diff before committing with `--apply`.
 *
 * Usage
 * ─────
 *   npx tsx scripts/backfill-core-fields.ts            # dry-run (default)
 *   npx tsx scripts/backfill-core-fields.ts --apply    # actually write
 */

import "dotenv/config";
import { PrismaClient } from "../src/generated/prisma/client.js";
import { PrismaPg } from "@prisma/adapter-pg";
import pg from "pg";

const APPLY = process.argv.includes("--apply");

const url = new URL(process.env.DATABASE_URL!);
url.searchParams.delete("sslmode");
const pool = new pg.Pool({
  connectionString: url.toString(),
  max: 5,
  ssl: { rejectUnauthorized: false },
});
const prisma = new PrismaClient({ adapter: new PrismaPg(pool) });

const PROMOTABLE_STRING = new Set([
  "firstName", "lastName", "name",
  "gender", "dateOfBirth",
  "maritalStatus", "maritalStatusOther", "visitType",
  "cellPhone", "homePhone", "email",
  "address", "aptNumber", "city", "state", "zip",
  "prayerRequests", "messageTopicsOther", "attendanceDuration",
  "campusPreferenceOther", "howHeardOther", "serviceAttended",
  "followUp", "notes", "serviceTime", "planningCenter",
]);
const PROMOTABLE_BOOL = new Set([
  "prayerForTeam", "prayerConfidential",
  "iSaidYesBookSent", "ftGuestLetterSent",
]);
const PROMOTABLE_ARRAY = new Set([
  "messageTopics", "nextStep", "campusPreference", "howHeard",
]);
const PROMOTABLE_DATE = new Set([
  "firstTimeGuestDate", "salvationDate",
]);

const ALL_PROMOTABLE_KEYS = new Set<string>([
  ...PROMOTABLE_STRING,
  ...PROMOTABLE_BOOL,
  ...PROMOTABLE_ARRAY,
  ...PROMOTABLE_DATE,
]);

function isEmpty(v: unknown): boolean {
  if (v === null || v === undefined) return true;
  if (typeof v === "string") return v.trim() === "";
  if (Array.isArray(v)) return v.length === 0;
  return false;
}

function valuesEqual(a: unknown, b: unknown): boolean {
  if (a === b) return true;
  if (a == null || b == null) return false;
  if (a instanceof Date && b instanceof Date) return a.getTime() === b.getTime();
  if (Array.isArray(a) && Array.isArray(b)) {
    if (a.length !== b.length) return false;
    return a.every((v, i) => v === b[i]);
  }
  return String(a) === String(b);
}

async function flipIsCoreOnTemplateFields(): Promise<void> {
  console.log("\n1. Promoting FormField rows to isCore = true…");
  const candidates = await prisma.formField.findMany({
    where: { isCore: false, key: { in: Array.from(ALL_PROMOTABLE_KEYS) } },
    select: { id: true, key: true, label: true, formTemplateId: true },
  });
  console.log(`   Found ${candidates.length} non-core field(s) that should be core`);
  if (candidates.length === 0) return;

  for (const f of candidates.slice(0, 10)) {
    console.log(`     - ${f.key}  (template ${f.formTemplateId})`);
  }
  if (candidates.length > 10) console.log(`     … and ${candidates.length - 10} more`);

  if (!APPLY) return;
  const ids = candidates.map((c) => c.id);
  const res = await prisma.formField.updateMany({
    where: { id: { in: ids } },
    data: { isCore: true },
  });
  console.log(`   Updated ${res.count} field(s)`);
}

async function backfillResponseCards(): Promise<void> {
  console.log("\n2. Backfilling ResponseCard top-level columns from fieldData…");

  const BATCH = 200;
  let cursor: string | undefined;
  let scanned = 0;
  let cardsWithPromotion = 0;
  let cardsWithNameFix = 0;
  let totalFieldsPromoted = 0;
  let totalFillEmpty = 0;
  let totalOverwrite = 0;

  const sample: string[] = [];
  const conflicts: string[] = [];

  while (true) {
    const cards = await prisma.responseCard.findMany({
      take: BATCH,
      ...(cursor ? { skip: 1, cursor: { id: cursor } } : {}),
      orderBy: { id: "asc" },
    });
    if (cards.length === 0) break;

    for (const card of cards) {
      scanned++;
      const updates: Record<string, unknown> = {};
      const cardOverwrites: Array<{ key: string; oldVal: unknown; newVal: unknown }> = [];

      // (a) Promote fieldData keys → matching top-level columns. fieldData
      //     wins (see file header for reasoning).
      const fd = (card.fieldData as Record<string, unknown> | null) ?? null;
      if (fd && typeof fd === "object") {
        const cardAsAny = card as unknown as Record<string, unknown>;
        for (const [k, v] of Object.entries(fd)) {
          if (!ALL_PROMOTABLE_KEYS.has(k)) continue;
          if (isEmpty(v)) continue;

          let coerced: unknown;
          if (PROMOTABLE_STRING.has(k)) coerced = String(v);
          else if (PROMOTABLE_BOOL.has(k)) coerced = Boolean(v);
          else if (PROMOTABLE_ARRAY.has(k)) coerced = Array.isArray(v) ? v : null;
          else if (PROMOTABLE_DATE.has(k)) coerced = new Date(String(v));
          else continue;

          const currentVal = cardAsAny[k];
          if (valuesEqual(currentVal, coerced)) continue; // already in sync, skip

          updates[k] = coerced;
          if (isEmpty(currentVal)) {
            totalFillEmpty++;
          } else {
            totalOverwrite++;
            cardOverwrites.push({ key: k, oldVal: currentVal, newVal: coerced });
          }
        }
      }

      // (b) Recompute `name` from firstName/lastName when stale or missing.
      const nextFirst =
        ("firstName" in updates ? (updates.firstName as string | null) : card.firstName) ?? null;
      const nextLast =
        ("lastName" in updates ? (updates.lastName as string | null) : card.lastName) ?? null;
      const combined = [nextFirst, nextLast].filter(Boolean).join(" ").trim();
      const desiredName = combined || null;
      if (desiredName && desiredName !== card.name) {
        updates.name = desiredName;
      }

      if (Object.keys(updates).length === 0) continue;

      const promotedFieldKeys = Object.keys(updates).filter((k) => k !== "name");
      if (promotedFieldKeys.length > 0) {
        cardsWithPromotion++;
        totalFieldsPromoted += promotedFieldKeys.length;
      }
      if ("name" in updates) cardsWithNameFix++;

      if (sample.length < 5) {
        sample.push(
          `     - ${card.id}: ${Object.keys(updates).map((k) => `${k}=${JSON.stringify(updates[k])}`).join(", ")}`
        );
      }
      if (cardOverwrites.length > 0 && conflicts.length < 10) {
        for (const o of cardOverwrites) {
          if (conflicts.length >= 10) break;
          conflicts.push(
            `     - ${card.id}  ${o.key}:  ${JSON.stringify(o.oldVal)}  →  ${JSON.stringify(o.newVal)}`
          );
        }
      }

      if (APPLY) {
        await prisma.responseCard.update({
          where: { id: card.id },
          data: updates as Parameters<typeof prisma.responseCard.update>[0]["data"],
        });
      }
    }

    cursor = cards[cards.length - 1].id;
    if (cards.length < BATCH) break;
  }

  console.log(`   Scanned ${scanned} card(s)`);
  console.log(`   Promotion candidates: ${cardsWithPromotion} card(s), ${totalFieldsPromoted} field(s) total`);
  console.log(`     · fill-empty (column was blank): ${totalFillEmpty} field(s)`);
  console.log(`     · overwrite (column had a different value): ${totalOverwrite} field(s)`);
  console.log(`   Name-recompute candidates: ${cardsWithNameFix} card(s)`);
  if (sample.length) {
    console.log("   Sample changes:");
    for (const line of sample) console.log(line);
  }
  if (conflicts.length) {
    console.log(`   Sample conflicts (top-level → fieldData):`);
    for (const line of conflicts) console.log(line);
  }
}

async function main(): Promise<void> {
  console.log("=== ResponseCard core-fields backfill ===");
  console.log(APPLY ? "Mode: APPLY (writes enabled)" : "Mode: DRY-RUN (no writes — pass --apply to commit)");

  await flipIsCoreOnTemplateFields();
  await backfillResponseCards();

  console.log("\nDone.");
}

main()
  .catch((err) => {
    console.error("Backfill failed:", err);
    process.exitCode = 1;
  })
  .finally(async () => {
    await prisma.$disconnect();
    await pool.end();
  });