rstillwell/echos-ocr
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Backfill: prefer fieldData over top-level + auto-load .env

Browse source 
While running the backfill against production we found that many
cards had user-corrected values trapped in fieldData with stale
OCR originals in the top-level column (e.g. LaGay: top-level
email "lagayferters@yahoo.com" vs fieldData "lagayfenters@yahoo.com").

The previous "only promote when top-level is empty" rule skipped
these — so the list view, search, and CSV export still showed the
stale OCR data even though the detail view showed the correction.

New rule:
  - When fieldData[canonical] is non-empty, promote it to the
    top-level column (regardless of whether the top-level column
    already has a value). Reasoning: pre-fix, the UI saved
    non-core edits only to fieldData, so any non-empty
    fieldData[canonical] is the user's most recent value (or
    matches the OCR original — harmless either way). Verified
    against the prod DB that fieldData never contains
    firstName/lastName/name, so there is no risk of reverting
    user-edited names.

Also:
  - Adds `import "dotenv/config"` so `npx tsx` picks up .env.
  - Logs fill-empty vs overwrite counts and a sample of conflicts
    (top-level → fieldData) so the dry-run is easy to audit.

Applied against prod: 23 form fields flipped, ~3127 values
promoted across 901 cards, 282 names recomputed. Re-run dry-run
reports 0 remaining changes.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Randall Stillwell 2026-05-23 15:09:08 -05:00
parent 9c1aaaa61f
commit 3864219f0a
1 changed files with 58 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
scripts/backfill-core-fields.ts
View file

@ -20,17 +20,25 @@
* `key` corresponds to a real top-level `ResponseCard` column.
* 2. Walks every `ResponseCard` and:
* a. promotes any matching keys from `fieldData` up to the top-level
* columns (only when the column is currently empty — explicit
* column values win over stale-or-newer fieldData values);
* columns whenever `fieldData[key]` is non-empty. `fieldData`
* wins over the top-level column because pre-fix the UI saved
* non-core edits *only* to `fieldData`, so a non-empty
* `fieldData[canonical]` is the user's most recent value (or
* matches the OCR original anyway — harmless either way).
* b. recomputes `name` from `firstName` / `lastName` when it's stale
* or missing.
*
* Conflicts (where the top-level column already has a value but
* `fieldData` has a *different* value) are logged so you can audit
* the diff before committing with `--apply`.
*
* Usage
* ─────
* npx tsx scripts/backfill-core-fields.ts # dry-run (default)
* npx tsx scripts/backfill-core-fields.ts --apply # actually write
*/
import "dotenv/config";
import { PrismaClient } from "../src/generated/prisma/client.js";
import { PrismaPg } from "@prisma/adapter-pg";
import pg from "pg";
@ -81,6 +89,17 @@ function isEmpty(v: unknown): boolean {
return false;
}
function valuesEqual(a: unknown, b: unknown): boolean {
if (a === b) return true;
if (a == null || b == null) return false;
if (a instanceof Date && b instanceof Date) return a.getTime() === b.getTime();
if (Array.isArray(a) && Array.isArray(b)) {
if (a.length !== b.length) return false;
return a.every((v, i) => v === b[i]);
}
return String(a) === String(b);
}
async function flipIsCoreOnTemplateFields(): Promise<void> {
console.log("\n1. Promoting FormField rows to isCore = true…");
const candidates = await prisma.formField.findMany({
@ -113,9 +132,11 @@ async function backfillResponseCards(): Promise<void> {
let cardsWithPromotion = 0;
let cardsWithNameFix = 0;
let totalFieldsPromoted = 0;
let totalFillEmpty = 0;
let totalOverwrite = 0;
// Sample logging
const sample: string[] = [];
const conflicts: string[] = [];
while (true) {
const cards = await prisma.responseCard.findMany({
@ -128,21 +149,34 @@ async function backfillResponseCards(): Promise<void> {
for (const card of cards) {
scanned++;
const updates: Record<string, unknown> = {};
const cardOverwrites: Array<{ key: string; oldVal: unknown; newVal: unknown }> = [];
// (a) Promote fieldData keys → matching top-level columns when the
// top-level column is currently empty.
// (a) Promote fieldData keys → matching top-level columns. fieldData
// wins (see file header for reasoning).
const fd = (card.fieldData as Record<string, unknown> | null) ?? null;
if (fd && typeof fd === "object") {
const cardAsAny = card as unknown as Record<string, unknown>;
for (const [k, v] of Object.entries(fd)) {
if (!ALL_PROMOTABLE_KEYS.has(k)) continue;
if (!isEmpty(cardAsAny[k])) continue; // don't clobber existing values
if (isEmpty(v)) continue;
if (PROMOTABLE_STRING.has(k)) updates[k] = String(v);
else if (PROMOTABLE_BOOL.has(k)) updates[k] = Boolean(v);
else if (PROMOTABLE_ARRAY.has(k)) updates[k] = Array.isArray(v) ? v : null;
else if (PROMOTABLE_DATE.has(k)) updates[k] = new Date(String(v));
let coerced: unknown;
if (PROMOTABLE_STRING.has(k)) coerced = String(v);
else if (PROMOTABLE_BOOL.has(k)) coerced = Boolean(v);
else if (PROMOTABLE_ARRAY.has(k)) coerced = Array.isArray(v) ? v : null;
else if (PROMOTABLE_DATE.has(k)) coerced = new Date(String(v));
else continue;
const currentVal = cardAsAny[k];
if (valuesEqual(currentVal, coerced)) continue; // already in sync, skip
updates[k] = coerced;
if (isEmpty(currentVal)) {
totalFillEmpty++;
} else {
totalOverwrite++;
cardOverwrites.push({ key: k, oldVal: currentVal, newVal: coerced });
}
}
}
@ -153,8 +187,6 @@ async function backfillResponseCards(): Promise<void> {
("lastName" in updates ? (updates.lastName as string | null) : card.lastName) ?? null;
const combined = [nextFirst, nextLast].filter(Boolean).join(" ").trim();
const desiredName = combined || null;
// Only touch `name` if we have first/last to derive from AND the current
// stored value differs. This avoids wiping a legacy name with null.
if (desiredName && desiredName !== card.name) {
updates.name = desiredName;
}
@ -173,6 +205,14 @@ async function backfillResponseCards(): Promise<void> {
` - ${card.id}: ${Object.keys(updates).map((k) => `${k}=${JSON.stringify(updates[k])}`).join(", ")}`
);
}
if (cardOverwrites.length > 0 && conflicts.length < 10) {
for (const o of cardOverwrites) {
if (conflicts.length >= 10) break;
conflicts.push(
` - ${card.id} ${o.key}: ${JSON.stringify(o.oldVal)} → ${JSON.stringify(o.newVal)}`
);
}
}
if (APPLY) {
await prisma.responseCard.update({
@ -188,11 +228,17 @@ async function backfillResponseCards(): Promise<void> {
console.log(` Scanned ${scanned} card(s)`);
console.log(` Promotion candidates: ${cardsWithPromotion} card(s), ${totalFieldsPromoted} field(s) total`);
console.log(` · fill-empty (column was blank): ${totalFillEmpty} field(s)`);
console.log(` · overwrite (column had a different value): ${totalOverwrite} field(s)`);
console.log(` Name-recompute candidates: ${cardsWithNameFix} card(s)`);
if (sample.length) {
console.log(" Sample changes:");
for (const line of sample) console.log(line);
}
if (conflicts.length) {
console.log(` Sample conflicts (top-level → fieldData):`);
for (const line of conflicts) console.log(line);
}
}
async function main(): Promise<void> {