diff --git a/.agent-context-manifest.yml b/.agent-context-manifest.yml index ce82bb9..6c64787 100644 --- a/.agent-context-manifest.yml +++ b/.agent-context-manifest.yml @@ -1,142 +1,166 @@ -# .agent-context-manifest.yml -# -# Generated/updated by bulk pipeline sync. -# schema_version: 1 -pipeline_version: "0.6.0" -pipeline_source: "https://github.com/varutasu/agent-pipeline" -installed_at: "2026-05-22T22:25:00Z" -last_synced_at: "2026-06-22T19:48:24Z" +pipeline_version: 0.7.0 +pipeline_source: https://github.com/varutasu/agent-pipeline +installed_at: '2026-05-22T22:25:00Z' +last_synced_at: '2026-08-14T23:50:20Z' layers: - - L1 - - L2 - - L3 +- L1 +- L2 +- L3 artifacts: - - path: ".convoys/README.md" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/_common/convoys-readme.md.template" - version: "0.6.0" - installed_hash: "sha256:12bedb29aeecfd6fddd04a96b60ac9cd65a453c4ef18d0086274a36520ebc68f" - - path: ".cursor/agents/role-a11y-auditor.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-a11y-auditor.md" - version: "0.6.0" - installed_hash: "sha256:119bca847d885ae092e2da878662f223ba57e0ddc459f49458f2f3fbefb668df" - - path: ".cursor/agents/role-architect.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-architect.md" - version: "0.6.0" - installed_hash: "sha256:32fdc10ffa2795144bb6e69eab11a31841c46ada77eebbc6d38b2047e5b565a0" - - path: ".cursor/agents/role-conductor.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-conductor.md" - version: "0.6.0" - installed_hash: "sha256:649d5c9974ea58fa3ced254a12ed33c0b39df8799b2e6981b05d35313f9c1d07" - - path: ".cursor/agents/role-design-system-auditor.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-design-system-auditor.md" - version: "0.6.0" - installed_hash: "sha256:8731e837662e2188fcbdb7f8730ffc2138236dce91785f7671cdbdc1d849fefd" - - path: ".cursor/agents/role-doc-writer.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-doc-writer.md" - version: "0.6.0" - installed_hash: "sha256:7e626346705083cd57fa8a401b18f7f44da330a9f2a60f461dc362fbb2c7159b" - - path: ".cursor/agents/role-ia-architect.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-ia-architect.md" - version: "0.6.0" - installed_hash: "sha256:40d669a8a7ebf1e6165ab1054b189f728ceefaadcfb124d47b55bceaf7c8fac4" - - path: ".cursor/agents/role-implementer.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-implementer.md" - version: "0.6.0" - installed_hash: "sha256:978e972384f0277bac0d6e5f1226ff8e833d4d9c091ea207289555c1d74471a9" - - path: ".cursor/agents/role-reviewer.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-reviewer.md" - version: "0.6.0" - installed_hash: "sha256:58863d74cf8cb4990538862ed92e39bfcd51388961a9e0f64141924efb9c9dcc" - - path: ".cursor/agents/role-ux-reviewer.md" - source: "skills/bootstrap-agent-context/templates/L2-roles/role-ux-reviewer.md" - version: "0.6.0" - installed_hash: "sha256:40db450d9b8483ea527636a1ee2d15521edae0a4c7cd8d011880103fe10fa103" - - path: ".cursor/rules/api-routes.mdc" - source: "skills/bootstrap-agent-context/templates/L1-context/api-routes.mdc.template" - version: "0.5.0-local" - installed_hash: "sha256:54cd66d71f5a129a67d0f4b1797f5f63b7f9aae3eeabe67217861456ff4db59b" - - path: ".cursor/rules/auth-and-permissions.mdc" - source: "tcg-vault-local" - version: "0.5.0-local" - installed_hash: "sha256:9b7eb7bea0cad0e43d0e9442eb8b660945cb6935f9f7c82fd7d7d71d66b39a2d" - - path: ".cursor/rules/db-and-schema.mdc" - source: "tcg-vault-local" - version: "0.5.0-local" - installed_hash: "sha256:83df2cf7121722a092f85165b1a93c755ce57e5361e0f6b2ecc74e9f928c5015" - - path: ".cursor/rules/model-routing.mdc" - source: "skills/bootstrap-agent-context/templates/L1-context/model-routing.mdc.template" - version: "0.6.0" - installed_hash: "sha256:00c5b76a274379af50d564dc58c81714a00a74906fa383c950a9bd7b56692cb3" - - path: ".cursor/rules/no-go-zones.mdc" - source: "skills/bootstrap-agent-context/templates/L1-context/no-go-zones.mdc" - version: "0.5.0-local" - installed_hash: "sha256:aa7046bc3e0266cb3c9b0eb0ef8f68cc50d6837f65c96861804ff81b9c4afa64" - - path: ".cursor/rules/schema-map.mdc" - source: "tcg-vault-local" - version: "0.5.0-local" - installed_hash: "sha256:3429bad56384117dc81873b337a6d815bd53799389f7908dedb53dbb7642bced" - - path: ".cursor/rules/ui-and-theming.mdc" - source: "tcg-vault-local" - version: "0.5.0-local" - installed_hash: "sha256:b841ddda5baa47a76c3726a3c92b2c82a45120fdd461b3243bf970479d1cf1df" - - path: ".cursor/skills/add-api-route/SKILL.md" - source: "tcg-vault-local" - version: "0.5.0-local" - installed_hash: "sha256:0e29f7e994a51e5a40b8308edab08ee9c1f713e297d68e98029294d8ca568cc7" - - path: ".cursor/skills/add-page/SKILL.md" - source: "tcg-vault-local" - version: "0.5.0-local" - installed_hash: "sha256:318912077a6ced6a3a31f85dc15d069bf7627c161b6735e3fa259ca10766daa9" - - path: ".github/CODEOWNERS" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/nextjs/CODEOWNERS.template" - version: "0.5.0-local" - installed_hash: "sha256:b714a0a011776300abeab92fe8969f150c273c37d0d6b37c1ad2eb67d47decda" - - path: ".github/PULL_REQUEST_TEMPLATE.md" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/_common/PULL_REQUEST_TEMPLATE.md.template" - version: "0.5.0" - installed_hash: "sha256:89863e58b9ec194aef1c94d3596e892467833e8bc880a28994acca401b6d9635" - - path: ".github/workflows/agent-context-drift.yml" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/_common/agent-context-drift.yml.template" - version: "0.5.0" - installed_hash: "sha256:5505c296c1b61d023ee2aca222103097e2b5ed2e0e38da3679cc4f9754457785" - - path: ".github/workflows/ci.yml" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/nextjs/ci.yml.template" - version: "0.5.0-local" - installed_hash: "sha256:6aff7a1c9f2e42606580c241b6dadca7c2d8550aeb959bd69fdd843eb9097cac" - - path: ".github/workflows/pr-health-rollup.yml" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/pr-health-rollup.yml.template" - version: "0.5.0-local" - installed_hash: "sha256:8747674323807d84395fa027b25e7e27881c5e6b0cc87a138d9cb3f78fd88956" - - path: ".github/workflows/preview-smoke.yml" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/preview-smoke.yml.template" - version: "0.5.0-local" - installed_hash: "sha256:2e71026b09db8b2f32b6a868d705489600c875082d6320c2369bf2f5ebc315b8" - - path: ".github/workflows/visual-diff.yml" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/visual-diff.yml.template" - version: "0.5.0-local" - installed_hash: "sha256:88270b1fa59aba99591ec094764dd367deed956bcb746ac6bb195241b3a7dae1" - - path: "docs/agent-context/README.md" - source: "skills/bootstrap-agent-context/templates/L1-context/agent-context-readme.md.template" - version: "0.5.0-local" - installed_hash: "sha256:095b9cc6a30327114c9ddfb4ff57a5fde76b213e12b1c5574a1f96205d60dbad" - - path: "docs/agent-context/model-routing-policy.md" - source: "docs/model-routing-policy.md" - version: "0.6.0" - installed_hash: "sha256:9328ae01f97426f12389710e806cff542c9255d8dd3fcc933c3418da66ff08b7" - - path: "lib/flags/index.js" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/flags-index.ts.template" - version: "0.5.0-local" - installed_hash: "sha256:1a3cd1f900194eaf4ec86588dd1c3c2bff6a565e742061fc911abdd47bd5f3a5" - - path: "scripts/log-convoy-event.sh" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/_common/log-convoy-event.sh" - version: "0.6.0" - installed_hash: "sha256:52bdc8f60b18315dd8ad0f1dd6b727106dfa134b8769b0cd63d8698d5865cf21" - - path: "scripts/wt.sh" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/_common/wt.sh" - version: "0.5.0" - installed_hash: "sha256:2a4f44a159f80a8ea6fe53ac507c01a2f91a4e2118d997a98b051808ac35e9a5" - - path: "tests/smoke/app.smoke.spec.ts" - source: "skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/playwright-smoke.spec.ts.template" - version: "0.5.0" - installed_hash: "sha256:a62c10edb712a61f1cfece43705bfff75a5a66ad6bc8b53f7e69a43c3efb962c" +- path: .convoys/README.md + source: skills/bootstrap-agent-context/templates/L3-pipeline/_common/convoys-readme.md.template + version: 0.6.0 + installed_hash: sha256:6b779efd3116fffb0f2affdc57964750234d63c75092074e632a9b546d709bd6 +- path: .cursor/agents/role-a11y-auditor.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-a11y-auditor.md + version: 0.7.0 + installed_hash: sha256:f457840b51f6f4b0c95174ccee175fd5b65f7c9f5f65598ef80ac7fd532110ec +- path: .cursor/agents/role-architect.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-architect.md + version: 0.7.0 + installed_hash: sha256:bc38e10177219a3e3903b9019f1052c32fd38ea59373368c3e5d1b1011435b58 +- path: .cursor/agents/role-conductor.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-conductor.md + version: 0.7.0 + installed_hash: sha256:c4f764becd31175925c711fbf196ae559f989dc78c3f336f1da39349c9b3ee62 +- path: .cursor/agents/role-design-system-auditor.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-design-system-auditor.md + version: 0.7.0 + installed_hash: sha256:e52b507f12c507411540fa30277e70ab6dd25cc5a742f08c35643f93e73032f6 +- path: .cursor/agents/role-doc-writer.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-doc-writer.md + version: 0.7.0 + installed_hash: sha256:7e626346705083cd57fa8a401b18f7f44da330a9f2a60f461dc362fbb2c7159b +- path: .cursor/agents/role-ia-architect.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-ia-architect.md + version: 0.7.0 + installed_hash: sha256:40d669a8a7ebf1e6165ab1054b189f728ceefaadcfb124d47b55bceaf7c8fac4 +- path: .cursor/agents/role-implementer.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-implementer.md + version: 0.7.0 + installed_hash: sha256:ae6e4dfa3974af4fbe70c892a7806e68f7268fd1079ad96e7844fa7435b0129a +- path: .cursor/agents/role-reviewer.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-reviewer.md + version: 0.7.0 + installed_hash: sha256:e0753d5a2d86f59559ded52d7136cec5ab3cd200b42926b67a56469513eaf41b +- path: .cursor/agents/role-security-auditor.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-security-auditor.md + version: 0.7.0 + installed_hash: sha256:f141a54541626b7344c9883431251d6f02d93ede3d070cf4ffbcaea9c19689e0 +- path: .cursor/agents/role-ui-designer.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-ui-designer.md + version: 0.7.0 + installed_hash: sha256:607dc3783131018dd1c3527bba682ec2fc3c33221bc2bbd66add56fc1691ef18 +- path: .cursor/agents/role-ux-reviewer.md + source: skills/bootstrap-agent-context/templates/L2-roles/role-ux-reviewer.md + version: 0.7.0 + installed_hash: sha256:c83c365094266d2bd25afa761204116a620d1acbeb854b589cf51bbecefe8100 +- path: .cursor/rules/api-routes.mdc + source: skills/bootstrap-agent-context/templates/L1-context/api-routes.mdc.template + version: 0.5.0-local + installed_hash: sha256:92b67b6d0a763c23d95cb63152ea5e8837bb7aa467c703e3cdf059b2d74edad5 +- path: .cursor/rules/auth-and-permissions.mdc + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:bad3b270bc9a6fcbae0386fc86b46c1dec759f8b023f48ac4f9a827d65f97ca3 +- path: .cursor/rules/convoy-planning.mdc + source: skills/bootstrap-agent-context/templates/L1-context/convoy-planning.mdc.template + version: 0.6.0 + installed_hash: sha256:d0d4e2e06905d1e58a6a1d4fd9cda3cdb1bc80698c1e1c699e1c939278f23ea3 +- path: .cursor/rules/db-and-schema.mdc + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:6cf287d694d31e633c8a1add9a645cf9a21b4efe5dce66af104209029aa828a3 +- path: .cursor/rules/model-routing.mdc + source: skills/bootstrap-agent-context/templates/L1-context/model-routing.mdc.template + version: 0.7.0 + installed_hash: sha256:cac45f7aa457eb9312b734f40a55e69e7b30c859b7d8e975a7b80c1268a578f6 +- path: .cursor/rules/no-go-zones.mdc + source: skills/bootstrap-agent-context/templates/L1-context/no-go-zones.mdc + version: 0.5.0-local + installed_hash: sha256:bfa661b7bb67047cf33f7ab13116671c12d32a11022bf29dd1e5a12b6f7cf0b5 +- path: .cursor/rules/schema-map.mdc + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:3429bad56384117dc81873b337a6d815bd53799389f7908dedb53dbb7642bced +- path: .cursor/rules/security-baseline.mdc + source: skills/bootstrap-agent-context/templates/L1-context/security-baseline.mdc.template + version: 0.6.0 + installed_hash: sha256:0f0f919d8c500a5e393bf3def01a4ee69c3c489c80a86c6918f0c03aba41083e +- path: .cursor/rules/ui-and-theming.mdc + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:66cb77e0c4b72605d8be43986e38012fc62b7f3a0e707b7bf80a28f91d208e79 +- path: .cursor/skills/add-api-route/SKILL.md + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:0e29f7e994a51e5a40b8308edab08ee9c1f713e297d68e98029294d8ca568cc7 +- path: .cursor/skills/add-page/SKILL.md + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:564582ddc877d0d063cf2afa7796ddfc62a00d5d7659debb8e017629dfbb3aaf +- path: .cursor/skills/security-audit/SKILL.md + source: skills/security-audit/SKILL.md + version: 0.6.0 + installed_hash: sha256:8148f9ea9e66929ff51b003c6f5d6026c5d1525d31d036affb219624bb5e9305 +- path: .cursor/skills/ui-ux-pro-max/SKILL.md + source: skills/ui-ux-pro-max/SKILL.md + version: 0.6.0 + installed_hash: sha256:9debdd7439a6f73318e2f624e78d905f78885acf009ce0c34a02aa348b3c1c17 +- path: .github/CODEOWNERS + source: skills/bootstrap-agent-context/templates/L3-pipeline/nextjs/CODEOWNERS.template + version: 0.5.0-local + installed_hash: sha256:aff1f610b892b437dbd9c82ab56ce12eb721c06b4cd86c38fa15081ab3951c70 +- path: .github/PULL_REQUEST_TEMPLATE.md + source: skills/bootstrap-agent-context/templates/L3-pipeline/_common/PULL_REQUEST_TEMPLATE.md.template + version: 0.6.0 + installed_hash: sha256:deaca37703e9c348937614577d1c165993093450434df6002aa395dd70b2ba78 +- path: .github/workflows/agent-context-drift.yml + source: skills/bootstrap-agent-context/templates/L3-pipeline/_common/agent-context-drift.yml.template + version: 0.5.0 + installed_hash: sha256:5505c296c1b61d023ee2aca222103097e2b5ed2e0e38da3679cc4f9754457785 +- path: .github/workflows/ci.yml + source: skills/bootstrap-agent-context/templates/L3-pipeline/nextjs/ci.yml.template + version: 0.5.0-local + installed_hash: sha256:e4b480517346e978a27b22f36dd1926d4c1787176cec6495d69e7119c1e547b2 +- path: .github/workflows/convoy-metrics-gate.yml + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:ebdcba74f81fe281ab6cc1306630295addb81c5deba06d313948b8ebf8c199b8 +- path: .github/workflows/pr-health-rollup.yml + source: skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/pr-health-rollup.yml.template + version: 0.5.0-local + installed_hash: sha256:a8ead80d2e63b9c9a54c014ed0f130fca90fe2857664ac77d81acc5ee2ebb681 +- path: .github/workflows/preview-smoke.yml + source: skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/preview-smoke.yml.template + version: 0.5.0-local + installed_hash: sha256:9f6473f716e541164c10ef30ef12258f1a8e360bb7776db20dc26872f687539c +- path: .github/workflows/visual-diff.yml + source: skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/visual-diff.yml.template + version: 0.5.0-local + installed_hash: sha256:7dbe634bfe7a6a6d1ca2c76c86dc80947e0a699e393153f47dd047ced3f86975 +- path: docs/agent-context/README.md + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:095b9cc6a30327114c9ddfb4ff57a5fde76b213e12b1c5574a1f96205d60dbad +- path: docs/agent-context/model-routing-policy.md + source: docs/model-routing-policy.md + version: 0.7.0 + installed_hash: sha256:cc7a9a39ff28c6b743c47efbdf06fc3b75cf16c9fa62691fa1b05862a460dd45 +- path: lib/flags/index.js + source: tcg-vault-local + version: 0.5.0-local + installed_hash: sha256:1a3cd1f900194eaf4ec86588dd1c3c2bff6a565e742061fc911abdd47bd5f3a5 +- path: scripts/log-convoy-event.sh + source: skills/bootstrap-agent-context/templates/L3-pipeline/_common/log-convoy-event.sh + version: 0.6.0 + installed_hash: sha256:52bdc8f60b18315dd8ad0f1dd6b727106dfa134b8769b0cd63d8698d5865cf21 +- path: scripts/wt.sh + source: skills/bootstrap-agent-context/templates/L3-pipeline/_common/wt.sh + version: 0.5.0 + installed_hash: sha256:2a4f44a159f80a8ea6fe53ac507c01a2f91a4e2118d997a98b051808ac35e9a5 +- path: tests/smoke/app.smoke.spec.ts + source: skills/bootstrap-agent-context/templates/L3-pipeline/nextjs-prisma-vercel/playwright-smoke.spec.ts.template + version: 0.5.0 + installed_hash: sha256:a62c10edb712a61f1cfece43705bfff75a5a66ad6bc8b53f7e69a43c3efb962c diff --git a/.convoys/.metrics.jsonl b/.convoys/.metrics.jsonl index d3091f3..1eacda8 100644 --- a/.convoys/.metrics.jsonl +++ b/.convoys/.metrics.jsonl @@ -75,3 +75,59 @@ {"ts": "2026-06-13T06:20:00Z", "role": "role-architect", "convoy": "scanner-disambiguation-render-test", "repo": "tcg-vault", "skip_flags": [], "classification": "ci", "duration_s": 180, "outcome": "architecture-only"} {"ts": "2026-06-13T06:20:00Z", "role": "role-implementer", "convoy": "scanner-disambiguation-render-test", "repo": "tcg-vault", "skip_flags": [], "classification": "ci", "duration_s": 600, "outcome": "pr-open"} {"ts": "2026-06-14T13:06:36Z", "role": "role-implementer", "convoy": "multi-game-bulk-sync", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 600} +{"ts": "2026-08-14T23:57:39Z", "role": "role-conductor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "classification": "feature", "duration_s": 180, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:14:27Z", "role": "role-ia-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 95, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:15:31Z", "role": "role-ui-designer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:16:36Z", "role": "role-ux-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "scanner-identify-upgrade", "repo": "scanner-identify-upgrade", "skip_flags": [], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "visual", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "visual", "a11y", "design", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:40Z", "role": "role-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 420, "model": "composer-2.5", "model_tier": "standard"} +{"ts": "2026-08-15T00:19:56Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:20:23Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 120, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:22:15Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 3, "duration_s": 600, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:24:15Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 600, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:25:33Z", "role": "role-security-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 480, "model": "gpt-5.6-terra-medium", "model_tier": "security"} +{"ts": "2026-08-15T00:25:54Z", "role": "role-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 210, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T00:26:05Z", "role": "role-a11y-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 78, "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T00:55:56Z", "role": "role-reviewer", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 0, "duration_s": 180, "multitask_group": "audit-tighten-scan-identify-hot-path-uncommitted", "model": "cursor-grok-4.5-high", "model_tier": "fast"} +{"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-ux-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-architect", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 300, "model": "composer-2.5", "model_tier": "standard"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-implementer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 1, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-implementer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 2, "duration_s": 300, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T01:24:41Z", "role": "role-security-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"} +{"ts": "2026-08-15T01:24:41Z", "role": "role-a11y-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 60, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T01:35:27Z", "role": "role-ux-reviewer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:35:27Z", "role": "role-architect", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 360, "model": "composer-2.5", "model_tier": "standard"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 1, "duration_s": 300, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 2, "duration_s": 600, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 3, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-reviewer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-scan-visual-catalog-search-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-security-auditor", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-scan-visual-catalog-search-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"} +{"ts": "2026-08-15T12:41:53Z", "role": "role-conductor", "convoy": "scanner-identify-upgrade", "repo": "tcg-vault", "skip_flags": [], "duration_s": 120, "model": "auto", "model_tier": "auto"} +{"ts": "2026-08-15T21:30:59Z", "role": "role-doc-writer", "convoy": "reconcile-historical-add-scripts", "repo": "tcg-vault", "skip_flags": [], "duration_s": 120, "outcome": "complete", "model": "auto", "model_tier": "auto"} +{"ts": "2026-08-15T21:39:17Z", "role": "role-conductor", "convoy": "dashboard-home-realignment", "repo": "tcg-vault", "skip_flags": ["ui-design"], "classification": "feature", "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:42:03Z", "role": "role-reviewer", "convoy": "dashboard-home-realignment", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "outcome": "comment-only", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T21:42:19Z", "role": "role-ia-architect", "convoy": "dashboard-home-realignment", "repo": "tcg-vault", "skip_flags": [], "duration_s": 120, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:42:20Z", "role": "role-ux-reviewer", "convoy": "dashboard-home-realignment", "repo": "tcg-vault", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:42:20Z", "role": "role-architect", "convoy": "dashboard-home-realignment", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "model": "composer-2.5", "model_tier": "standard"} +{"ts": "2026-08-15T21:42:20Z", "role": "role-implementer", "convoy": "dashboard-home-realignment", "repo": "tcg-vault", "skip_flags": [], "duration_s": 600, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:44:54Z", "role": "role-conductor", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "classification": "feature", "duration_s": 900, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:48:07Z", "role": "role-ia-architect", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "duration_s": 120, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:49:34Z", "role": "role-ui-designer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "duration_s": 300, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:50:19Z", "role": "role-ux-reviewer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "duration_s": 180, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:51:58Z", "role": "role-architect", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "duration_s": 720, "model": "composer-2.5", "model_tier": "standard"} +{"ts": "2026-08-15T21:53:14Z", "role": "role-implementer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "brief": 5, "duration_s": 120, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:53:18Z", "role": "role-implementer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "brief": 2, "duration_s": 120, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:53:18Z", "role": "role-implementer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "brief": 1, "duration_s": 420, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:53:19Z", "role": "role-implementer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "brief": 3, "duration_s": 420, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:53:28Z", "role": "role-implementer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "brief": 4, "duration_s": 420, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:55:19Z", "role": "role-implementer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "brief": 6, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T21:56:43Z", "role": "role-reviewer", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "duration_s": 58, "multitask_group": "audit-scanner-desktop-layout-local", "model": "cursor-grok-4.5-high", "model_tier": "fast"} +{"ts": "2026-08-15T21:56:53Z", "role": "role-security-auditor", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "duration_s": 300, "multitask_group": "audit-scanner-desktop-layout-local", "model": "gpt-5.6-terra-medium", "model_tier": "fast"} +{"ts": "2026-08-15T21:57:11Z", "role": "role-design-system-auditor", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "duration_s": 165, "multitask_group": "audit-scanner-desktop-layout-local", "model": "cursor-grok-4.5-high", "model_tier": "fast"} +{"ts": "2026-08-15T21:57:53Z", "role": "role-a11y-auditor", "convoy": "scanner-desktop-layout", "repo": "scanner-desktop-layout", "skip_flags": [], "duration_s": 120, "multitask_group": "audit-scanner-desktop-layout-local", "model": "cursor-grok-4.5-high", "model_tier": "fast"} diff --git a/.convoys/README.md b/.convoys/README.md index 9790199..37b8f47 100644 --- a/.convoys/README.md +++ b/.convoys/README.md @@ -2,6 +2,8 @@ A **convoy** is a multi-PR work-stream coordinated by an agent pipeline. One convoy = one feature, bug fix, or epic. Each convoy is a Markdown file in this directory plus an optional sub-directory of implementer briefs. +> **Not Cursor Plan mode.** Pipeline convoys live here (`.convoys/*.md`). Cursor's native Plan feature writes to `.cursor/plans/*.plan.md` — a different artifact. For pipeline work, invoke `role-conductor` and write `.convoys/.md`; do not create `.cursor/plans/` files. See `.cursor/rules/convoy-planning.mdc`. + ## File layout ``` @@ -30,6 +32,16 @@ created: model_policy: default_session: auto roles: { ... } # see docs/model-routing-policy.md +design_direction: # optional — set by role-ui-designer (planning lock) + source: role-ui-designer + skill: ui-ux-pro-max + skill_version: "2.5.0" + version: 1 + locked_at: YYYY-MM-DD + product_type: "" + pattern: "" + style: "" + stack: nextjs --- ``` @@ -40,8 +52,9 @@ Body sections (added in order by the pipeline roles): 3. `## Roles invoked` (Conductor) 4. `## Todos` (Conductor → refined by Architect) 5. `## IA` (IA Architect) -6. `## UX` (UX Reviewer) -7. `## Architecture` (Architect) +6. `## Design direction` (UI Designer — optional; skip when `ui-design` set) +7. `## UX` (UX Reviewer) +8. `## Architecture` (Architect) After Architect, briefs live in `.convoys//brief-N-*.md`. Implementers read only their brief, not the whole convoy. @@ -53,12 +66,14 @@ The Conductor sets `skip:` based on classification. These flags map to pipeline | --- | --- | | `ia` | IA Architect | | `ux` | UX Reviewer | +| `ui-design` | UI Designer (planning; `ui-ux-pro-max` skill) | | `arch` | Architect | | `test` | Component tests | | `review` | Reviewer | | `visual` | Visual diff | | `a11y` | A11y auditor | | `design` | Design-system auditor | +| `security` | Security auditor | | `smoke` | Staging smoke | | `qa` | Manual QA | | `docs` | Doc Writer | @@ -92,7 +107,7 @@ See `.cursor/agents/role-conductor.md` for the Conductor's full spec. **Audit fan-out** — after an implementer ships a PR draft: ``` -/multitask role-reviewer + role-design-system-auditor + role-a11y-auditor +/multitask role-reviewer + role-security-auditor + role-design-system-auditor + role-a11y-auditor ``` All three read the same diff and emit independent comments. Use group id `audit--` so analytics can compute wall-clock savings. @@ -109,7 +124,7 @@ See the [multitask playbook](https://github.com/varutasu/agent-pipeline/blob/mai ## Self-analytics -Each L2 role appends one event to `.convoys/.metrics.jsonl` via `scripts/log-convoy-event.sh`. The file is gitignored by default — events stay local. To opt-in to commit team-shared metrics, remove `.convoys/.metrics.jsonl` from `.gitignore`. +Each L2 role appends one event to `.convoys/.metrics.jsonl` via `scripts/log-convoy-event.sh`. **This repo tracks metrics in git** so convoy PRs can prove telemetry was logged (see `.github/workflows/convoy-metrics-gate.yml`). Events contain metadata only — no code, no prompts. Aggregate across repos and render a dashboard with the [agent-pipeline analytics scripts](https://github.com/varutasu/agent-pipeline/tree/main/analytics): diff --git a/.convoys/dashboard-home-realignment.md b/.convoys/dashboard-home-realignment.md new file mode 100644 index 0000000..0d36a63 --- /dev/null +++ b/.convoys/dashboard-home-realignment.md @@ -0,0 +1,266 @@ +--- +name: dashboard-home-realignment +classification: feature +success_metric: | + A logged-in user opening /dashboard sees a flat product nav + (Dashboard, My Collection, Lists, Decks, Scanner, then Cards + + Community), stats sourced from /api/user/stats with no placeholder + tiles, Recently added / My activity / last-card spotlight using + that user's real data, the top-bar notifications + inbox + profile + cluster flush-right, and Admin reachable only from the profile + dropdown (not the left nav). +skip: + - ui-design +status: shipped +shipped: 2026-08-15 +created: 2026-08-15 +model_policy: + default_session: auto + roles: + role-conductor: composer-2.5-fast + role-architect: composer-2.5 + role-ia-architect: composer-2.5-fast + role-ux-reviewer: composer-2.5-fast + role-ui-designer: composer-2.5-fast + role-implementer: composer-2.5-fast + role-reviewer: cursor-grok-4.5-high + role-security-auditor: gpt-5.6-terra-medium + role-design-system-auditor: cursor-grok-4.5-high + role-a11y-auditor: cursor-grok-4.5-high + role-doc-writer: auto + escalate_to: claude-sonnet-5-thinking-medium + escalate_to_premium: claude-4.6-opus-high-thinking + never_premium: + - role-reviewer + - role-security-auditor + - role-design-system-auditor + - role-a11y-auditor + - role-ui-designer + - role-doc-writer +--- + +# Convoy: dashboard-home-realignment + +Make `/dashboard` an honest home for returning collectors, flatten +the sidebar to match the product we actually have, and fix top-bar +chrome (right-aligned actions + admin only in the profile menu). + +## Why + +The `redesign-v2-from-mockups` epic shipped the visual shell (glass +sidebar, TopSearchBar, StatCard row, featured grid, activity panel, +spotlight rail). The screen still does not behave like a home: + +- Sidebar IA is nested and mislabeled. `/dashboard` hides under a + "My Collection" parent; ownership (`/my-cards`) is a sub-item + named "Cards"; Lists / Decks / Analytics nest underneath. The + prototype's Wishlist / Trades / Market / Events items do not + exist as product — adding them as "Coming soon" would be + graveyard nav. +- Dashboard stats sum list `cardCount` / `value` (double-counts + cards in multiple lists) and show dead **Rare Cards** / **Wishlist + Items** tiles at `0` / "Coming soon". `/api/user/stats` already + has ownership totals, value, decks, rarity breakdown, and recent + adds — the page does not call it. +- Featured Collection / Recent Activity / Card Spotlight mix real + thumbnails with fake trades, fake market charts, and a demo + Emberclaw Dragon. That trains people the home is marketing. +- TopSearchBar search uses `flex-1 max-w-2xl`, so notifications, + inbox, and profile sit mid-bar on wide viewports instead of + flush-right. +- Admins see **Admin Tools** in the left nav (`/admin/card-editor`) + *and* **Admin Panel** in the profile dropdown (`/admin`). Operator + wants a single entry: the profile menu. + +`/my-cards` stays a browse/manage page (not a second dashboard). +This convoy only promotes it in nav and aligns chrome. + +## Scope + +### In scope + +- **Flatten desktop + mobile nav.** First-class: Dashboard + (`/dashboard`), My Collection (`/my-cards`), Lists + (`/collections`), Decks (`/decks`), Scanner (`/scanner`). Below a + divider: Cards catalog (`/cards`), Community (existing collapse). + Keep locked vocab (`My Collection` / `Lists`). Do not add + Wishlist, Trades, Market, Events, Analytics, or a top-level + Activity item. +- **Remove left-nav Admin Tools.** Keep admin entry in + `TopSearchBar` `` (already gated on `user.role === + 'admin'`). Architect picks one href (`/admin` vs + `/admin/card-editor`) and applies it in the profile menu + any + leftover mobile-drawer copy of `UserProfileDropdown`. Do not + resurrect admin in `NavigationContent`. +- **Right-align top-bar actions.** Notifications, inbox, and + profile cluster flush to the right edge of `TopSearchBar`. Search + stays left / grows; leftover width after `max-w-2xl` must not + sit to the right of the cluster. Hide the notification badge + when count is `0` (do not invent a fake `3`). +- **Honest dashboard stats.** Fetch `/api/user/stats`. Ship three + real tiles: Total Cards (from `user_cards`, not list sums), + Collection Value, and a third computable metric (This month / + Lists / Decks — IA + Architect lock). Hide Rare Cards and + Wishlist until those features exist. Use `delta` / `subtitle` + only when the number is real. +- **Honest home panels.** Rename Featured Collection → Recently + added (same 8-up + empty-slot CTAs, real `/api/user-cards` or + stats recent rows). Wire Recent Activity to the signed-in user's + adds (stats `recentActivity` is an acceptable v1). Spotlight the + last added / last scanned card with real metadata; hide + price-trend, market-overview, and watchlist until a market-data + convoy. +- **Home CTAs.** Keep Scan / Create List only for empty or new + collections, or drop the welcome header in favor of stats — + UX locks this. Do not keep both a long greeting *and* two + always-on buttons. +- **Collection page chrome only.** Same max-width / padding rhythm + as dashboard; optional one-line summary from stats. No second + stat row, no spotlight rail on `/my-cards`. + +### Out of scope + +- Wishlist / Trades / Market / Events product work (queued + follow-ups from `redesign-v2-from-mockups`, still queued). +- Daily Ember backend, Level / XP, federated ⌘K search. +- Renaming Lists → Binders. +- Real notification / inbox backends. +- Market-price history or a charting library. +- Redesigning `CardItem`, scanner, or community pages. +- Changing `/api/user/stats` auth or adding new tables. + +## Roles invoked + +1. `role-ia-architect` — lock the flat nav map (desktop + mobile), + dashboard information hierarchy, and Collection-page-as-browse + (not a second home). +2. `role-ux-reviewer` — empty-state CTAs vs greeting, third stat + tile, activity/spotlight honesty, top-bar alignment + badge + rules. `ui-design` skipped: incremental inside Liquid Glass; + no new visual language. +3. `role-architect` — briefs for nav + admin relocation, TopSearchBar + alignment, dashboard data wiring (reuse `/api/user/stats`, no + new tables). +4. `role-implementer` — per brief. +5. Audit fan-out after PR draft: reviewer + security-auditor + + design-system-auditor + a11y-auditor. +6. `role-doc-writer` — AGENTS.md / Layout convention note if nav + map changes. + +## Todos + +- [x] IA: publish the locked nav map and dashboard outline +- [x] UX: lock greeting/CTA rule, third stat, badge-at-zero, spotlight contents +- [x] Architect: split briefs (nav+admin, top-bar, dashboard data) +- [x] Flatten `NavigationContent` + `MobileNavigation`; drop left-nav Admin Tools +- [x] Right-align TopSearchBar action cluster; badge hidden at 0 +- [x] Dashboard consumes `/api/user/stats`; hide placeholder tiles +- [x] Recently added / My activity / last-card spotlight use real user data +- [x] `/my-cards` padding / max-width aligned with dashboard +- [x] Audits + docs pass + +## IA + +**Affected routes** + +- `[modified]` `/dashboard` — honest home; stats from `/api/user/stats` +- `[modified]` `/my-cards` — summary line + shared max-width container +- `[impacted]` `/collections`, `/collection/[id]`, `/decks`, `/deck/*`, `/scanner`, `/cards`, `/community/*` — nav active states only + +**User flow** + +```mermaid +flowchart LR + Login --> Dashboard + Dashboard --> MyCards["My Collection"] + Dashboard --> Scanner + Dashboard --> Lists + MyCards --> CardDetail["/card/:id"] +``` + +**Screen inventory** + +| Screen | Path | Change | Notes | +| --- | --- | --- | --- | +| Dashboard home | `/dashboard` | Modified | 3 stat tiles, recently added, activity, latest card | +| My Collection | `/my-cards` | Modified | Browse surface; optional stats summary | +| Layout chrome | all auth pages | Modified | Flat nav; admin in profile menu only | + +**Content / data model deltas** + +- No schema changes. Reuse `GET /api/user/stats` and `GET /api/user-cards`. +- `user-cards` GET adds `market_price` to SELECT for spotlight display. + +**Open IA questions** + +- None — third stat locked to **Decks** (`totalDecks`). + +## UX + +**Decisions (locked)** + +1. **Empty vs returning home:** Welcome + Scan/Browse CTAs only when stats load successfully *and* both `totalCards === 0` and no recent cards. Returning users see stats row only (no greeting strip). +2. **Stat row:** Three tiles — Total cards, Collection value, Decks. No Rare/Wishlist placeholders. +3. **Panels:** Featured → **Recently added**; activity from user's `recentActivity`; spotlight shows latest owned card with real fields only (no demo charts). +4. **Top bar:** Notifications badge hidden at `0`. Action cluster flush-right via `ml-auto`. +5. **Admin:** Single **Admin Tools** entry in profile dropdown (`/admin`); removed from sidebar. + +## Architecture + +**File plan** + +| File | Action | Purpose | +| --- | --- | --- | +| `components/Layout.js` | Modified | Flat nav; remove sidebar admin | +| `components/MobileNavigation.js` | Modified | Dashboard hub + Collection/Scanner/Decks | +| `components/ui/TopSearchBar.js` | Modified | Right-align actions; Admin Tools label | +| `pages/dashboard.js` | Modified | Wire `/api/user/stats` | +| `components/Dashboard*.js` | Modified | Real data panels | +| `pages/my-cards.js` | Modified | Container + summary | +| `lib/format-relative-time.js` | New | Activity timestamps | +| `pages/api/user-cards.js` | Modified | Include `market_price` in GET | + +**API surface** + +- No new routes. Consumers: `GET /api/user/stats`, `GET /api/user-cards` (auth required). + +**Schema diff** + +- None. + +**Test plan** + +- `test/lib/format-relative-time.test.js` (new) +- Existing `Layout.test.js`, `StatCard.test.js` regression + +**Decomposition** + +| Brief # | Title | Files | Depends on | +| --- | --- | --- | --- | +| 1 | Flat nav + admin relocation | Layout, MobileNavigation, TopSearchBar | — | +| 2 | Honest dashboard home | dashboard.js, Dashboard*.js, format-relative-time | — | +| 3 | My Collection chrome | my-cards.js, user-cards API | — | + +```yaml +slice_dependencies: + - brief: 1 + depends_on: [] + files: [components/Layout.js, components/MobileNavigation.js, components/ui/TopSearchBar.js] + - brief: 2 + depends_on: [] + files: [pages/dashboard.js, components/DashboardFeaturedCollection.js, components/DashboardRecentActivity.js, components/DashboardCardSpotlight.js, lib/format-relative-time.js] + - brief: 3 + depends_on: [] + files: [pages/my-cards.js, pages/api/user-cards.js] +``` + +## Audits (2026-08-15) + +- **Reviewer:** No blockers after fixes (stats empty-state gate, divider, Lists active on `/collection/*`, vocab, admin label). +- **Security:** No medium+ findings; auth boundaries unchanged. +- **Design-system / a11y:** Incremental Liquid Glass; nav items retain focus rings; activity list uses semantic text. + +## Doc note + +Authenticated sidebar nav (2026-08-15): Dashboard → My Collection → Lists → Decks → Scanner; divider; Cards catalog + Community. Admin only in TopSearchBar profile menu. + diff --git a/.convoys/dashboard-home-realignment/brief-1-flat-nav-and-top-bar.md b/.convoys/dashboard-home-realignment/brief-1-flat-nav-and-top-bar.md new file mode 100644 index 0000000..4d10b29 --- /dev/null +++ b/.convoys/dashboard-home-realignment/brief-1-flat-nav-and-top-bar.md @@ -0,0 +1,25 @@ +--- +convoy: dashboard-home-realignment +brief_number: 1 +depends_on: [] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - components/Layout.js + - components/MobileNavigation.js + - components/ui/TopSearchBar.js +--- + +# Brief 1: Flat nav + top bar alignment + +## Goal + +Flatten authenticated navigation and right-align TopSearchBar actions; admin only in profile menu. + +## Acceptance criteria + +- [x] Flat primary nav with Dashboard, My Collection, Lists, Decks, Scanner +- [x] Secondary: Cards (+ Scanner when logged out), Community collapsible +- [x] No sidebar Admin Tools +- [x] TopSearchBar actions flush-right (`ml-auto`) +- [x] Admin Tools in UserMenu only diff --git a/.convoys/dashboard-home-realignment/brief-2-honest-dashboard.md b/.convoys/dashboard-home-realignment/brief-2-honest-dashboard.md new file mode 100644 index 0000000..7353618 --- /dev/null +++ b/.convoys/dashboard-home-realignment/brief-2-honest-dashboard.md @@ -0,0 +1,27 @@ +--- +convoy: dashboard-home-realignment +brief_number: 2 +depends_on: [] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - pages/dashboard.js + - components/DashboardFeaturedCollection.js + - components/DashboardRecentActivity.js + - components/DashboardCardSpotlight.js + - lib/format-relative-time.js + - test/lib/format-relative-time.test.js +--- + +# Brief 2: Honest dashboard home + +## Goal + +Wire dashboard to `/api/user/stats` and remove demo panels/placeholder stat tiles. + +## Acceptance criteria + +- [x] Three real stat tiles (total cards, value, decks) +- [x] Recently added grid + real activity + latest card spotlight +- [x] Empty-state CTAs only when collection is truly empty +- [x] formatRelativeTime helper + unit test diff --git a/.convoys/dashboard-home-realignment/brief-3-my-cards-chrome.md b/.convoys/dashboard-home-realignment/brief-3-my-cards-chrome.md new file mode 100644 index 0000000..15db8f4 --- /dev/null +++ b/.convoys/dashboard-home-realignment/brief-3-my-cards-chrome.md @@ -0,0 +1,22 @@ +--- +convoy: dashboard-home-realignment +brief_number: 3 +depends_on: [] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - pages/my-cards.js + - pages/api/user-cards.js +--- + +# Brief 3: My Collection chrome alignment + +## Goal + +Align `/my-cards` layout with dashboard container and show optional ownership summary. + +## Acceptance criteria + +- [x] `max-w-[1500px] mx-auto` container +- [x] One-line summary from `/api/user/stats` +- [x] `market_price` on user-cards GET for downstream spotlight diff --git a/.convoys/improve-scan-card-detection.md b/.convoys/improve-scan-card-detection.md new file mode 100644 index 0000000..fb4722d --- /dev/null +++ b/.convoys/improve-scan-card-detection.md @@ -0,0 +1,218 @@ +--- +name: improve-scan-card-detection +classification: feature +success_metric: | + After ship, L2 result_kind=not_a_card ≤15% of L2 (was 44.6%) on a + comparable scan_attempts window; Layer-1 escalate of L1 ≤ the Phase 1 + post-ship rate (better crops should not regress it). +skip: + - ia + - ui-design + - flag +status: shipped +created: 2026-08-14 +depends_on: + - tighten-scan-identify-hot-path +umbrella: scanner-identify-upgrade +model_policy: + default_session: auto + roles: + role-conductor: composer-2.5-fast + role-architect: composer-2.5 + role-ia-architect: composer-2.5-fast + role-ux-reviewer: composer-2.5-fast + role-ui-designer: composer-2.5-fast + role-implementer: composer-2.5-fast + role-reviewer: cursor-grok-4.5-high + role-security-auditor: gpt-5.6-terra-medium + role-design-system-auditor: cursor-grok-4.5-high + role-a11y-auditor: cursor-grok-4.5-high + role-doc-writer: auto + escalate_to: claude-sonnet-5-thinking-medium + escalate_to_premium: claude-4.6-opus-high-thinking + never_premium: + - role-reviewer + - role-security-auditor + - role-design-system-auditor + - role-a11y-auditor + - role-ui-designer + - role-doc-writer +--- + +# Convoy: improve-scan-card-detection + +Phase 2 of `scanner-identify-upgrade`. Replace the hand-rolled 320×240 +Sobel + aspect-ratio hunt with a detector that returns a **warped, +axis-aligned card crop**. Queued since `add-real-ocr-layer` (2026-05-27) +and never opened; telemetry now justifies it: **44.6% of Gemini calls +are `not_a_card`**. + +`depends_on: tighten-scan-identify-hot-path` is a measurement +dependency (do not retune Phase 1 constants in this PR). File sets are +otherwise disjoint — Architect may mark this parallel with Phase 1 if +Phase 1 has already locked the new gate values. + +## Why + +`detectCardShapesFromFrame` is not OpenCV (despite the hook comment). +It downscales to 320×240, runs a Sobel magnitude threshold, then a +nested box search for aspect 0.63–0.77. There is no four-corner +homography. Crooked, foil, or off-center cards produce junk name +strips (L1 escalate text is 3–20+ chars of noise) and wasted L2 calls. + +A better crop improves Tesseract **and** Gemini without changing +either model. + +## Scope + +### In scope + +- Replace or wrap `detectCardShapesFromFrame` so tracked bounds are + card-shaped **and** a perspective-corrected JPEG can be produced for + identify. +- Architect picks **one** in-browser approach (do not land both): + 1. **OpenCV.js contours + approxPolyDP + warpPerspective** — no + training, larger WASM. + 2. **YOLO11n (or similar) ONNX in-browser (~5MB) + warp** — better + on video; needs a card-detection weight file hosted (Blob or + `/public`, license-clean). +- Keep the existing tracker merge + (`mergeDetectedShapesIntoTrackedCards`, overlap, stale 3s). +- Overlay brackets in `components/scanner/ScannerCamera.js` keep + consuming `{x,y,width,height}` (or four corners mapped to a rect). + Do not redesign the overlay. +- Unit tests for warp math / tracker merge; add a fixture crop test + if Architect wants a checked-in card JPEG. + +### Out of scope + +- Identify / Gemini / Tesseract / catalog match (Phase 1 + 3). +- Python/CUDA microservice, PaddleOCR server, Roboflow-hosted detect. +- Scanner chrome / cart (`scanner-mobile-checkout`). +- Training our own detector from scratch unless a public TCG-card + weight with a clear license is documented in the brief. + +## Roles invoked + +1. `role-ux-reviewer` — time-to-bracket, false-positive boxes, dual-card + frames. +2. `role-architect` — OpenCV.js vs YOLO11n; WASM load strategy + (Turbopack + Next 16); where weights live; 1–2 briefs. +3. `role-implementer`. +4. Audit: reviewer + security-auditor + a11y-auditor (overlay still + needs labels). Design-system only if overlay styling changes. + +## Todos + +- [x] Architect: pick detector; document WASM / weight budget +- [x] Brief 1 — detect + warp library + tests +- [x] Brief 2 — wire `use-camera-scanner.js` + crop used by + `identifyTrackedCardCapture` +- [x] Confirm `scanner-mobile-checkout` overlay still maps bounds +- [x] Re-measure L2 `not_a_card` share (2026-08-15 — 43.9% all-time; 38.9% on Aug 15 n=26) + +## Post-ship (PR #158, 2026-08-14) + +**Shipped:** perspective warp + `scanner-card-warp.js`; warped JPEG feeds OCR +and vision paths. + +**Telemetry:** L2 `not_a_card` 43.9% all-time vs 44.6% baseline — flat. +Aug 15 burst (n=26) showed 38.9% (directional improvement only). L1 OCR +quality regressed (shorter garbage strips). + +**Verdict:** Better crops shipped; Gemini still sees bad frames often. +Combined with Phase 3 backfill for printing-accurate matches. + +## Likely file ownership + +| Area | Files | +| --- | --- | +| Detect | `lib/scanner-card-detection.js`, `test/lib/scanner-card-detection.test.js` | +| Warp helper | new `lib/scanner-card-warp.js` (if Architect splits) | +| Camera loop | `lib/use-camera-scanner.js` | +| Crop consumer | `lib/scanner-card-identify.js` (`captureCardRegionFromVideo`) | +| Overlay | `components/scanner/ScannerCamera.js` only if bounds shape changes | + +Do not edit `lib/ocr-worker.js` or `lib/scan-vision.js` here. + +## Multitask dispatch + +Serial unless Architect splits detect-lib vs camera-wire with disjoint +files. + +Audit group id: `audit-improve-scan-card-detection-`. + +## CI impact + +| Workflow / job | Behavior | +| --- | --- | +| `visual-diff.yml` | **May fire** if `ScannerCamera.js` changes | +| `preview-smoke.yml` | Fires | +| `ci.yml` test | New/updated detection tests | + +## Operator action + +If YOLO weights are used: confirm license + host on Vercel Blob or +`public/` (cache-Control immutable). No new secrets. + +## Conductor notes + +Do not drop OpenCV.js in only to reimplement the current rectangle +hunt. Success is a **rectified card image**, not a prettier box. +This convoy unblocks Phase 3 — embeddings on unwarped phone photos +will miss. + +## UX + +No new routes or screens. Brackets still render from axis-aligned +`bounds`; the user sees the same overlay while the backend crop +becomes perspective-corrected. + +### Timing + +- Bracket appearance unchanged (200ms detect interval, 800ms verify gate + from Phase 1). +- Warp adds ~20–40ms on capture only — not on the detect loop. + +### Dual-card frames + +Tracker merge behavior unchanged: overlapping boxes collapse to one +tracked card. Warp runs per tracked card at verify time. + +### Failure modes + +When corner refinement fails validation, fall back to the axis-aligned +margin crop (same as pre-Phase-2). No new error toast. + +## Architecture + +### Decision D1 — Pure-JS contour corners + homography warp (not OpenCV.js WASM) + +OpenCV.js adds ~8MB WASM and a Turbopack dynamic-import footgun. +Instead: keep the 320×240 Sobel edge map, refine four corners per +candidate bbox via quadrant edge search, validate with +`isValidCardQuad`, and warp with a small homography helper. + +Rejected: YOLO11n ONNX (weight hosting + license review), OpenCV.js +(full WASM budget). + +### Decision D2 — Corners ride on tracked cards + +`mergeDetectedShapesIntoTrackedCards` stores `corners` alongside +`bounds`. Overlay continues to use `bounds` only. + +### Decision D3 — Warp at capture time only + +`captureCardRegionFromVideo` calls `warpCardCaptureFromVideo` when +four video-space corners exist; otherwise axis-aligned crop. + +### slice_dependencies + +| Brief | depends_on | files | +| --- | --- | --- | +| 1 detect + warp | [] | `lib/scanner-card-warp.js`, `lib/scanner-card-detection.js`, tests | +| 2 wire capture | [1] | `lib/scanner-card-identify.js`, `lib/use-camera-scanner.js`, `ScannerCamera.js` | + +Serial implement: Brief 1 → Brief 2. + +Audit group id: `audit-improve-scan-card-detection-`. diff --git a/.convoys/improve-scan-card-detection/brief-1-detect-warp-lib.md b/.convoys/improve-scan-card-detection/brief-1-detect-warp-lib.md new file mode 100644 index 0000000..dd57282 --- /dev/null +++ b/.convoys/improve-scan-card-detection/brief-1-detect-warp-lib.md @@ -0,0 +1,27 @@ +--- +convoy: improve-scan-card-detection +brief_number: 1 +depends_on: [] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/scanner-card-warp.js + - lib/scanner-card-detection.js + - test/lib/scanner-card-warp.test.js + - test/lib/scanner-card-detection.test.js +--- + +# Brief 1: Quad corner detection + perspective warp + +## Goal + +Replace axis-only Sobel box hunt with four-corner refinement and a +homography warp helper that produces rectified card JPEGs. + +## Acceptance criteria + +- [ ] `lib/scanner-card-warp.js` exports `orderQuadCorners`, + `computeHomography`, `warpCardCaptureFromVideo` +- [ ] `detectCardShapesFromFrame` returns `corners` in video space +- [ ] Tracker merge preserves `corners` on tracked cards +- [ ] Unit tests for warp math + corner refinement diff --git a/.convoys/improve-scan-card-detection/brief-2-wire-capture.md b/.convoys/improve-scan-card-detection/brief-2-wire-capture.md new file mode 100644 index 0000000..1a6f554 --- /dev/null +++ b/.convoys/improve-scan-card-detection/brief-2-wire-capture.md @@ -0,0 +1,25 @@ +--- +convoy: improve-scan-card-detection +brief_number: 2 +depends_on: [1] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/scanner-card-identify.js + - lib/use-camera-scanner.js + - components/scanner/ScannerCamera.js +--- + +# Brief 2: Wire warped capture into identify + overlay a11y + +## Goal + +Use perspective-corrected crops for L1/L2 identify; keep overlay on +axis-aligned bounds; add live-region label on detection brackets. + +## Acceptance criteria + +- [ ] `identifyTrackedCardCapture` passes `cardTracker.corners` to capture +- [ ] `captureCardRegionFromVideo` warps when corners present, else fallback +- [ ] `use-camera-scanner.js` comment no longer claims OpenCV +- [ ] `DetectionFrame` exposes `role="status"` + `aria-label` diff --git a/.convoys/migrate-neon-to-homelab.md b/.convoys/migrate-neon-to-homelab.md new file mode 100644 index 0000000..159ea05 --- /dev/null +++ b/.convoys/migrate-neon-to-homelab.md @@ -0,0 +1,97 @@ +--- +name: migrate-neon-to-homelab +classification: infra +success_metric: | + Deck Hearth runs on CT 102 Postgres + MinIO + Redis, app on Dokploy (CT 112); + Neon and Vercel hosting decommissioned; Vercel AI Gateway retained. +status: shipped +created: 2026-08-15 +depends_on: [] +skip: + - ia + - ui-design + - ux + - visual + - a11y + - design + - flag +--- + +# Convoy: migrate-neon-to-homelab + +Move Deck Hearth from Neon (free tier full ~490 MB) to **CT 102 Postgres** +on the axiom homelab. Supabase path rejected — operator already has pgvector +Postgres + Coolify on the LAN. + +## Why homelab fits + +| Asset | CT 102 | +| --- | --- | +| Postgres | `pgvector/pgvector:pg17`, port 5432 | +| Disk | `/apps` ZFS mirror — not capped at 512 MB | +| CI | `deckhearth_ci` already used by migrate job | +| App host | Dokploy on CT 112 (Traefik on CT 100) | + +## Blocker: Vercel ↔ private IP + +Vercel cannot connect to `192.168.68.102`. Production cutover requires +**Dokploy deploy** (`deckhearth.stillwell.cloud`) on CT 112. + +## Phases + +| Phase | Work | Owner | Status | +| --- | --- | --- | --- | +| 1 | `axiom-server`: `ct102/init/03-deckhearth.sql` | operator | ready | +| 2 | `lib/sql.js` + import swap off `@vercel/postgres` | code | done | +| 3 | `docs/HOMELAB_DATABASE.md` runbook | code | done | +| 4 | Operator: provision DB, `npm run migrate up` | operator | done | +| 5 | MinIO + Redis wiring, Dockerfile, Dokploy docs | code | done | +| 6 | `npm run migrate-neon-to-homelab` data copy | operator | pending | +| 7 | Dokploy app + Traefik route | operator | in_progress | +| 8 | Decommission Neon + Vercel | operator | pending | + +## Env contract + +```bash +POSTGRES_URL=postgresql://deckhearth:…@192.168.68.102:5432/deckhearth +POSTGRES_URL_DIRECT=… # same on homelab +NEON_DATABASE_URL=… # one-time source only +``` + +## axiom-server changes + +- `proxmox/ct102/init/03-deckhearth.sql` — `deckhearth` + pgvector + `deckhearth_ci` +- `.cursor/rules/ct102-databases.mdc` — table row + +## tcg-vault changes + +- `lib/sql.js`, `scripts/migrate-neon-to-homelab.js` +- `docs/HOMELAB_DATABASE.md` + +## Risks + +| Risk | Mitigation | +| --- | --- | +| LAN-only DB | ✅ RESOLVED — app now on Dokploy CT 112, DB on CT 102 same LAN | +| `deckhearth_ci` password drift | Match `HOMELAB_CI_POSTGRES_PASSWORD` GitHub secret | +| Init SQL on live CT 102 | Manual `docker exec psql` apply, not initdb.d replay | + +## As-shipped + +**Phase 1–5 (code):** Shipped. CT 102 Postgres + pgvector provisioned; `lib/sql.js` +uses the `postgres` package with `POSTGRES_URL`; Dockerfile present; Dokploy +runbook complete; CI already gates against `deckhearth.stillwell.cloud`. + +**Phase 6 (data copy):** Pending operator action. Run +`npm run migrate-neon-to-homelab` once — copies live Neon data to CT 102 via +`pg_dump` → `pg_restore`. Script is production-ready; see `scripts/migrate-neon-to-homelab.js`. +If starting fresh with no Neon data, skip this step. + +**Phase 7 (Dokploy app):** Live. Smoke + visual CI workflows hit +`https://deckhearth.stillwell.cloud` and return 200. Dokploy app + Traefik +route confirmed operational. + +**Phase 8 (decommission):** Pending operator action. Steps in +`docs/DOKPLOY_DEPLOY.md` § 6. Data copy (phase 6) should run before the +Vercel project is deleted. `vercel.json` and `.vercel/` are removed from the +tree; remaining decommission is a Vercel dashboard operation. diff --git a/.convoys/reconcile-historical-add-scripts.md b/.convoys/reconcile-historical-add-scripts.md new file mode 100644 index 0000000..7635f2f --- /dev/null +++ b/.convoys/reconcile-historical-add-scripts.md @@ -0,0 +1,790 @@ +--- +name: reconcile-historical-add-scripts +classification: quality +priority: P1 (last open P1 in launch queue) +success_metric: | + A brand-new Neon branch can be onboarded by `npm install` → + `npm run setup-db` alone. After `setup-db` exits 0, the fresh-env + schema (`information_schema.tables` + `.columns` + `.table_constraints` + + `pg_indexes`) is structurally equivalent to prod for every + table/column/index/constraint that runtime code in `pages/api/**`, + `lib/**`, `scripts/**`, and the 4 prior post-backfill migrations + depend on. `.convoys/ship-readiness.md` § "Queued convoys" → + `reconcile-historical-add-scripts` flips from open → RESOLVED; + `retire-graveyard-scripts-after-audit` unblocks as a follow-up. +skip: + - role-design-system-auditor + - role-a11y-auditor + - role-ux-reviewer + - role-ia-architect +status: shipped +created: 2026-06-14 +parent: migration-tool +addresses: migration-tool § R1 (prod schema drift from setup-neon-db.js DDL) +depends_on: + - migration-tool (PR #32 — provides `node-pg-migrate` + initial backfill) +--- + +# Convoy: reconcile-historical-add-scripts + +Fold the effects of the 13 historical `scripts/add-*.js` / `scripts/fix-*.js` +/ `scripts/seed-*.js` jobs into the migration history so a brand-new Neon +branch can be onboarded by `npm install` → `npm run setup-db` alone, +without manually replaying the historical scripts. Closes the last gap +identified by `migration-tool` § R1 ("Prod schema drift from +`setup-neon-db.js` DDL"). + +## Background — the graveyard residue + +`migration-tool` (PR #32, 2026-05-26) adopted `node-pg-migrate` and +captured `scripts/setup-neon-db.js`'s 7-table bootstrap DDL into +`migrations/1779853647564_initial-schema.js`. Four post-backfill +migrations followed (`add-pg-trgm-card-name-index`, `add-scan-tables`, +`add-user-cards-scan-image-url`, `system-collection-description`). The +migration history's current shape is: + +| Migration | Captures | +| --- | --- | +| `1779853647564_initial-schema` | The 7 bootstrap tables: `users`, `cards`, `user_cards`, `collections`, `collection_cards`, `decks`, `deck_cards` | +| `1779853647565_add-pg-trgm-card-name-index` | `pg_trgm` extension + `idx_cards_name_trgm` GIN | +| `1779853647566_add-scan-tables` | `card_submissions`, `scan_attempts`, 2 indexes | +| `1779908094455_add-user-cards-scan-image-url` | `user_cards.scan_image_url` column | +| `1780378340194_system-collection-description` | `collections.is_system_collection` column + description text backfill | + +`migration-tool` § R1 documented that the bootstrap shape captured by +the initial backfill is **not** the full prod shape. Between +`setup-neon-db.js` and prod-today, 13 historical scripts have added +columns, tables, indexes, constraints, and one-shot data migrations +that are baked into every long-lived environment but are NOT +reproduced by `npm run migrate up` on a brand-new Neon branch. Code +in `pages/api/**`, `lib/**`, and the schema-aware test fixtures all +assume the post-historical shape; a fresh-env onboarding therefore +breaks the first time runtime code touches an uncaptured column or +table (`user_favorites`, `collection_permissions`, `user_settings`, +`user_avatars`, `users.first_name`, `collections.visibility`, etc.). + +This convoy reconstructs the missing migration history by reading +each historical script's SQL, classifying it, and (where the DDL is +not already captured) writing a new idempotent `node-pg-migrate` +migration that brings fresh envs to parity with prod. Historical +scripts themselves are NOT edited — they remain no-go-zones per +`.cursor/rules/no-go-zones.mdc`. After this convoy lands, the queued +`retire-graveyard-scripts-after-audit` (P3) unblocks. + +## Inventory (13 scripts audited) + +Legend: +- **C** = captured (DDL already present in `migrations/`) +- **M** = missed (DDL exists in prod via the script but no migration captures it) +- **DML-only** = no DDL; either pure data migration or dev-fixture seed +- **Mixed** = DDL captured but accompanying DML backfill not captured + +### add-* (9) + +| # | Script | DDL summary | DML summary | Classification | Captured by | +| --- | --- | --- | --- | --- | --- | +| 1 | `add-card-columns.js` | `ALTER TABLE cards ADD COLUMN IF NOT EXISTS quantity INTEGER DEFAULT 0`; `ALTER TABLE cards ADD COLUMN IF NOT EXISTS favorited BOOLEAN DEFAULT false` | — | **M** | none — neither column is in `initial-schema`'s `CREATE TABLE cards`. Both flagged as "unused" smells in `docs/SCHEMA_MAP.md` § "Known schema smells" #3 | +| 2 | `add-collaboration-features.js` | `ALTER collections ADD visibility VARCHAR(20) DEFAULT 'private', tcg VARCHAR(50) DEFAULT 'MTG', tags TEXT`; `CREATE TABLE collection_permissions` (incl. role/status CHECKs + invite_token UNIQUE + `UNIQUE(collection_id, user_id)`); `CREATE TABLE collection_activity` (`details JSONB`); `ALTER users ADD is_pending BOOLEAN DEFAULT false`; 4 indexes | One-time owner-permission backfill: `INSERT INTO collection_permissions ... 'owner', 'active'` for every existing collection lacking one | **M** | none — `initial-schema` only has the 3-column `collections` bootstrap and no `collection_permissions` / `collection_activity` | +| 3 | `add-collection-slugs.js` | `ALTER collections ADD slug VARCHAR(100) UNIQUE`; `CREATE UNIQUE INDEX idx_collections_slug`; `ALTER ... ADD CONSTRAINT check_slug_format CHECK (slug ~ '^[a-z0-9]([a-z0-9-]*[a-z0-9])?$' AND length(slug) <= 50)` | Per-row UPDATE to backfill slugs from `name` via `lib/slug-utils.js::generateUniqueSlug` | **M** | none | +| 4 | `add-favorites-system.js` | `CREATE TABLE user_favorites (id, user_id FK CASCADE, item_type VARCHAR(50), item_id INTEGER, created_at, UNIQUE(user_id, item_type, item_id))`; 4 indexes | — | **M** | none | +| 5 | `add-image-column.js` | `ALTER collections ADD image TEXT` | — | **M** | none | +| 6 | `add-system-collection-column.js` | `ALTER collections ADD is_system_collection BOOLEAN DEFAULT false` | Per-user backfill: creates one `'All My Cards'` system collection + owner permission row for every user lacking one | **Mixed** | `1780378340194_system-collection-description.js` captures the DDL (`ADD COLUMN IF NOT EXISTS is_system_collection`). The per-user backfill DML is genuinely captured by the user-registration hook at `pages/api/auth/register.js:97-119` (`INSERT INTO collections ... is_system_collection=true` on every new user — verified by parent agent post-architect-pass, 2026-06-14). No further work required; see Finding 4 RESOLVED below. | +| 7 | `add-updated-at-column.js` | `ALTER cards ADD updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP` | — | **C** | `1779853647564_initial-schema.js` — `cards` `CREATE TABLE` already includes `updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP` (line 67). Script is now a no-op on fresh envs. No work required. | +| 8 | `add-user-profile-columns.js` | `ALTER users ADD first_name, last_name VARCHAR(255), username VARCHAR(255) UNIQUE, profile_image_url TEXT` | Per-row UPDATE: defaults `first_name='User'`, `last_name=`, `username='_'` | **M** (overlaps #9 — see Drift findings) | +| 9 | `add-user-profile-fields.js` | `ALTER users ADD first_name, last_name VARCHAR(255), username VARCHAR(255) UNIQUE, bio TEXT, avatar_url TEXT`; `ALTER users ADD favorite_games JSONB DEFAULT '["MTG"]', collection_visibility VARCHAR(20) DEFAULT 'private', preferred_currency VARCHAR(3) DEFAULT 'USD', cards_per_page INTEGER DEFAULT 50, default_view VARCHAR(10) DEFAULT 'grid'`; `ALTER users ADD notifications_email BOOLEAN DEFAULT true, notifications_marketing BOOLEAN DEFAULT false, two_factor_enabled BOOLEAN DEFAULT false`; `ALTER users ADD theme VARCHAR(10) DEFAULT 'system', language VARCHAR(5) DEFAULT 'en'`; `CREATE TABLE user_settings (id, user_id FK CASCADE, setting_key VARCHAR(100), setting_value JSONB, UNIQUE(user_id, setting_key))`; `CREATE TABLE user_avatars (id, user_id FK CASCADE, filename, original_name, mime_type, file_size, file_path, is_active BOOLEAN DEFAULT true)`; 6 CHECK constraints (`check_collection_visibility`, `check_preferred_currency`, `check_cards_per_page`, `check_default_view`, `check_theme`, `check_language`); 6 indexes | UPDATE on users to write defaults for any row with NULLs in the new columns | **M** | + +### fix-* (2) + +| # | Script | DDL summary | DML summary | Classification | Captured by | +| --- | --- | --- | --- | --- | --- | +| 10 | `fix-lorcana-images.js` | — | UPDATE on `cards WHERE game='Lorcana' AND image_url LIKE '%-716.webp' OR '%-512.webp'`: rewrite to `-1024.webp` for `image_url`, keep small as `stock_image_url` | **DML-only** | n/a — content-level data fix tied to a specific historical image-CDN payload shape. Re-running it on a fresh env that imports Lorcana via the canonical `import-lorcana` path would do nothing (new imports already use `-1024.webp`). | +| 11 | `fix-user-cards-constraints.js` | `ALTER user_cards ADD CONSTRAINT user_cards_user_card_unique UNIQUE (user_id, card_id)`; `ALTER collection_cards ADD CONSTRAINT collection_cards_collection_card_unique UNIQUE (collection_id, card_id)` | Dedup `user_cards` duplicates by `user_id, card_id`; sync owned cards to each user's `'All My Cards'` system collection | **M** (with conflict — see Drift findings) | DDL is **not** captured; the constraint exists in prod but not on fresh envs. Conflict: `initial-schema` already declares `UNIQUE(user_id, card_id, is_foil)` (3-col) on `user_cards`; this script adds a **stricter** 2-col `UNIQUE(user_id, card_id)` that contradicts the foil-distinguishing semantics encoded in the bootstrap. Halt-and-ask flagged below. | + +### seed-* (2) + +| # | Script | DDL summary | DML summary | Classification | Captured by | +| --- | --- | --- | --- | --- | --- | +| 12 | `seed-collections-alice-bob.js` | — | `DELETE FROM collection_cards / collection_permissions / collections` (destructive!); then INSERT 4 Alice + 5 Bob fixture collections | **DML-only** (dev fixture) | n/a — wires test users for local UI work; not migration material. | +| 13 | `seed-collections-with-cards.js` | — | Same destructive wipe; INSERT 13 sample cards (Black Lotus, Charizard, Elsa, …); INSERT 6 fixture collections with varied empty/thumbnail/card states | **DML-only** (dev fixture) | n/a — UI demo content; not migration material. | + +### Inventory counts + +- **Already captured (no work):** 2 — `add-updated-at-column.js` (#7) fully; `add-system-collection-column.js` (#6) DDL by `1780378340194` + DML backfill by `pages/api/auth/register.js:97-119` (Finding 4 RESOLVED) +- **Missed DDL (needs migration):** 7 — #1, #2, #3, #4, #5, #8∪#9 (deduplicate). #11's DDL is NOT folded into a migration (Finding 1 RESOLVED → Outcome A; see below) +- **DML-only (not migration material):** 3 — #10 fix-lorcana, #12 seed-alice-bob, #13 seed-with-cards +- **Halt-and-ask:** 0 (Finding 1 RESOLVED below) + +## Drift findings + +### Finding 1 — `user_cards` UNIQUE constraint conflict (RESOLVED — Outcome A, 2026-06-14) + +**Resolution:** Operator ratified **Outcome A** post-architect-pass. The +3-col `UNIQUE(user_id, card_id, is_foil)` from +`migrations/1779853647564_initial-schema.js` is canonical and matches +the actual prod state. `scripts/fix-user-cards-constraints.js` was +either never applied to prod OR was applied + reverted at some point; +the canonical current state has the 3-col constraint, foil/non-foil +distinction is a real product invariant, and a separate fixed 2-col +`UNIQUE(user_id, card_id)` is **not present** on the prod `user_cards` +table. + +**Consequence for this convoy:** B6 collapses entirely. No migration +is written for `fix-user-cards-constraints.js`. Per the recommendation +in the operator decision, a no-op documentation-only migration would +add clutter to `pgmigrations` for zero functional benefit; instead, +this Drift Finding entry + a sentence in B7's `docs/SCHEMA_MAP.md` +update + a sentence in the As-shipped section document the reasoning. + +The historical script remains a no-go-zone per +`.cursor/rules/no-go-zones.mdc`; the queued +`retire-graveyard-scripts-after-audit` (P3) will delete or move it +along with the other 12 historical scripts. + +The original audit narrative is preserved below for cross-reference. + +--- + +**Original audit (pre-resolution):** + +`migrations/1779853647564_initial-schema.js` line 82 declares: + +``` +UNIQUE(user_id, card_id, is_foil) +``` + +`scripts/fix-user-cards-constraints.js` lines 50-54 adds (separately, as +a named constraint): + +``` +ALTER TABLE user_cards +ADD CONSTRAINT user_cards_user_card_unique UNIQUE (user_id, card_id) +``` + +These are **semantically incompatible** when more than one foil/non-foil +copy of the same `(user_id, card_id)` exists: + +- The bootstrap 3-column tuple allows `(1, 42, false)` AND `(1, 42, true)` + (user owns one regular + one foil copy of card 42). +- The fix-script's 2-column constraint **forbids** that pair. + +In any prod environment where `fix-user-cards-constraints.js` was run +successfully (the script's dedup-first step would have removed +duplicates pre-constraint), the stricter constraint is in force AND +the looser tuple-implicit one also exists (autogen-named like +`user_cards_user_id_card_id_is_foil_key`). Two coexisting constraints +do not break Postgres semantics — the stricter wins for write +rejection. + +Same situation on `collection_cards`: initial-schema's +`UNIQUE(collection_id, card_id)` (2-column) and the fix script's +`collection_cards_collection_card_unique` are **identical** in column +set, so this half is benign (Postgres rejects the duplicate at ADD +CONSTRAINT time; the script's caught `if (error.message.includes('already exists'))` swallows it). + +**Why this is a halt-and-ask:** the runtime code path that touches +`is_foil` differentiation (see `pages/api/cards/[id]/own.js` and +`pages/api/user-cards.js`) needs to be audited to decide which +constraint matches actual product intent. Possible outcomes: + +- **Outcome A:** Foil/non-foil distinction is a real product invariant + (a user should be able to track foil + non-foil copies of the same + card separately). The new migration should `DROP CONSTRAINT + user_cards_user_card_unique IF EXISTS` on prod, then ensure the + tuple constraint is the only one. Fresh envs already have the + correct tuple constraint from `initial-schema`; no add needed. +- **Outcome B:** Foil/non-foil should be unified at the user_cards row + level (use a separate `foil_quantity` column instead). This is a + product decision that belongs to a separate convoy + (`unify-user-cards-foil-tracking`). +- **Outcome C:** Both are tolerable (current prod state). The new + migration should add `ALTER user_cards ADD CONSTRAINT + user_cards_user_card_unique UNIQUE (user_id, card_id) IF NOT + EXISTS`-equivalent on fresh envs to match prod, AND surface the + smell in `docs/SCHEMA_MAP.md`. + +**Halt point:** before B6 is written, operator picks A / B / C. **Recommended +default** if no decision arrives in 48h: **C** (replicate prod as-is; +surface as smell). Outcome A is the closest to "intent" but adding a +mid-convoy DROP CONSTRAINT on prod data deserves its own scoped +review. + +_— Resolved 2026-06-14 as Outcome A (no DROP needed — the strict +constraint is not actually present on prod). See resolution block at +the top of this Finding._ + +### Finding 2 — `users.username` `profile_image_url` vs `avatar_url` redundancy + +Scripts #8 (`add-user-profile-columns.js`) and #9 (`add-user-profile-fields.js`) +both add `first_name`, `last_name`, `username VARCHAR(255) UNIQUE` — the +overlap is idempotent (both use `ADD COLUMN IF NOT EXISTS`) so prod +ended up with the union. But: + +- #8 adds `profile_image_url TEXT` +- #9 adds `avatar_url TEXT` + +`docs/SCHEMA_MAP.md` § "Known schema smells" #1 already flags this +redundancy ("Pick one"). For this convoy: the new migration MUST add +**both** columns to match prod-as-is (since runtime code may read +either — to be verified). Surface as follow-up +`unify-user-avatar-column`. + +### Finding 3 — `cards.quantity` and `cards.favorited` are dead columns + +`add-card-columns.js` adds `quantity INTEGER DEFAULT 0` and `favorited +BOOLEAN DEFAULT false` to the `cards` table. `docs/SCHEMA_MAP.md` +flags both as **Unused** (smell #3). The actual `quantity` / `favorited` +semantics live on `user_cards` / `user_favorites`. + +For this convoy: add both columns to fresh envs to match prod. Do NOT +drop them in prod (separate convoy). Surface as follow-up +`drop-dead-cards-columns` (deferred until a query-trace audit confirms +zero readers). + +### Finding 4 — `add-system-collection-column.js` DML backfill (RESOLVED — verified, 2026-06-14) + +**Resolution:** Verified by parent agent post-architect-pass. The +`pages/api/auth/register.js` handler at **lines 97-119** creates the +system collection on every new user signup: + +```js +const collectionResult = await sql` + INSERT INTO collections ( + name, description, tcg, is_public, user_id, slug, + is_system_collection, created_at, updated_at + ) + VALUES ( + ${SYSTEM_COLLECTION_DB_NAME}, + ${VOCAB.SYSTEM_COLLECTION_SEED_DESCRIPTION}, + 'All', false, ${user.id}, ${uniqueSlug}, + true, -- is_system_collection + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + RETURNING id +`; +``` + +The runtime invariant is intact. The historical script's per-user +backfill DML was the **one-time** reconciliation for legacy users +created before the register-hook existed; new envs never need it +because their users are all created post-hook. + +For this convoy: **no migration, no follow-up convoy needed.** The +`add-system-collection-on-register` follow-up originally surfaced in +the prior architect pass is **withdrawn**. B7 will cite this +verification in `docs/SCHEMA_MAP.md` § `collections.is_system_collection` +notes. + +### Finding 5 — Seed scripts wipe collection data + +Both `seed-collections-alice-bob.js` and `seed-collections-with-cards.js` +begin with `DELETE FROM collection_cards / collection_permissions / +collections` — destructive against any environment where real user +data exists. They are clearly tagged as dev fixtures (the test users +`alice@tcgvault.com` / `bob@tcgvault.com` are local-only). Per D4 +below, these stay out of migrations entirely. + +## Decisions (post-IA round; this is a P1 quality convoy that skipped IA/UX/A11y/Design — see frontmatter `skip:`) + +### D1 — Brief grouping: option (b), grouped by logical surface + +**Ratified: 6 implementer briefs grouped by table/feature surface.** +(Originally 7; B6 collapsed post-Finding-1 resolution — see § Brief outline.) + +Missed-DDL count is 8 (above the 1-3 threshold for option (d) and +above the 4-thing threshold that triggered option (b/c) in the +parent prompt). One-brief-per-script (option a, 13 briefs) is +excessive — many scripts touch the same DDL surface and ship one +migration each would create unnecessary `pgmigrations` rows and +make rollback narrative confusing. + +The 6-brief plan groups every missed DDL by the table or feature it +touches: + +- **B1** — Cards table reconciliation (#1, audit #7) +- **B2** — Collections table reconciliation (#2 partial, #3, #5) +- **B3** — Collaboration tables (#2 partial — `collection_permissions`, + `collection_activity`, `users.is_pending`) +- **B4** — Favorites system (#4) +- **B5** — User profile reconciliation (#8 ∪ #9, dedup'd; + + `user_settings`, `user_avatars`; + CHECK constraints; + indexes) +- **B7** — Documentation + verification runbook (`docs/SCHEMA_MAP.md` + refresh, AGENTS.md Gotcha #6 update, operator runbook) + +**B6 was removed post-Finding-1 resolution** (Outcome A — the strict +2-col `UNIQUE(user_id, card_id)` is not present on prod; the canonical +3-col `UNIQUE(user_id, card_id, is_foil)` from `initial-schema` is +what fresh envs already get). No migration is needed. The brief +numbering preserves the original gap (B1-B5 + B7) so cross-references +to "B6" in prior drafts of this convoy file remain unambiguous (they +point to the removed brief). + +Rationale: +1. **Independence.** Each of B1-B6 writes a new file under `migrations/` + with a fresh timestamp; the files don't overlap, so B1-B6 are + trivially parallel (subject to timestamp ordering — see slice + dependencies block below). +2. **Reviewability.** Each PR is one migration + one optional + `docs/SCHEMA_MAP.md` section update; reviewer can verify against + the historical script in a single sitting. +3. **Rollback granularity.** If B5's user-profile migration is found + buggy post-merge, the other 5 migrations are unaffected — they all + `pgmigrations`-row independently. +4. **Estimated LOC per brief stays under 400.** B5 is the largest + (~250 LOC of SQL across 13 ALTERs + 2 CREATE TABLE + 6 CHECKs + 6 + indexes) — comfortably under budget. + +Considered alternatives: + +| Option | Why rejected | +| --- | --- | +| (a) one brief per script (13 briefs) | Splits #2 across `collections`-columns vs `collection_permissions` vs `users.is_pending` artificially. Pads review surface 2x. | +| (c) one DDL + one DML + one verification brief (3 briefs) | A single DDL brief would be a 500+ LOC mega-migration. Hard to review, hard to rollback. | +| (d) single PR | Missed-DDL count >> 1; rejected by spec. | + +### D2 — Idempotency pattern: raw `pgm.sql(...)` with `IF [NOT] EXISTS` guards + +**Ratified: raw `pgm.sql(...)`-style SQL matching `1779853647564_initial-schema.js`'s pattern, every statement guarded with `IF NOT EXISTS` (for CREATE) or `IF EXISTS` (for DROP).** + +Rationale: +1. **Style consistency.** The five existing migrations all use + `pgm.sql()`. Mixing `pgm.createTable()` helpers would introduce a + second pattern for no operational benefit. +2. **Verbatim SQL transparency.** The reviewer can grep the new + migration's SQL string against the historical script's SQL string + and confirm byte-equivalent intent. +3. **Idempotency.** Every CREATE / ALTER ADD COLUMN / CREATE INDEX + statement uses the appropriate `IF NOT EXISTS` guard so re-running + against prod is a documented no-op. CHECK constraints don't accept + `IF NOT EXISTS` directly — wrap in a `DO $$ BEGIN ... EXCEPTION + WHEN duplicate_object THEN NULL; END $$;` block (matching the + historical script's try/catch pattern). + +`node-pg-migrate` helpers (`pgm.createTable`, `pgm.addColumns`) would +work fine technically, but the existing migration corpus is 100% raw +SQL. Optimize for review uniformity. + +### D3 — Drift detection: option (a) — assume change is in prod, add a migration to bring fresh envs to parity + +**Ratified: (a) with a documented "evidence" trail per script in the Inventory section above.** + +For every script in the **M** classification, the assumption is that +the script ran successfully against prod at some point and its DDL is +now baked into the prod schema. Evidence supporting this assumption: + +- `docs/SCHEMA_MAP.md` (last reviewed 2026-05-22) documents all of these + columns/tables as live. +- Runtime code in `pages/api/**` reads/writes `user_favorites`, + `collection_permissions`, `collection_activity`, `user_settings`, + `user_avatars`, `users.first_name`, `users.username`, `users.bio`, + `users.avatar_url`, `users.theme`, `users.language`, + `collections.visibility`, `collections.tcg`, `collections.tags`, + `collections.slug`, `collections.image` — confirmed via grep + (`pages/api/user/settings.js`, `pages/api/user/profile.js`, + `pages/api/favorites.js`, `pages/api/invite/{accept,decline}.js`, + `pages/api/collections/[identifier]/permissions.js`, etc.). +- Each historical script is itself idempotent (`ADD COLUMN IF NOT + EXISTS`), so prod-vs-fresh divergence is the live state. + +For Finding 1 (UNIQUE constraint conflict), Finding 4 (system-coll +backfill), and Finding 5 (seed wipes): per the recommendation in the +parent prompt, halt-and-ask is reserved for destructive changes. (1) +qualifies (DROP CONSTRAINT) → halt-and-ask. (4) is operationally +benign (no DROP) → defer to follow-up. (5) is destructive but the +recommended choice is "don't fold into migrations at all" — D4 below. + +### D4 — Seed scripts: option (a) — leave alone + +**Ratified: dev-fixture seed scripts stay out of the migration history.** + +`seed-collections-alice-bob.js` and `seed-collections-with-cards.js` +are dev-fixture loaders tied to `alice@tcgvault.com` / +`bob@tcgvault.com` (which only exist in `scripts/create-test-users.js`, +gated post-`purge-weak-creds-from-helpers` behind `TEST_USERS_PASSWORD`). +They: + +- Wipe destructive collection data (incompatible with any env with real users). +- Reference Disney/Lorcana sample art URLs (`example.com/elsa.jpg`). +- Are clearly UI-demo fodder. + +They have no place in `setup-db`'s migrate pipeline. They will be +retired (or moved to `scripts/historical/`) by the queued +`retire-graveyard-scripts-after-audit` convoy. + +If multiple developers feel friction managing dev-fixture state in +the future, surface a separate `consolidate-dev-seeds` follow-up +that designs a non-destructive `npm run seed:dev` flow keyed off a +fresh DB. Not surfaced from this convoy because no friction has been +reported yet. + +### D5 — Verification: option (a) — manual operator runbook + +**Ratified: post-merge manual verification by spinning up a fresh +Neon branch + running `npm run setup-db` + diffing schema against +prod.** + +Automated CI verification belongs to the queued `wire-migrate-into-ci` +convoy and is out of scope here. The manual runbook will live in this +convoy file's "Verification plan" section (below) and be promoted to +`docs/operations/RECONCILE-VERIFICATION.md` by B7. + +### D6 — `scripts/migrations/2026-05-24-rename-admin-email.js`: option (b) — leave where it is + +**Ratified: do not move into `migrations/`.** + +The lone pre-tool migration sits at `scripts/migrations/2026-05-24-rename-admin-email.js` +and was applied to every long-lived env at `pick-a-name` time +(2026-05-24). Moving it into `migrations/` would require backfilling a +`pgmigrations` row on every existing env, which is operationally +risky for zero functional benefit (the migration is already applied; +the runtime never re-checks it). + +For fresh envs, the migration is a no-op (the seed admin row is +already created at `admin@deckhearth.com` by post-rename +`scripts/setup-neon-db.js`; there's no `@tcgvault.com` row to rename). + +Document the rationale in B7's `docs/SCHEMA_MAP.md` update + a brief +note in AGENTS.md Gotcha #4's "Rotation script" subsection. If a +future fresh-env onboarding ever tries to roll back to pre-rename +state, surface `fold-rename-admin-email-into-migrations` as a +follow-up. + +### D7 — `pgmigrations` table state on fresh envs: no special handling required + +**Ratified: rely on `node-pg-migrate`'s standard sequential apply.** + +On a fresh Neon branch, `npm run migrate up` will run all migrations +in timestamp order: + +1. `1779853647564_initial-schema` (7 bootstrap tables) +2. `1779853647565_add-pg-trgm-card-name-index` +3. `1779853647566_add-scan-tables` +4. `1779908094455_add-user-cards-scan-image-url` +5. `1780378340194_system-collection-description` +6. **NEW: B1** — cards columns (quantity, favorited) +7. **NEW: B2** — collections columns (visibility, tcg, tags, slug, slug index/constraint, image) +8. **NEW: B3** — collaboration tables (collection_permissions, collection_activity, users.is_pending, 4 indexes) +9. **NEW: B4** — favorites system (user_favorites + 4 indexes) +10. **NEW: B5** — user profile reconciliation (15 ALTER-ADD-COLUMN + 2 CREATE TABLE + 6 CHECK + 6 indexes) + +Each new migration is additive against the post-initial-schema state +that prior migrations leave behind; no inter-migration dependencies +crossed within this convoy. + +Audited ordering risks for the new migrations: + +- B1 depends on `cards` (initial-schema) ✅ +- B2 depends on `collections` (initial-schema) ✅ +- B3 depends on `collections` + `users` (initial-schema) ✅ +- B4 depends on `users` (initial-schema) ✅ +- B5 depends on `users` (initial-schema) ✅ + +For long-lived envs that already have all post-historical columns: +every new migration is a documented no-op due to `IF NOT EXISTS` +guards. The only state change is the `pgmigrations` row insertion. + +## Brief outline + +Six implementer briefs. Each B1-B5 ships exactly one new file under +`migrations/_.js`. B7 updates docs only. Implementer +briefs are drafted under `.convoys/reconcile-historical-add-scripts/brief--.md`. + +| Brief | Title | Files (new) | Depends on | Est. LOC | Notes | +| --- | --- | --- | --- | --- | --- | +| B1 | Reconcile cards columns | `migrations/1781000000001_reconcile-cards-columns.js` | — | ~40 | Captures script #1; script #7 already captured by initial-schema (verify & document in PR body) | +| B2 | Reconcile collections columns | `migrations/1781000000002_reconcile-collections-columns.js` | — | ~120 | Captures #2 (collections half: visibility/tcg/tags), #3 (slug + index + CHECK), #5 (image). Defer slug backfill DML — generating slugs on a fresh env is moot. | +| B3 | Reconcile collaboration tables | `migrations/1781000000003_reconcile-collaboration-tables.js` | — | ~130 | Captures #2 (collaboration half: collection_permissions, collection_activity, users.is_pending, 4 indexes). Defer owner-permission backfill DML — fresh envs have no pre-existing collections needing backfill. | +| B4 | Reconcile favorites system | `migrations/1781000000004_reconcile-favorites-system.js` | — | ~50 | Captures #4 (user_favorites + 4 indexes). | +| B5 | Reconcile user profile | `migrations/1781000000005_reconcile-user-profile.js` | — | ~250 | Captures #8 ∪ #9 (deduplicated; both add columns coexist in prod). Includes user_settings, user_avatars, 6 CHECK constraints (wrapped in `DO $$ EXCEPTION` blocks for idempotency), 6 indexes. Defer defaults-backfill UPDATE — column DEFAULTs handle it. | +| ~~B6~~ | ~~Reconcile user_cards / collection_cards UNIQUE constraints~~ | ~~removed~~ | — | 0 | **REMOVED 2026-06-14 — Finding 1 RESOLVED as Outcome A.** The strict 2-col `UNIQUE(user_id, card_id)` is not present on prod; the canonical 3-col tuple is already on fresh envs via `initial-schema`. Documented in Drift Finding 1 + B7's SCHEMA_MAP update; no migration written. | +| B7 | Documentation + verification | `docs/SCHEMA_MAP.md` (update); `docs/MIGRATION_VERIFICATION_RUNBOOK.md` (new); `AGENTS.md` (Gotcha #6 audit); `.convoys/ship-readiness.md` (flip queued entry) | B1-B5 merged | ~150 | SCHEMA_MAP smell-list updates per Findings 1-3. Verification runbook (see § Verification plan below). AGENTS.md Gotcha #6 already RESOLVED post-migration-tool; verify and leave alone if accurate. ship-readiness.md "Queued convoys" → flip this convoy entry to RESOLVED + unblock `retire-graveyard-scripts-after-audit` + add `unify-user-avatar-column` + `drop-dead-cards-columns` follow-ups. | + +Per-brief acceptance criteria (sketch — formalized in each brief file): + +- Each migration file has a `down()` that **throws** with a clear message + (these are reconciliation migrations; the prod schema state is the + source of truth and rolling back would create inconsistency). +- Each migration's SQL is **byte-equivalent in intent** to the + historical script's SQL (per-statement comments cite the script + and line range). +- Re-running `npm run migrate up` against any current long-lived env + is a no-op (all guards trigger). +- `npm run migrate up` against a fresh Neon branch followed by the + verification runbook (D5) confirms structural parity with prod. + +### Slice dependencies (multitask-ready) + +```yaml +slice_dependencies: + - brief: 1 + depends_on: [] + files: [migrations/1781000000001_reconcile-cards-columns.js] + - brief: 2 + depends_on: [] + files: [migrations/1781000000002_reconcile-collections-columns.js] + - brief: 3 + depends_on: [] + files: [migrations/1781000000003_reconcile-collaboration-tables.js] + - brief: 4 + depends_on: [] + files: [migrations/1781000000004_reconcile-favorites-system.js] + - brief: 5 + depends_on: [] + files: [migrations/1781000000005_reconcile-user-profile.js] + # brief 6 removed — Finding 1 RESOLVED as Outcome A; timestamp 1781000000006 is unused + - brief: 7 + depends_on: [1, 2, 3, 4, 5] + files: + - docs/SCHEMA_MAP.md + - docs/MIGRATION_VERIFICATION_RUNBOOK.md + - AGENTS.md + - .convoys/ship-readiness.md +``` + +**Timestamp coordination.** Each B1-B5 writes a +`migrations/_*.js` file with a pre-assigned timestamp +(reservation token, not literal `Date.now()`). Pre-assigned +timestamps avoid the parallel-implementer collision risk documented +in `scaffold-nextjs-app` retro recommendation #4. The brief frontmatter +in each `.convoys/reconcile-historical-add-scripts/brief--*.md` +file declares the exact path; implementers MUST use that exact +filename (not `npm run migrate create`, which would `Date.now()`). + +## Verification plan (D5 operator runbook) + +To verify the reconstructed migration history produces parity with +prod, after B1-B6 merge: + +```bash +# 1. Snapshot prod's structural shape (run against prod POSTGRES_URL) +# Schema-only dump, no data, no owner/ACL noise +POSTGRES_URL= pg_dump --schema-only --no-owner --no-acl \ + --schema=public > /tmp/prod-schema.sql + +# 2. Create a clean Neon branch from EMPTY (no parent branch) and onboard via setup-db +# (use the Neon dashboard or `neon branches create --empty`) +POSTGRES_URL= \ +ADMIN_INITIAL_PASSWORD=$(openssl rand -base64 24) \ + npm run setup-db + +# 3. Snapshot the fresh branch's structural shape +POSTGRES_URL= pg_dump --schema-only --no-owner --no-acl \ + --schema=public > /tmp/fresh-schema.sql + +# 4. Diff. Expected differences are limited to: +# - constraint/index NAME differences (autogen tuple-UNIQUE names vs explicit names) +# - column-ORDER differences (prod has columns in historical-script-ALTER order; +# fresh envs have them in migration-order) +# Both are semantically irrelevant. Material differences = bug; flag and reopen. +diff <(sort /tmp/prod-schema.sql) <(sort /tmp/fresh-schema.sql) +``` + +Supplementary information-schema spot-checks for the highest-risk surfaces: + +```sql +-- Every column on every table +SELECT table_name, column_name, data_type, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = 'public' +ORDER BY table_name, ordinal_position; + +-- Every constraint +SELECT table_name, constraint_name, constraint_type +FROM information_schema.table_constraints +WHERE table_schema = 'public' +ORDER BY table_name, constraint_name; + +-- Every index +SELECT tablename, indexname, indexdef +FROM pg_indexes +WHERE schemaname = 'public' +ORDER BY tablename, indexname; +``` + +Run both against prod and fresh-branch; the column-count + constraint-count ++ index-count totals should match exactly. Mismatch = bug. + +B7 will move this runbook to `docs/operations/RECONCILE-VERIFICATION.md` +and link it from `AGENTS.md` Gotcha #6 + `migration-tool` § R1's +"resolved-by" note. + +## Risks + +### R1 — Missed DDL that is NOT actually in prod ("ghost migration") + +The assumption (D3) is that every historical script ran successfully +against every long-lived env. If a script in fact failed silently +mid-execution on prod (e.g. `add-collaboration-features.js`'s +`CREATE INDEX idx_collections_visibility` errored partway through), +prod might not actually have that index even though SCHEMA_MAP says +it does. + +**Mitigation:** the verification plan (D5) catches this. The fresh-env +`pg_dump` would contain the index; prod's `pg_dump` would not; the +diff would surface it. If found, operator decides: (a) the script's +intent was sound, apply the missed DDL to prod manually with +`POSTGRES_URL= psql -c "CREATE INDEX IF NOT EXISTS ..."`; or +(b) the index is unwanted, drop it from the new migration and document. + +### R2 — `pgmigrations` row state on existing prod envs + +After this convoy merges, an operator running `npm run migrate up` +against prod will see 6 new migrations apply (B1-B6) as no-ops (every +guarded statement triggers `IF [NOT] EXISTS`-skip). Six new +`pgmigrations` rows record successful application. + +If for some reason a long-lived prod env genuinely lacks one of the +historical-script columns (Risk R1 above), the corresponding migration +will **add** that column on apply, no-op-ing the others. The +`pgmigrations` row records success; subsequent applies are no-ops. +This is the correct behavior, but the operator should run the +verification plan post-apply to confirm. + +**Mitigation:** the verification plan covers prod-vs-fresh diff after +B1-B6 land. Run it once on each prod-shaped env immediately after +merge. + +### R3 — Ordering: new migration depends on prior schema-state that doesn't exist at its execution point + +Audited in D7. All B1-B6 dependencies on prior tables (`cards`, +`collections`, `users`, `user_cards`, `collection_cards`) are +satisfied by `initial-schema` (timestamp `1779853647564`, runs first +on fresh envs). No B-to-B inter-dependency required. + +### R4 — Seed scripts depend on test users that don't exist on fresh envs + +Out of scope (D4). The two seed scripts are dev fixtures and are not +folded into migrations. Test users (`alice@`, `bob@`) are managed by +`scripts/create-test-users.js` (post-`purge-weak-creds-from-helpers`, +gated behind `TEST_USERS_PASSWORD`). Surface as `consolidate-dev-seeds` +follow-up only if a developer reports friction. + +### R5 — CHECK constraint reapplication on prod is loud + +`add-user-profile-fields.js` wraps each CHECK constraint ADD in a +JS try/catch that swallows `already exists` errors. Postgres doesn't +accept `ADD CONSTRAINT ... IF NOT EXISTS` for CHECK; the SQL has to be +wrapped in `DO $$ ... EXCEPTION WHEN duplicate_object THEN NULL END +$$;`. B5's migration must use the exception-handling form so reapply +against prod is silent. Implementer brief will spell this out. + +### R6 — Mid-convoy timestamp collision when implementers spawn in parallel + +B1-B5 are mutually independent and can dispatch via `/multitask`. +Pre-assigned timestamps (D7 table) avoid the `Date.now()`-collision +risk that bit the `scaffold-nextjs-app` convoy. Each implementer +brief will name its file's exact timestamp; deviations require a +re-plan. + +### R7 — Operator approval lag on Finding 1 (RESOLVED 2026-06-14) + +Originally a risk because B6 was gated on operator decision. Finding 1 +RESOLVED as Outcome A; B6 removed. No lag risk remains. + +## Follow-ups + +Surfaced by this convoy: + +- **`retire-graveyard-scripts-after-audit`** (priority: **P3 polish — now + UNBLOCKED** once this convoy lands). Once B1-B6 capture every + missed historical DDL into the migration history, the 13 historical + scripts can be safely deleted or moved to `scripts/historical/` and + the corresponding no-go-zones rule line can be removed. Documented + in `.convoys/migration-tool.md` § Follow-ups; ship-readiness.md + "Queued convoys" entry; AGENTS.md Gotcha #6 cross-reference. +- **`wire-migrate-into-ci`** (priority: P2 CI infra — pre-existing). + Adds a CI job that runs `npm run migrate up` against a test DB on + every PR. This convoy's verification plan (D5) is the manual + precursor; the CI job is the automation upgrade. Already on the + follow-up list per `migration-tool` § Follow-ups + ship-readiness.md + "Queued convoys". +- **`unify-user-avatar-column`** (priority: P3 hygiene — NEW). Driven + by Finding 2. Two redundant TEXT columns + (`users.profile_image_url` and `users.avatar_url`) coexist; pick one + canonical column, migrate the other's data, drop the loser, update + runtime readers. Requires a query-trace audit first. +- **`drop-dead-cards-columns`** (priority: P3 hygiene — NEW). Driven + by Finding 3. `cards.quantity` and `cards.favorited` are documented + as unused. After a query-trace audit confirms zero readers, ship a + migration that DROPs them (with proper `down()` recreate). +- ~~**`add-system-collection-on-register`**~~ — **WITHDRAWN 2026-06-14**, Finding 4 RESOLVED. Verified the register hook exists at `pages/api/auth/register.js:97-119`. +- **`fold-rename-admin-email-into-migrations`** (priority: P3 polish, + conditional — NEW per D6). Only surface if a future fresh-env + onboarding needs the rename migration applied in order. Until then, + the script-shaped migration at + `scripts/migrations/2026-05-24-rename-admin-email.js` is left in + place per D6. +- ~~**`unify-user-cards-foil-tracking`**~~ — **WITHDRAWN 2026-06-14**, Finding 1 RESOLVED as Outcome A (not Outcome B). The 3-col tuple stays canonical; no foil-tracking redesign needed. + +Not surfaced (no friction yet): + +- `consolidate-dev-seeds` — D4 noted this can wait until multiple + developers report friction with the current per-script dev-fixture + loaders. + +## As-shipped + +Six implementer briefs (B1–B6) landed between 2026-06-14 and 2026-07-06. +Brief 7 (documentation + verification runbook) closed the convoy on +2026-08-15 — a post-hoc docs-only pass delayed by an operator-initiated +pause on a self-hosted runner infra issue that did not affect migration +content correctness. + +### Per-brief delivery + +| Brief | PR | Squash | Merged | Migration file | +| --- | --- | --- | --- | --- | +| B1 cards columns | #148 | `a35ce01` | 2026-06-14 | `migrations/1781442330001_reconcile-cards-columns.js` | +| B2 collections columns | #150 | `e8619c5` | 2026-06-14 | `migrations/1781442330002_reconcile-collections-columns.js` | +| B3 collaboration tables | #151 | `15e02ee` | 2026-07-06 | `migrations/1781000000003_reconcile-collaboration-tables.js` | +| B4 favorites system | #152 | `ef2cfb8` | 2026-06-14 | `migrations/1781442330004_reconcile-favorites-system.js` | +| B5 user profile | #153 | `ee7da9a` | 2026-06-14 | `migrations/1781442330005_reconcile-user-profile.js` | +| B6 user_cards UNIQUE | #149 | `40402eb` | 2026-06-14 | `migrations/1781442330006_reconcile-user-cards-unique.js` | +| B7 docs + runbook | _(this PR)_ | — | 2026-08-15 | _(no migration — docs only)_ | + +All six implementer PRs were green at merge on the (then-flaky) self-hosted +axiom runner pool. + +### Reservation-timestamp rename (`ec9bb2b`) + +The architect pre-assigned timestamps `1781000000001` through +`1781000000006` to avoid `Date.now()` collisions during parallel +implementer dispatch. Commit `ec9bb2b` (`feat(catalog): unified multi-game +bulk sync + schema map update`) renamed all except B3 to +`1781442330001`–`1781442330006` so they run **after** the catalog-sync +migrations that landed in the same period: + +- `1781440700404_add-scryfall-bulk-columns` +- `1781440721350_add-tagger-tables` +- `1781442175729_normalize-lorcana-set-codes` +- `1781442329511_add-catalog-sync-log` + +B3 kept its original `1781000000003` timestamp because it merged after the +rename commit. The rename is filename-only; migration `up()` content is +identical. + +### B6 deviation from architect plan + +The architect's gate-1 plan collapsed Finding 1 (Outcome A) into B7 as a +doc-only note. Implementers shipped B6 anyway as a tiny belt-and-suspenders +migration (`1781442330006_reconcile-user-cards-unique.js`) that: + +- Confirms the canonical 3-column `UNIQUE(user_id, card_id, is_foil)` from + `initial-schema` is present. +- Drops the historical 2-column `user_cards_user_card_unique` constraint if + any env ran `fix-user-cards-constraints.js`. + +The migration is a no-op on prod (Outcome A already holds) but guards +future envs that might have the stricter 2-col variant. + +### Verification + +Manual operator runbook for confirming fresh-env vs prod schema parity: +[`docs/MIGRATION_VERIFICATION_RUNBOOK.md`](../docs/MIGRATION_VERIFICATION_RUNBOOK.md) +(per architect Decision D5). Automated CI verification remains queued as +`wire-migrate-into-ci`. + +### B7 deviation from spec + +B7 shipped ~2 months post-hoc (2026-08-15) rather than immediately after +B1–B6. The delay was operator-initiated (runner-infra pause) and did not +block migration correctness — only documentation closure. diff --git a/.convoys/reconcile-historical-add-scripts/brief-1-reconcile-cards-columns.md b/.convoys/reconcile-historical-add-scripts/brief-1-reconcile-cards-columns.md new file mode 100644 index 0000000..fb33295 --- /dev/null +++ b/.convoys/reconcile-historical-add-scripts/brief-1-reconcile-cards-columns.md @@ -0,0 +1,228 @@ +--- +convoy: reconcile-historical-add-scripts +brief_number: 1 +depends_on: [] +files: + - migrations/1781000000001_reconcile-cards-columns.js +--- + +# Brief 1: Reconcile `cards` columns into migration history + +## Goal (1 sentence) + +Capture the DDL added by `scripts/add-card-columns.js` (the `cards.quantity` + `cards.favorited` columns) into a single new `node-pg-migrate` migration so a brand-new Neon branch ends up with both columns after `npm run setup-db`. + +## Scope (files in scope — do not edit anything else) + +- `migrations/1781000000001_reconcile-cards-columns.js` — **new** + +## Source script (read-only audit reference; DO NOT EDIT — no-go-zone) + +`scripts/add-card-columns.js` lines 22-33 (verbatim): + +```js +await sql` + ALTER TABLE cards + ADD COLUMN IF NOT EXISTS quantity INTEGER DEFAULT 0 +`; +// ... +await sql` + ALTER TABLE cards + ADD COLUMN IF NOT EXISTS favorited BOOLEAN DEFAULT false +`; +``` + +**Sibling script `scripts/add-updated-at-column.js`** is already captured by `migrations/1779853647564_initial-schema.js` (the `cards` `CREATE TABLE` at lines 45-69 already declares `updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP`). Do NOT add an ALTER for `updated_at` — it would be redundant noise on `pgmigrations`. Mention this in the PR body so reviewers don't ask. + +## Target migration file + +Path: `migrations/1781000000001_reconcile-cards-columns.js` + +Contents: + +```js +/** + * Reconcile historical `scripts/add-card-columns.js` into the migration + * history. Adds `cards.quantity` and `cards.favorited` columns so a + * brand-new Neon branch ends up with the same shape that prod has had + * since the historical script's one-shot run. + * + * Both columns are flagged as **unused** in docs/SCHEMA_MAP.md + * § "Known schema smells" #3 — `quantity` lives on `user_cards`, + * `favorited` lives on `user_favorites`. They're added here for + * fresh-env parity with prod. A follow-up `drop-dead-cards-columns` + * convoy (queued, P3 hygiene) will drop both columns once a query-trace + * audit confirms zero runtime readers. + * + * Idempotent: re-running against any long-lived prod env is a no-op + * because both ALTERs use IF NOT EXISTS. + * + * Note: `scripts/add-updated-at-column.js` (the sibling historical + * script in the same convoy) is NOT reconciled here because + * `cards.updated_at` is already declared in + * `migrations/1779853647564_initial-schema.js`'s `CREATE TABLE cards` + * (line 67). No further work needed for that script. + * + * @type {import('node-pg-migrate').ColumnDefinitions | undefined} + */ +export const shorthands = undefined; + +/** + * @param {import('node-pg-migrate').MigrationBuilder} pgm + */ +export const up = (pgm) => { + pgm.sql(` + ALTER TABLE cards + ADD COLUMN IF NOT EXISTS quantity INTEGER DEFAULT 0; + + ALTER TABLE cards + ADD COLUMN IF NOT EXISTS favorited BOOLEAN DEFAULT false; + `); +}; + +/** + * Down-migration intentionally throws. Dropping these columns on + * long-lived envs requires the `drop-dead-cards-columns` convoy's + * query-trace audit — bypassing it via a casual rollback risks + * dropping data on prod. Use `drop-dead-cards-columns` when ready. + * + * @returns {void} + */ +export const down = () => { + throw new Error( + '[migration:1781000000001_reconcile-cards-columns] Down not supported. ' + + 'Dropping cards.quantity / cards.favorited belongs to the queued ' + + 'drop-dead-cards-columns convoy, which performs a query-trace audit ' + + 'before the DROP. Do not rollback this migration directly.' + ); +}; +``` + +## Conventions to follow + +- `.cursor/rules/db-and-schema.mdc` § "Schema source of truth" — `migrations/` is the canonical home for schema changes; one column-group per migration. +- `.cursor/rules/no-go-zones.mdc` — `scripts/add-card-columns.js` is append-only history. **Do not edit it.** +- Style: match the existing migrations under `migrations/`. Use raw `pgm.sql(...)` template literals (the `migration-tool` convoy ratified this in D2 of `.convoys/reconcile-historical-add-scripts.md`). +- ESM exports (`export const up = ...`, `export const down = ...`); no `module.exports`. The repo is `"type": "module"` per `package.json` line 5. +- Use `IF NOT EXISTS` on every ALTER — idempotent re-apply is a documented requirement of this convoy. +- Add a JSDoc docstring at the top of the file explaining what's being reconciled, citing the source script + the convoy file. + +## Acceptance criteria + +- [ ] `migrations/1781000000001_reconcile-cards-columns.js` exists with the exact filename above (the timestamp `1781000000001` is the reservation token assigned by the architect — do NOT use `npm run migrate create`, which would call `Date.now()` and assign a different timestamp). +- [ ] The file's `up()` adds both columns via `IF NOT EXISTS`. +- [ ] The file's `down()` throws with a clear message pointing at the `drop-dead-cards-columns` follow-up. +- [ ] The file's docstring cites `scripts/add-card-columns.js` and `.convoys/reconcile-historical-add-scripts.md`. +- [ ] `node --check migrations/1781000000001_reconcile-cards-columns.js` passes (syntactic validity). +- [ ] `node -e "import('./migrations/1781000000001_reconcile-cards-columns.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined` (module loads cleanly). +- [ ] `npm run lint` exits clean against the baseline (no new lint errors introduced by this file). +- [ ] `npm run test:run` reports 21/21 passing (no test surface changes). +- [ ] The PR body documents that `add-updated-at-column.js` is already captured by `initial-schema` (per the source-script section above) and explains why no second migration is added. + +## Verification + +Run the following from the convoy worktree (`tcg-vault-worktrees/reconcile-historical-add-scripts/`) BEFORE opening the PR: + +```bash +node --check migrations/1781000000001_reconcile-cards-columns.js +node -e "import('./migrations/1781000000001_reconcile-cards-columns.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))" +npm run lint +npm run test:run +``` + +Expected: +- `node --check` exits 0 with no output. +- The `node -e` line prints exactly: `function function undefined`. +- `npm run lint` matches the existing baseline (no new errors). +- `npm run test:run` reports 21/21 tests passing. + +**Do NOT** run `npm run migrate up` against any environment as part of this brief — that's a post-merge operator step covered by Brief 7's verification runbook. + +## Commit message + +``` +feat(migrations): reconcile add-card-columns into migration history (brief 1/7) + +Captures the DDL effect of scripts/add-card-columns.js (cards.quantity ++ cards.favorited columns) into a new node-pg-migrate migration. Both +columns are flagged as unused in docs/SCHEMA_MAP.md § "Known schema +smells" #3 — added here for fresh-env parity with prod; a follow-up +drop-dead-cards-columns convoy will drop them after a query-trace +audit. + +Sibling script add-updated-at-column.js is already captured by +migrations/1779853647564_initial-schema.js (cards.updated_at is in the +CREATE TABLE); no second migration needed. + +Per .convoys/reconcile-historical-add-scripts.md § Brief outline → B1. +Idempotent re-apply (IF NOT EXISTS guards). +``` + +## PR shape + +**Title:** `feat(migrations): reconcile add-card-columns into migration history (brief 1/7)` + +**Body template:** + +```markdown +Brief 1 of the `reconcile-historical-add-scripts` convoy. See +[`.convoys/reconcile-historical-add-scripts.md`](../.convoys/reconcile-historical-add-scripts.md) +for the full plan and rationale. + +## What this PR does + +Adds `migrations/1781000000001_reconcile-cards-columns.js` — a new +`node-pg-migrate` migration that adds two columns to `cards` via +`ADD COLUMN IF NOT EXISTS`: + +- `quantity INTEGER DEFAULT 0` +- `favorited BOOLEAN DEFAULT false` + +Both columns already exist in long-lived prod environments (added by +the historical `scripts/add-card-columns.js`). This migration brings +fresh Neon branches to parity so `npm install` → `npm run setup-db` +alone produces the prod shape, without manually replaying historical +scripts. + +## What this PR does NOT do + +- Does **NOT** edit `scripts/add-card-columns.js` (no-go-zone). +- Does **NOT** add a migration for `scripts/add-updated-at-column.js` + — `cards.updated_at` is already declared in + `migrations/1779853647564_initial-schema.js` line 67. +- Does **NOT** edit `scripts/setup-neon-db.js`, `package.json`, README, + or AGENTS.md. +- Does **NOT** run `npm run migrate up` against any environment (that's + the post-merge operator step covered by Brief 7's runbook). +- Does **NOT** drop the columns (deferred to follow-up + `drop-dead-cards-columns`). + +## Verification checklist + +- [ ] `node --check migrations/1781000000001_reconcile-cards-columns.js` exits 0 +- [ ] `node -e "import('./migrations/1781000000001_reconcile-cards-columns.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined` +- [ ] `npm run lint` matches baseline (no new errors) +- [ ] `npm run test:run` reports 21/21 passing +- [ ] Did not run `npm run migrate up` against any environment in this PR +- [ ] Operator post-merge: run the verification runbook from Brief 7 (`docs/MIGRATION_VERIFICATION_RUNBOOK.md` once it lands) + +## Cross-references + +- Convoy file: `.convoys/reconcile-historical-add-scripts.md` +- Source script (no-go-zone, audit reference only): `scripts/add-card-columns.js` +- Follow-up after this convoy lands: `drop-dead-cards-columns` (P3 hygiene) +``` + +## DO NOT + +- DO NOT edit `scripts/add-card-columns.js` or any other file under `scripts/` — append-only no-go-zone per `.cursor/rules/no-go-zones.mdc`. +- DO NOT edit any other file under `migrations/` — each migration is pinned by its `pgmigrations` row. +- DO NOT edit `scripts/setup-neon-db.js`. +- DO NOT edit `package.json` (no new deps). +- DO NOT edit `AGENTS.md` or `docs/SCHEMA_MAP.md` — that's Brief 7's job. +- DO NOT run `npm run migrate up` against any environment. +- DO NOT call `npm run migrate create` to scaffold the file — it uses `Date.now()` for the timestamp prefix, which would collide with the architect's pre-assigned reservation tokens for parallel briefs. + +## Rationale (≤3 sentences) + +Capturing `cards.quantity` + `cards.favorited` in a single small migration matches the per-table grouping of the convoy's D1 decision and keeps the diff easy to review. Sibling `add-updated-at-column.js` is already captured by `initial-schema`, so reconciling it would create a `pgmigrations` row for zero functional benefit. The columns themselves are dead per SCHEMA_MAP smell #3, but parity with prod is the convoy's success metric — the actual DROP is the `drop-dead-cards-columns` follow-up's responsibility. diff --git a/.convoys/reconcile-historical-add-scripts/brief-2-reconcile-collections-columns.md b/.convoys/reconcile-historical-add-scripts/brief-2-reconcile-collections-columns.md new file mode 100644 index 0000000..d76aefb --- /dev/null +++ b/.convoys/reconcile-historical-add-scripts/brief-2-reconcile-collections-columns.md @@ -0,0 +1,307 @@ +--- +convoy: reconcile-historical-add-scripts +brief_number: 2 +depends_on: [] +files: + - migrations/1781000000002_reconcile-collections-columns.js +--- + +# Brief 2: Reconcile `collections` columns into migration history + +## Goal (1 sentence) + +Capture the `collections`-table DDL added by three historical scripts (`add-collaboration-features.js` columns half, `add-collection-slugs.js`, `add-image-column.js`) into a single new `node-pg-migrate` migration so a brand-new Neon branch ends up with `visibility`, `tcg`, `tags`, `slug` (+ index + CHECK constraint), and `image` columns on `collections` after `npm run setup-db`. + +## Scope (files in scope — do not edit anything else) + +- `migrations/1781000000002_reconcile-collections-columns.js` — **new** + +## Source scripts (read-only audit reference; DO NOT EDIT — all three are no-go-zones) + +### `scripts/add-collaboration-features.js` lines 15-20 (collections half only — the `collection_permissions` / `collection_activity` / `users.is_pending` half is Brief 3's scope): + +```js +await sql` + ALTER TABLE collections + ADD COLUMN IF NOT EXISTS visibility VARCHAR(20) DEFAULT 'private', + ADD COLUMN IF NOT EXISTS tcg VARCHAR(50) DEFAULT 'MTG', + ADD COLUMN IF NOT EXISTS tags TEXT +`; +``` + +Plus the visibility index at lines 77-80: + +```js +await sql` + CREATE INDEX IF NOT EXISTS idx_collections_visibility + ON collections(visibility) +`; +``` + +### `scripts/add-collection-slugs.js` lines 17-99 (relevant DDL only): + +```js +// Step 1: Add slug column +await sql` + ALTER TABLE collections + ADD COLUMN IF NOT EXISTS slug VARCHAR(100) UNIQUE +`; + +// Step 5: Add unique index +await sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_collections_slug ON collections(slug)`; + +// Step 6: Add format CHECK constraint +await sql`ALTER TABLE collections ADD CONSTRAINT check_slug_format CHECK (slug ~ '^[a-z0-9]([a-z0-9-]*[a-z0-9])?$' AND length(slug) <= 50)`; +``` + +The script also backfills slugs via per-row UPDATE using `lib/slug-utils.js::generateUniqueSlug` (lines 41-73). **Do NOT fold the backfill DML into this migration.** Fresh envs have no pre-existing `collections` rows to backfill; on prod, the backfill ran once historically. Future migrations that need slug generation should run that logic in application code, not in a migration. + +### `scripts/add-image-column.js` lines 14-17 (entire DDL): + +```js +await sql` + ALTER TABLE collections + ADD COLUMN IF NOT EXISTS image TEXT +`; +``` + +## Target migration file + +Path: `migrations/1781000000002_reconcile-collections-columns.js` + +Contents: + +```js +/** + * Reconcile three historical scripts that all added columns to the + * `collections` table: + * + * - scripts/add-collaboration-features.js (visibility, tcg, tags + + * idx_collections_visibility index) — collections-table half only; + * the collection_permissions / collection_activity / users.is_pending + * half is reconciled by migrations/1781000000003_reconcile-collaboration-tables.js + * - scripts/add-collection-slugs.js (slug + idx_collections_slug + * unique index + check_slug_format CHECK constraint) + * - scripts/add-image-column.js (image) + * + * Grouped into one migration per D1 of .convoys/reconcile-historical-add-scripts.md + * (one migration per table/feature surface). All historical DDL was + * idempotent (ADD COLUMN IF NOT EXISTS / CREATE INDEX IF NOT EXISTS); + * this migration preserves that. CHECK constraint adds via a + * DO $$ EXCEPTION block because Postgres doesn't accept + * ADD CONSTRAINT ... IF NOT EXISTS for CHECK. + * + * Per-row slug backfill DML from add-collection-slugs.js is intentionally + * NOT folded in — fresh envs have no pre-existing collections to + * backfill; on prod, the backfill ran once historically and is baked in. + * + * Notes on adjacent state: + * - `is_system_collection` was added by migrations/1780378340194_system-collection-description.js + * (and the runtime register hook at pages/api/auth/register.js:97-119 + * creates the per-user system collection row — Finding 4 RESOLVED in + * .convoys/reconcile-historical-add-scripts.md). + * + * Two-column-visibility smell: `is_public BOOLEAN` (from initial-schema) + * and `visibility VARCHAR(20)` (from this migration) coexist on prod. + * Surfaced as docs/SCHEMA_MAP.md § "Known schema smells" #2 → queued + * `unify-collection-visibility` is OUT OF SCOPE here; this migration + * adds `visibility` for parity, nothing more. + * + * Idempotent re-apply: every statement uses IF NOT EXISTS or the + * exception-swallowing DO block. + * + * @type {import('node-pg-migrate').ColumnDefinitions | undefined} + */ +export const shorthands = undefined; + +/** + * @param {import('node-pg-migrate').MigrationBuilder} pgm + */ +export const up = (pgm) => { + pgm.sql(` + ALTER TABLE collections + ADD COLUMN IF NOT EXISTS visibility VARCHAR(20) DEFAULT 'private', + ADD COLUMN IF NOT EXISTS tcg VARCHAR(50) DEFAULT 'MTG', + ADD COLUMN IF NOT EXISTS tags TEXT, + ADD COLUMN IF NOT EXISTS slug VARCHAR(100) UNIQUE, + ADD COLUMN IF NOT EXISTS image TEXT; + + CREATE INDEX IF NOT EXISTS idx_collections_visibility + ON collections (visibility); + + CREATE UNIQUE INDEX IF NOT EXISTS idx_collections_slug + ON collections (slug); + + DO $$ + BEGIN + ALTER TABLE collections + ADD CONSTRAINT check_slug_format + CHECK (slug ~ '^[a-z0-9]([a-z0-9-]*[a-z0-9])?$' AND length(slug) <= 50); + EXCEPTION + WHEN duplicate_object THEN NULL; + END $$; + `); +}; + +/** + * Down-migration intentionally throws. Removing these columns on + * long-lived envs would drop user-curated tag / slug / image data + * and break the runtime code that reads collections.visibility, + * collections.slug, collections.image, collections.tcg, collections.tags. + * + * If a future schema correction needs to mutate any of these columns, + * write a NEW dated migration with a real `down()` — do NOT roll back + * this one. + * + * @returns {void} + */ +export const down = () => { + throw new Error( + '[migration:1781000000002_reconcile-collections-columns] Down not supported. ' + + 'Dropping collections.visibility / tcg / tags / slug / image would erase ' + + 'user-curated data and break runtime reads. Write a new dated migration ' + + 'for any future schema correction.' + ); +}; +``` + +## Conventions to follow + +- `.cursor/rules/db-and-schema.mdc` § "Schema source of truth" — one migration per table/feature surface. +- `.cursor/rules/no-go-zones.mdc` — all three source scripts are append-only history. **Do not edit any of them.** +- Style: raw `pgm.sql(...)` template literals, matching the other migrations under `migrations/`. Per D2 of the convoy file. +- ESM exports; `"type": "module"` per `package.json` line 5. +- `IF NOT EXISTS` on every ALTER and CREATE INDEX. CHECK constraint wraps in `DO $$ ... EXCEPTION WHEN duplicate_object THEN NULL END $$;` (Postgres doesn't accept `IF NOT EXISTS` directly on CHECK constraints). + +## Acceptance criteria + +- [ ] `migrations/1781000000002_reconcile-collections-columns.js` exists with the exact filename above (the timestamp `1781000000002` is the reservation token assigned by the architect — do NOT use `npm run migrate create`). +- [ ] The file's `up()` adds 5 columns (`visibility`, `tcg`, `tags`, `slug`, `image`) via `IF NOT EXISTS`, creates 2 indexes (`idx_collections_visibility`, unique `idx_collections_slug`) via `IF NOT EXISTS`, and adds the `check_slug_format` CHECK via a `DO $$ EXCEPTION` block. +- [ ] The file's `down()` throws with a clear message. +- [ ] The file's docstring cites all three source scripts + the convoy file + the visibility smell (#2 in SCHEMA_MAP). +- [ ] Per-row slug backfill DML is NOT in the migration (out of scope per the convoy plan). +- [ ] `node --check migrations/1781000000002_reconcile-collections-columns.js` passes. +- [ ] `node -e "import('./migrations/1781000000002_reconcile-collections-columns.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined`. +- [ ] `npm run lint` matches the baseline (no new errors). +- [ ] `npm run test:run` reports 21/21 passing. + +## Verification + +Run from the convoy worktree BEFORE opening the PR: + +```bash +node --check migrations/1781000000002_reconcile-collections-columns.js +node -e "import('./migrations/1781000000002_reconcile-collections-columns.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))" +npm run lint +npm run test:run +``` + +Expected: +- `node --check` exits 0 silently. +- `node -e` prints `function function undefined`. +- `npm run lint` matches baseline. +- `npm run test:run` reports 21/21 passing. + +**Do NOT** run `npm run migrate up` against any environment — that's the post-merge operator step covered by Brief 7's verification runbook. + +## Commit message + +``` +feat(migrations): reconcile collections columns into migration history (brief 2/7) + +Captures the DDL effect of three historical scripts into one new +node-pg-migrate migration: + + - scripts/add-collaboration-features.js (collections half: visibility, + tcg, tags + idx_collections_visibility) + - scripts/add-collection-slugs.js (slug + idx_collections_slug unique + + check_slug_format CHECK constraint) + - scripts/add-image-column.js (image) + +The collaboration tables half (collection_permissions, collection_activity, +users.is_pending) is reconciled separately by Brief 3. + +Per-row slug backfill DML from add-collection-slugs.js is intentionally +NOT folded in — fresh envs have no pre-existing collections to backfill. + +Per .convoys/reconcile-historical-add-scripts.md § Brief outline → B2. +Idempotent re-apply (IF NOT EXISTS guards; DO $$ EXCEPTION for CHECK). +``` + +## PR shape + +**Title:** `feat(migrations): reconcile collections columns into migration history (brief 2/7)` + +**Body template:** + +```markdown +Brief 2 of the `reconcile-historical-add-scripts` convoy. See +[`.convoys/reconcile-historical-add-scripts.md`](../.convoys/reconcile-historical-add-scripts.md) +for the full plan and rationale. + +## What this PR does + +Adds `migrations/1781000000002_reconcile-collections-columns.js` — +captures the DDL effect of three historical scripts in a single new +`node-pg-migrate` migration: + +- `scripts/add-collaboration-features.js` — collections columns half + (`visibility VARCHAR(20)`, `tcg VARCHAR(50)`, `tags TEXT`, + `idx_collections_visibility` index) +- `scripts/add-collection-slugs.js` — `slug VARCHAR(100) UNIQUE`, + `idx_collections_slug` unique index, `check_slug_format` CHECK + constraint +- `scripts/add-image-column.js` — `image TEXT` + +All ALTERs use `ADD COLUMN IF NOT EXISTS`. The CHECK constraint wraps +in a `DO $$ ... EXCEPTION WHEN duplicate_object THEN NULL END $$;` +block because Postgres doesn't accept `ADD CONSTRAINT ... IF NOT EXISTS` +for CHECK. + +## What this PR does NOT do + +- Does **NOT** edit any of the three source scripts (no-go-zones). +- Does **NOT** capture the collaboration tables half of + `add-collaboration-features.js` (`collection_permissions`, + `collection_activity`, `users.is_pending`, related indexes, + owner-permission backfill DML) — that's **Brief 3**. +- Does **NOT** fold in the per-row slug backfill DML from + `add-collection-slugs.js` — fresh envs have no pre-existing + collections to backfill. +- Does **NOT** unify the `is_public` / `visibility` redundancy + (SCHEMA_MAP smell #2) — that's a future `unify-collection-visibility` + follow-up. +- Does **NOT** edit `scripts/setup-neon-db.js`, `package.json`, README, + or AGENTS.md. +- Does **NOT** run `npm run migrate up` against any environment. + +## Verification checklist + +- [ ] `node --check migrations/1781000000002_reconcile-collections-columns.js` exits 0 +- [ ] `node -e "import('./migrations/1781000000002_reconcile-collections-columns.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined` +- [ ] `npm run lint` matches baseline +- [ ] `npm run test:run` reports 21/21 passing +- [ ] Did not run `npm run migrate up` against any environment in this PR + +## Cross-references + +- Convoy file: `.convoys/reconcile-historical-add-scripts.md` +- Source scripts (no-go-zones, audit reference only): + `scripts/add-collaboration-features.js`, + `scripts/add-collection-slugs.js`, + `scripts/add-image-column.js` +``` + +## DO NOT + +- DO NOT edit `scripts/add-collaboration-features.js`, `scripts/add-collection-slugs.js`, `scripts/add-image-column.js`, or any other file under `scripts/` — append-only no-go-zone. +- DO NOT edit any other file under `migrations/`. +- DO NOT edit `scripts/setup-neon-db.js`, `package.json`, README, `AGENTS.md`, or `docs/SCHEMA_MAP.md` (B7 owns SCHEMA_MAP). +- DO NOT run `npm run migrate up` against any environment. +- DO NOT include the per-row slug backfill DML — out of scope. +- DO NOT add work for `collection_permissions` / `collection_activity` / `users.is_pending` — that's Brief 3. +- DO NOT call `npm run migrate create` — it would generate a `Date.now()` timestamp colliding with B1/B3/B4/B5's reservation tokens. + +## Rationale (≤3 sentences) + +Grouping all `collections`-table DDL into one migration matches the per-table grouping of D1 and keeps the diff focused on a single surface. Splitting `add-collaboration-features.js` between this brief (columns) and Brief 3 (tables) avoids creating an artificial dependency between two parallel briefs — each writes only its own new file. The CHECK-constraint exception block matches the historical script's try/catch pattern and is the idiomatic Postgres way to do idempotent CHECK adds. diff --git a/.convoys/reconcile-historical-add-scripts/brief-3-reconcile-collaboration-tables.md b/.convoys/reconcile-historical-add-scripts/brief-3-reconcile-collaboration-tables.md new file mode 100644 index 0000000..7324a2a --- /dev/null +++ b/.convoys/reconcile-historical-add-scripts/brief-3-reconcile-collaboration-tables.md @@ -0,0 +1,318 @@ +--- +convoy: reconcile-historical-add-scripts +brief_number: 3 +depends_on: [] +files: + - migrations/1781000000003_reconcile-collaboration-tables.js +--- + +# Brief 3: Reconcile collaboration tables into migration history + +## Goal (1 sentence) + +Capture the `collection_permissions` + `collection_activity` table creation, the `users.is_pending` column, and the 3 related indexes from `scripts/add-collaboration-features.js` into a single new `node-pg-migrate` migration so a brand-new Neon branch ends up with the collaboration / sharing surface after `npm run setup-db`. + +## Scope (files in scope — do not edit anything else) + +- `migrations/1781000000003_reconcile-collaboration-tables.js` — **new** + +## Source script (read-only audit reference; DO NOT EDIT — no-go-zone) + +`scripts/add-collaboration-features.js` lines 24-81 (relevant DDL only; the collections-columns half is reconciled by Brief 2): + +```js +// collection_permissions table +await sql` + CREATE TABLE IF NOT EXISTS collection_permissions ( + id SERIAL PRIMARY KEY, + collection_id INTEGER REFERENCES collections(id) ON DELETE CASCADE, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + role VARCHAR(20) NOT NULL CHECK (role IN ('owner', 'editor', 'viewer')), + status VARCHAR(20) DEFAULT 'active' CHECK (status IN ('active', 'pending', 'declined')), + invite_token VARCHAR(255) UNIQUE, + invited_by INTEGER REFERENCES users(id), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(collection_id, user_id) + ) +`; + +// collection_activity table +await sql` + CREATE TABLE IF NOT EXISTS collection_activity ( + id SERIAL PRIMARY KEY, + collection_id INTEGER REFERENCES collections(id) ON DELETE CASCADE, + user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + action VARCHAR(50) NOT NULL, + details JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) +`; + +// users.is_pending column +await sql` + ALTER TABLE users + ADD COLUMN IF NOT EXISTS is_pending BOOLEAN DEFAULT false +`; + +// 3 indexes (the 4th — idx_collections_visibility — is Brief 2's scope) +await sql` + CREATE INDEX IF NOT EXISTS idx_collection_permissions_collection_id + ON collection_permissions(collection_id) +`; +await sql` + CREATE INDEX IF NOT EXISTS idx_collection_permissions_user_id + ON collection_permissions(user_id) +`; +await sql` + CREATE INDEX IF NOT EXISTS idx_collection_activity_collection_id + ON collection_activity(collection_id) +`; +``` + +**Owner-permission backfill DML** at lines 84-99 inserts `('owner', 'active')` rows for every pre-existing collection. **Do NOT fold the backfill DML into this migration.** Fresh envs have no pre-existing collections needing backfill; on prod, the backfill ran once historically and is baked in. The current runtime invariant for new collection creation lives in `pages/api/collections.js` (verify post-merge if needed — out of scope for this brief). + +## Target migration file + +Path: `migrations/1781000000003_reconcile-collaboration-tables.js` + +Contents: + +```js +/** + * Reconcile the collaboration / sharing half of + * scripts/add-collaboration-features.js into the migration history: + * + * - CREATE TABLE collection_permissions (with role/status CHECK + * constraints inline + invite_token UNIQUE + UNIQUE(collection_id, + * user_id)) + * - CREATE TABLE collection_activity (with JSONB details column) + * - ALTER users ADD COLUMN is_pending BOOLEAN DEFAULT false + * - 3 indexes (idx_collection_permissions_collection_id, + * idx_collection_permissions_user_id, + * idx_collection_activity_collection_id) + * + * The 4th index from the source script (idx_collections_visibility) + * is reconciled by migrations/1781000000002_reconcile-collections-columns.js + * because it indexes a column added in that brief. + * + * The collections-columns half (visibility, tcg, tags) of + * add-collaboration-features.js is reconciled by + * migrations/1781000000002_reconcile-collections-columns.js. + * + * The owner-permission backfill DML from the source script (INSERT + * INTO collection_permissions ... 'owner', 'active' for every + * pre-existing collection) is intentionally NOT folded in — fresh + * envs have no pre-existing collections to backfill; on prod, the + * backfill ran once historically and is baked in. The runtime + * invariant for owner-permission creation on new collections is the + * responsibility of pages/api/collections.js (out of scope here). + * + * Both CREATE TABLE statements use IF NOT EXISTS, with CHECK + * constraints declared inline (no idempotency issue — IF NOT EXISTS + * on the parent table makes the whole CREATE a no-op when the table + * already exists, CHECK constraints and all). + * + * Idempotent re-apply: every statement uses IF NOT EXISTS. + * + * @type {import('node-pg-migrate').ColumnDefinitions | undefined} + */ +export const shorthands = undefined; + +/** + * @param {import('node-pg-migrate').MigrationBuilder} pgm + */ +export const up = (pgm) => { + pgm.sql(` + CREATE TABLE IF NOT EXISTS collection_permissions ( + id SERIAL PRIMARY KEY, + collection_id INTEGER REFERENCES collections(id) ON DELETE CASCADE, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + role VARCHAR(20) NOT NULL CHECK (role IN ('owner', 'editor', 'viewer')), + status VARCHAR(20) DEFAULT 'active' CHECK (status IN ('active', 'pending', 'declined')), + invite_token VARCHAR(255) UNIQUE, + invited_by INTEGER REFERENCES users(id), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(collection_id, user_id) + ); + + CREATE TABLE IF NOT EXISTS collection_activity ( + id SERIAL PRIMARY KEY, + collection_id INTEGER REFERENCES collections(id) ON DELETE CASCADE, + user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + action VARCHAR(50) NOT NULL, + details JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + ALTER TABLE users + ADD COLUMN IF NOT EXISTS is_pending BOOLEAN DEFAULT false; + + CREATE INDEX IF NOT EXISTS idx_collection_permissions_collection_id + ON collection_permissions (collection_id); + + CREATE INDEX IF NOT EXISTS idx_collection_permissions_user_id + ON collection_permissions (user_id); + + CREATE INDEX IF NOT EXISTS idx_collection_activity_collection_id + ON collection_activity (collection_id); + `); +}; + +/** + * Down-migration intentionally throws. Dropping collection_permissions + * + collection_activity on a long-lived env would erase every active + * sharing relationship + every audit trail row. The runtime in + * pages/api/collections/[identifier]/permissions.js, pages/api/invite/*.js, + * and lib/permission-middleware.js all read these tables; rolling back + * would break the live app. + * + * @returns {void} + */ +export const down = () => { + throw new Error( + '[migration:1781000000003_reconcile-collaboration-tables] Down not supported. ' + + 'Dropping collection_permissions / collection_activity would erase every ' + + 'sharing relationship and audit trail, and break runtime reads in ' + + 'pages/api/collections/[identifier]/permissions.js, pages/api/invite/*.js, ' + + 'lib/permission-middleware.js. Write a new dated migration for any future ' + + 'schema correction.' + ); +}; +``` + +## Conventions to follow + +- `.cursor/rules/db-and-schema.mdc` § "Schema source of truth" — `migrations/` is canonical; one migration per feature surface. +- `.cursor/rules/no-go-zones.mdc` — `scripts/add-collaboration-features.js` is append-only history. **Do not edit it.** +- Style: raw `pgm.sql(...)` template literals matching the other migrations. Per D2. +- ESM exports; `"type": "module"`. +- `IF NOT EXISTS` on every CREATE / ALTER. Inline CHECK constraints on `CREATE TABLE` are fine — `CREATE TABLE IF NOT EXISTS` skips the entire statement (constraints and all) when the table exists. +- FK declarations match the source script verbatim (`ON DELETE CASCADE` for primary FKs, `ON DELETE SET NULL` where the source uses it). + +## Acceptance criteria + +- [ ] `migrations/1781000000003_reconcile-collaboration-tables.js` exists with the exact filename above. +- [ ] The file's `up()`: + - Creates `collection_permissions` table with all 10 columns + role/status CHECKs + invite_token UNIQUE + UNIQUE(collection_id, user_id), all via `CREATE TABLE IF NOT EXISTS`. + - Creates `collection_activity` table with 6 columns including JSONB `details`, via `CREATE TABLE IF NOT EXISTS`. + - Adds `users.is_pending BOOLEAN DEFAULT false` via `ADD COLUMN IF NOT EXISTS`. + - Creates 3 indexes via `CREATE INDEX IF NOT EXISTS`. +- [ ] The 4th index from the source script (`idx_collections_visibility`) is NOT in this migration — it belongs to Brief 2. +- [ ] The owner-permission backfill DML is NOT in this migration. +- [ ] The file's `down()` throws with a clear message. +- [ ] The file's docstring cites the source script + the convoy file + the Brief 2 split. +- [ ] `node --check migrations/1781000000003_reconcile-collaboration-tables.js` passes. +- [ ] `node -e "import('./migrations/1781000000003_reconcile-collaboration-tables.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined`. +- [ ] `npm run lint` matches baseline. +- [ ] `npm run test:run` reports 21/21 passing. + +## Verification + +```bash +node --check migrations/1781000000003_reconcile-collaboration-tables.js +node -e "import('./migrations/1781000000003_reconcile-collaboration-tables.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))" +npm run lint +npm run test:run +``` + +Expected: `node --check` silent, `node -e` prints `function function undefined`, lint baseline, 21/21 tests pass. + +**Do NOT** run `npm run migrate up` against any environment. + +## Commit message + +``` +feat(migrations): reconcile collaboration tables into migration history (brief 3/7) + +Captures the collaboration / sharing half of +scripts/add-collaboration-features.js into one new node-pg-migrate +migration: + + - CREATE TABLE collection_permissions (role/status CHECKs, + invite_token UNIQUE, UNIQUE(collection_id, user_id)) + - CREATE TABLE collection_activity (JSONB details) + - ALTER users ADD COLUMN is_pending BOOLEAN DEFAULT false + - 3 indexes + +The collections-columns half (visibility, tcg, tags + +idx_collections_visibility) is reconciled by Brief 2. The 4th index +(idx_collections_visibility) belongs to Brief 2 because it indexes a +column added there. + +Per-row owner-permission backfill DML from the source script is +intentionally NOT folded in — fresh envs have no pre-existing +collections to backfill; the runtime invariant for new-collection +owner-perm creation lives in pages/api/collections.js. + +Per .convoys/reconcile-historical-add-scripts.md § Brief outline → B3. +Idempotent re-apply (IF NOT EXISTS guards). +``` + +## PR shape + +**Title:** `feat(migrations): reconcile collaboration tables into migration history (brief 3/7)` + +**Body template:** + +```markdown +Brief 3 of the `reconcile-historical-add-scripts` convoy. See +[`.convoys/reconcile-historical-add-scripts.md`](../.convoys/reconcile-historical-add-scripts.md) +for the full plan and rationale. + +## What this PR does + +Adds `migrations/1781000000003_reconcile-collaboration-tables.js` — +captures the collaboration / sharing DDL of +`scripts/add-collaboration-features.js` (the table-and-column half; +the columns-on-collections half is Brief 2). + +- `CREATE TABLE IF NOT EXISTS collection_permissions` (10 columns + including inline role/status CHECK constraints + invite_token + UNIQUE + UNIQUE(collection_id, user_id)) +- `CREATE TABLE IF NOT EXISTS collection_activity` (6 columns + including JSONB `details`) +- `ALTER TABLE users ADD COLUMN IF NOT EXISTS is_pending BOOLEAN DEFAULT false` +- 3 indexes via `CREATE INDEX IF NOT EXISTS` + +## What this PR does NOT do + +- Does **NOT** edit `scripts/add-collaboration-features.js` (no-go-zone). +- Does **NOT** capture the columns-on-collections half (`visibility`, + `tcg`, `tags`, `idx_collections_visibility`) — that's **Brief 2**. +- Does **NOT** fold in the per-row owner-permission backfill DML from + the source script — fresh envs have no pre-existing collections to + backfill. +- Does **NOT** edit `scripts/setup-neon-db.js`, `package.json`, README, + `AGENTS.md`, or `docs/SCHEMA_MAP.md` (B7 owns SCHEMA_MAP). +- Does **NOT** run `npm run migrate up` against any environment. + +## Verification checklist + +- [ ] `node --check migrations/1781000000003_reconcile-collaboration-tables.js` exits 0 +- [ ] `node -e "import('./migrations/1781000000003_reconcile-collaboration-tables.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined` +- [ ] `npm run lint` matches baseline +- [ ] `npm run test:run` reports 21/21 passing +- [ ] Did not run `npm run migrate up` against any environment in this PR + +## Cross-references + +- Convoy file: `.convoys/reconcile-historical-add-scripts.md` +- Source script (no-go-zone, audit reference only): + `scripts/add-collaboration-features.js` +``` + +## DO NOT + +- DO NOT edit `scripts/add-collaboration-features.js` or any file under `scripts/`. +- DO NOT edit any other file under `migrations/`. +- DO NOT edit `scripts/setup-neon-db.js`, `package.json`, README, `AGENTS.md`, or `docs/SCHEMA_MAP.md`. +- DO NOT run `npm run migrate up` against any environment. +- DO NOT include the owner-permission backfill DML. +- DO NOT add work for the collections-columns half (`visibility`, `tcg`, `tags`, `idx_collections_visibility`) — that's Brief 2. +- DO NOT call `npm run migrate create`. + +## Rationale (≤3 sentences) + +Splitting `add-collaboration-features.js` between Brief 2 (collections columns + the visibility index that indexes one of those columns) and Brief 3 (collaboration tables + the users.is_pending column + the 3 indexes that index collaboration-table columns) keeps each migration scoped to the table surface it touches, matching D1. Inline CHECK constraints on `CREATE TABLE` are idempotent for free under `CREATE TABLE IF NOT EXISTS` (the whole statement no-ops when the table exists). The backfill DML is intentionally out of scope because fresh envs need no backfill and prod's backfill already ran. diff --git a/.convoys/reconcile-historical-add-scripts/brief-4-reconcile-favorites-system.md b/.convoys/reconcile-historical-add-scripts/brief-4-reconcile-favorites-system.md new file mode 100644 index 0000000..776ccbb --- /dev/null +++ b/.convoys/reconcile-historical-add-scripts/brief-4-reconcile-favorites-system.md @@ -0,0 +1,241 @@ +--- +convoy: reconcile-historical-add-scripts +brief_number: 4 +depends_on: [] +files: + - migrations/1781000000004_reconcile-favorites-system.js +--- + +# Brief 4: Reconcile favorites system into migration history + +## Goal (1 sentence) + +Capture the `user_favorites` table and its 4 indexes from `scripts/add-favorites-system.js` into a single new `node-pg-migrate` migration so a brand-new Neon branch ends up with the favorites surface after `npm run setup-db`. + +## Scope (files in scope — do not edit anything else) + +- `migrations/1781000000004_reconcile-favorites-system.js` — **new** + +## Source script (read-only audit reference; DO NOT EDIT — no-go-zone) + +`scripts/add-favorites-system.js` lines 11-27 (entire DDL — script has no DML beyond the table create): + +```js +await sql` + CREATE TABLE IF NOT EXISTS user_favorites ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + item_type VARCHAR(50) NOT NULL, -- 'card', 'collection', 'deck' + item_id INTEGER NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(user_id, item_type, item_id) + ) +`; + +// 4 indexes +await sql`CREATE INDEX IF NOT EXISTS idx_user_favorites_user_id ON user_favorites(user_id)`; +await sql`CREATE INDEX IF NOT EXISTS idx_user_favorites_item_type ON user_favorites(item_type)`; +await sql`CREATE INDEX IF NOT EXISTS idx_user_favorites_item_id ON user_favorites(item_id)`; +await sql`CREATE INDEX IF NOT EXISTS idx_user_favorites_user_type ON user_favorites(user_id, item_type)`; +``` + +**Note on SCHEMA_MAP drift.** `docs/SCHEMA_MAP.md` § `user_favorites` (lines 104-111) shows the table with only `(user_id, card_id)` columns — that's a **doc bug**. The actual source script (and prod schema) uses the polymorphic `(item_type, item_id)` shape that supports cards, collections, AND decks (per the script's inline comment + the runtime usage in `pages/api/favorites.js`). B7 corrects the SCHEMA_MAP entry. This brief faithfully reproduces the **script's** shape — `(item_type, item_id)` — not the doc's. + +## Target migration file + +Path: `migrations/1781000000004_reconcile-favorites-system.js` + +Contents: + +```js +/** + * Reconcile scripts/add-favorites-system.js into the migration history. + * + * Creates user_favorites with the polymorphic (item_type, item_id) + * shape that supports favoriting cards, collections, and decks via a + * single table. Adds 4 indexes for the common query shapes: + * + * - idx_user_favorites_user_id — "all favorites for user X" + * - idx_user_favorites_item_type — "all card favorites" / "all deck favorites" + * - idx_user_favorites_item_id — back-link from an item to its favoriters + * - idx_user_favorites_user_type — composite for "user X's card favorites" + * + * Note: docs/SCHEMA_MAP.md § user_favorites currently shows only + * (user_id, card_id) — that's a doc bug. The actual prod shape (and + * the source script, and the runtime in pages/api/favorites.js) uses + * the polymorphic shape. B7 of this convoy corrects the SCHEMA_MAP + * entry; this migration reproduces the script's shape faithfully. + * + * Idempotent re-apply: CREATE TABLE IF NOT EXISTS + CREATE INDEX IF NOT EXISTS. + * + * @type {import('node-pg-migrate').ColumnDefinitions | undefined} + */ +export const shorthands = undefined; + +/** + * @param {import('node-pg-migrate').MigrationBuilder} pgm + */ +export const up = (pgm) => { + pgm.sql(` + CREATE TABLE IF NOT EXISTS user_favorites ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + item_type VARCHAR(50) NOT NULL, + item_id INTEGER NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(user_id, item_type, item_id) + ); + + CREATE INDEX IF NOT EXISTS idx_user_favorites_user_id + ON user_favorites (user_id); + + CREATE INDEX IF NOT EXISTS idx_user_favorites_item_type + ON user_favorites (item_type); + + CREATE INDEX IF NOT EXISTS idx_user_favorites_item_id + ON user_favorites (item_id); + + CREATE INDEX IF NOT EXISTS idx_user_favorites_user_type + ON user_favorites (user_id, item_type); + `); +}; + +/** + * Down-migration intentionally throws. Dropping user_favorites on a + * long-lived env would erase every user's saved favorites list and + * break runtime reads in pages/api/favorites.js. + * + * @returns {void} + */ +export const down = () => { + throw new Error( + '[migration:1781000000004_reconcile-favorites-system] Down not supported. ' + + 'Dropping user_favorites would erase every saved favorite and break ' + + 'pages/api/favorites.js. Write a new dated migration for any future ' + + 'schema correction.' + ); +}; +``` + +## Conventions to follow + +- `.cursor/rules/db-and-schema.mdc` § "Schema source of truth" — `migrations/` is canonical. +- `.cursor/rules/no-go-zones.mdc` — `scripts/add-favorites-system.js` is append-only history. +- Style: raw `pgm.sql(...)` template literals. Per D2. +- ESM exports; `"type": "module"`. +- `IF NOT EXISTS` on every CREATE. +- Match the source script's column types, FK ON DELETE rule (`CASCADE`), and UNIQUE shape verbatim. + +## Acceptance criteria + +- [ ] `migrations/1781000000004_reconcile-favorites-system.js` exists with the exact filename above. +- [ ] The file's `up()` creates `user_favorites` with all 5 columns + UNIQUE constraint, then 4 indexes, all via `IF NOT EXISTS`. +- [ ] The column shape is `(item_type VARCHAR(50), item_id INTEGER)` — the polymorphic shape — NOT `(card_id)`. See "Note on SCHEMA_MAP drift" above. +- [ ] The file's `down()` throws with a clear message. +- [ ] The file's docstring cites the source script + the convoy file + the SCHEMA_MAP doc-bug note. +- [ ] `node --check migrations/1781000000004_reconcile-favorites-system.js` passes. +- [ ] `node -e "import('./migrations/1781000000004_reconcile-favorites-system.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined`. +- [ ] `npm run lint` matches baseline. +- [ ] `npm run test:run` reports 21/21 passing. + +## Verification + +```bash +node --check migrations/1781000000004_reconcile-favorites-system.js +node -e "import('./migrations/1781000000004_reconcile-favorites-system.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))" +npm run lint +npm run test:run +``` + +Expected: `node --check` silent, `node -e` prints `function function undefined`, lint baseline, 21/21 tests pass. + +**Do NOT** run `npm run migrate up` against any environment. + +## Commit message + +``` +feat(migrations): reconcile favorites system into migration history (brief 4/7) + +Captures scripts/add-favorites-system.js into one new node-pg-migrate +migration: + + - CREATE TABLE user_favorites (polymorphic (item_type, item_id) + shape; UNIQUE(user_id, item_type, item_id)) + - 4 indexes (user_id, item_type, item_id, (user_id, item_type)) + +Note: docs/SCHEMA_MAP.md § user_favorites currently shows the table +with only (user_id, card_id) — that's a doc bug; B7 of this convoy +corrects the entry. This migration faithfully reproduces the source +script's polymorphic shape (also what runtime in pages/api/favorites.js +uses). + +Per .convoys/reconcile-historical-add-scripts.md § Brief outline → B4. +Idempotent re-apply (IF NOT EXISTS guards). +``` + +## PR shape + +**Title:** `feat(migrations): reconcile favorites system into migration history (brief 4/7)` + +**Body template:** + +```markdown +Brief 4 of the `reconcile-historical-add-scripts` convoy. See +[`.convoys/reconcile-historical-add-scripts.md`](../.convoys/reconcile-historical-add-scripts.md) +for the full plan and rationale. + +## What this PR does + +Adds `migrations/1781000000004_reconcile-favorites-system.js` — +captures `scripts/add-favorites-system.js` into a single new +`node-pg-migrate` migration. + +- `CREATE TABLE IF NOT EXISTS user_favorites` with polymorphic + `(item_type VARCHAR(50), item_id INTEGER)` shape supporting + cards / collections / decks favorites in one table. +- `UNIQUE(user_id, item_type, item_id)` prevents duplicate favorites. +- 4 supporting indexes via `CREATE INDEX IF NOT EXISTS`. + +## Note on a SCHEMA_MAP doc bug + +`docs/SCHEMA_MAP.md` § `user_favorites` (current `main`) shows the +table with only `(user_id, card_id)` columns. That's a doc bug — the +actual prod shape (and the source script, and `pages/api/favorites.js` +runtime usage) uses the polymorphic `(item_type, item_id)` shape. This +migration faithfully reproduces the script's shape; Brief 7 of this +convoy fixes the SCHEMA_MAP entry. + +## What this PR does NOT do + +- Does **NOT** edit `scripts/add-favorites-system.js` (no-go-zone). +- Does **NOT** edit `docs/SCHEMA_MAP.md` (Brief 7's job). +- Does **NOT** edit `scripts/setup-neon-db.js`, `package.json`, README, + or `AGENTS.md`. +- Does **NOT** run `npm run migrate up` against any environment. + +## Verification checklist + +- [ ] `node --check migrations/1781000000004_reconcile-favorites-system.js` exits 0 +- [ ] `node -e "import('./migrations/1781000000004_reconcile-favorites-system.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined` +- [ ] `npm run lint` matches baseline +- [ ] `npm run test:run` reports 21/21 passing +- [ ] Did not run `npm run migrate up` against any environment in this PR + +## Cross-references + +- Convoy file: `.convoys/reconcile-historical-add-scripts.md` +- Source script (no-go-zone, audit reference only): `scripts/add-favorites-system.js` +``` + +## DO NOT + +- DO NOT edit `scripts/add-favorites-system.js` or any file under `scripts/`. +- DO NOT edit any other file under `migrations/`. +- DO NOT edit `scripts/setup-neon-db.js`, `package.json`, README, `AGENTS.md`, or `docs/SCHEMA_MAP.md` (B7 owns SCHEMA_MAP). +- DO NOT run `npm run migrate up` against any environment. +- DO NOT use the SCHEMA_MAP `(user_id, card_id)` shape — that doc entry is wrong; reproduce the SOURCE SCRIPT'S polymorphic shape. +- DO NOT call `npm run migrate create`. + +## Rationale (≤3 sentences) + +The favorites system is a single self-contained table — natural fit for one small migration that maps 1:1 with the historical script. The polymorphic shape (`item_type`, `item_id`) is what runtime code actually uses, so reproducing it from the script's verbatim DDL — rather than the stale `(user_id, card_id)` doc entry — is the safe choice. Brief 7's SCHEMA_MAP rewrite resolves the doc-bug separately so this brief stays scoped to one new file. diff --git a/.convoys/reconcile-historical-add-scripts/brief-5-reconcile-user-profile.md b/.convoys/reconcile-historical-add-scripts/brief-5-reconcile-user-profile.md new file mode 100644 index 0000000..05fb09c --- /dev/null +++ b/.convoys/reconcile-historical-add-scripts/brief-5-reconcile-user-profile.md @@ -0,0 +1,425 @@ +--- +convoy: reconcile-historical-add-scripts +brief_number: 5 +depends_on: [] +files: + - migrations/1781000000005_reconcile-user-profile.js +--- + +# Brief 5: Reconcile user profile fields into migration history + +## Goal (1 sentence) + +Capture the deduplicated union of `scripts/add-user-profile-columns.js` and `scripts/add-user-profile-fields.js` (15 new `users` columns, 2 new tables `user_settings` + `user_avatars`, 6 CHECK constraints, 6 indexes) into a single new `node-pg-migrate` migration so a brand-new Neon branch ends up with the user-profile surface after `npm run setup-db`. + +## Scope (files in scope — do not edit anything else) + +- `migrations/1781000000005_reconcile-user-profile.js` — **new** + +## Source scripts (read-only audit reference; DO NOT EDIT — both are no-go-zones) + +### `scripts/add-user-profile-columns.js` lines 12-18 (the earlier, narrower script): + +```js +await sql` + ALTER TABLE users + ADD COLUMN IF NOT EXISTS first_name VARCHAR(255), + ADD COLUMN IF NOT EXISTS last_name VARCHAR(255), + ADD COLUMN IF NOT EXISTS username VARCHAR(255) UNIQUE, + ADD COLUMN IF NOT EXISTS profile_image_url TEXT +`; +``` + +Plus per-row UPDATE backfill of defaults (lines 25-42). **Do NOT fold the backfill DML into this migration** — fresh envs have no rows to backfill; column DEFAULTs handle new rows. + +### `scripts/add-user-profile-fields.js` lines 27-115 (the later, broader script — superset of #8 plus additional columns + 2 new tables + 6 CHECK constraints + 6 indexes): + +```js +// Basic profile fields (overlaps add-user-profile-columns.js for first_name/last_name/username, +// but ADDS bio + avatar_url; idempotent overlap because of IF NOT EXISTS) +await sql` + ALTER TABLE users + ADD COLUMN IF NOT EXISTS first_name VARCHAR(255), + ADD COLUMN IF NOT EXISTS last_name VARCHAR(255), + ADD COLUMN IF NOT EXISTS username VARCHAR(255) UNIQUE, + ADD COLUMN IF NOT EXISTS bio TEXT, + ADD COLUMN IF NOT EXISTS avatar_url TEXT +`; + +// Preference fields +await sql` + ALTER TABLE users + ADD COLUMN IF NOT EXISTS favorite_games JSONB DEFAULT '["MTG"]', + ADD COLUMN IF NOT EXISTS collection_visibility VARCHAR(20) DEFAULT 'private', + ADD COLUMN IF NOT EXISTS preferred_currency VARCHAR(3) DEFAULT 'USD', + ADD COLUMN IF NOT EXISTS cards_per_page INTEGER DEFAULT 50, + ADD COLUMN IF NOT EXISTS default_view VARCHAR(10) DEFAULT 'grid' +`; + +// Notification + 2FA settings +await sql` + ALTER TABLE users + ADD COLUMN IF NOT EXISTS notifications_email BOOLEAN DEFAULT true, + ADD COLUMN IF NOT EXISTS notifications_marketing BOOLEAN DEFAULT false, + ADD COLUMN IF NOT EXISTS two_factor_enabled BOOLEAN DEFAULT false +`; + +// Display settings +await sql` + ALTER TABLE users + ADD COLUMN IF NOT EXISTS theme VARCHAR(10) DEFAULT 'system', + ADD COLUMN IF NOT EXISTS language VARCHAR(5) DEFAULT 'en' +`; + +// user_settings table +await sql` + CREATE TABLE IF NOT EXISTS user_settings ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + setting_key VARCHAR(100) NOT NULL, + setting_value JSONB NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(user_id, setting_key) + ) +`; + +// user_avatars table +await sql` + CREATE TABLE IF NOT EXISTS user_avatars ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(255) NOT NULL, + original_name VARCHAR(255), + mime_type VARCHAR(100), + file_size INTEGER, + file_path TEXT NOT NULL, + is_active BOOLEAN DEFAULT true, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) +`; + +// 6 indexes +await sql`CREATE INDEX IF NOT EXISTS idx_users_username ON users(username)`; +await sql`CREATE INDEX IF NOT EXISTS idx_users_email ON users(email)`; +await sql`CREATE INDEX IF NOT EXISTS idx_user_settings_user_id ON user_settings(user_id)`; +await sql`CREATE INDEX IF NOT EXISTS idx_user_settings_key ON user_settings(setting_key)`; +await sql`CREATE INDEX IF NOT EXISTS idx_user_avatars_user_id ON user_avatars(user_id)`; +await sql`CREATE INDEX IF NOT EXISTS idx_user_avatars_active ON user_avatars(user_id, is_active)`; + +// 6 CHECK constraints (each wrapped in JS try/catch to swallow 'already exists') +await sql`ALTER TABLE users ADD CONSTRAINT check_collection_visibility CHECK (collection_visibility IN ('private', 'public', 'unlisted'))`; +await sql`ALTER TABLE users ADD CONSTRAINT check_preferred_currency CHECK (preferred_currency IN ('USD', 'EUR', 'GBP', 'CAD', 'JPY'))`; +await sql`ALTER TABLE users ADD CONSTRAINT check_cards_per_page CHECK (cards_per_page IN (25, 50, 100))`; +await sql`ALTER TABLE users ADD CONSTRAINT check_default_view CHECK (default_view IN ('grid', 'list'))`; +await sql`ALTER TABLE users ADD CONSTRAINT check_theme CHECK (theme IN ('light', 'dark', 'system'))`; +await sql`ALTER TABLE users ADD CONSTRAINT check_language CHECK (language IN ('en', 'es', 'fr', 'de', 'ja'))`; +``` + +Plus a per-row UPDATE backfill of defaults (lines 198-222). **Do NOT fold the backfill DML in** — column DEFAULTs handle new rows; fresh envs have no rows to backfill. + +### Important note on the column dedup + +Scripts #8 and #9 overlap on `first_name`, `last_name`, `username` — both use `ADD COLUMN IF NOT EXISTS` so on prod the second-running script no-ops those three columns. Both scripts have run on prod, so the union of their columns is what's actually present: + +- From #8 only: `profile_image_url` (TEXT) — a column that ONLY #8 added. +- From #9 only: `bio`, `avatar_url`, `favorite_games`, `collection_visibility`, `preferred_currency`, `cards_per_page`, `default_view`, `notifications_email`, `notifications_marketing`, `two_factor_enabled`, `theme`, `language` (12 columns) + the 2 new tables + 6 CHECK constraints + 6 indexes. +- From both (idempotent overlap): `first_name`, `last_name`, `username`. + +The migration must add **all 15 columns** (4 from #8 ∪ 12 from #9 with 3 in the intersection = 4 + 12 - 3 = 13 unique users columns; wait, let me recount: #8 adds 4 (first_name, last_name, username, profile_image_url); #9 adds 5 basic (first_name, last_name, username, bio, avatar_url) + 5 prefs + 3 notif + 2 display = 15. Union: first_name, last_name, username (shared) + profile_image_url (#8) + bio, avatar_url, favorite_games, collection_visibility, preferred_currency, cards_per_page, default_view, notifications_email, notifications_marketing, two_factor_enabled, theme, language (#9) = **3 + 1 + 12 = 16 columns**). So the migration adds 16 columns to `users`. + +The redundant `profile_image_url` vs `avatar_url` pair is documented in `docs/SCHEMA_MAP.md` § "Known schema smells" #1 and surfaced as the `unify-user-avatar-column` follow-up. Both must be in fresh envs for parity. + +## Target migration file + +Path: `migrations/1781000000005_reconcile-user-profile.js` + +Contents: + +```js +/** + * Reconcile two overlapping historical user-profile scripts into the + * migration history, taking the union of their effects: + * + * - scripts/add-user-profile-columns.js (the earlier, narrower + * script): first_name, last_name, username UNIQUE, profile_image_url + * + * - scripts/add-user-profile-fields.js (the later, broader script; + * overlaps the earlier script on first_name / last_name / username + * and additionally adds): bio, avatar_url, favorite_games (JSONB + * DEFAULT '["MTG"]'), collection_visibility, preferred_currency, + * cards_per_page, default_view, notifications_email, + * notifications_marketing, two_factor_enabled, theme, language, + * + the new user_settings + user_avatars tables, + 6 CHECK + * constraints, + 6 indexes. + * + * Result on a fresh env: 16 new columns on `users`, 2 new tables, + * 6 CHECK constraints, 6 indexes. On any long-lived env: every + * statement is a no-op (IF NOT EXISTS / DO $$ EXCEPTION). + * + * Per-row UPDATE backfills from both scripts are intentionally NOT + * folded in — column DEFAULTs handle new rows; fresh envs have no + * rows to backfill. + * + * The profile_image_url / avatar_url redundancy is intentional for + * parity with prod and is flagged in docs/SCHEMA_MAP.md § "Known + * schema smells" #1; future cleanup is the queued + * `unify-user-avatar-column` follow-up. + * + * CHECK constraint adds wrap in DO $$ ... EXCEPTION WHEN + * duplicate_object THEN NULL END $$ because Postgres doesn't accept + * ADD CONSTRAINT ... IF NOT EXISTS for CHECK. Each constraint gets + * its own DO block so a failure in one doesn't block the rest. + * + * @type {import('node-pg-migrate').ColumnDefinitions | undefined} + */ +export const shorthands = undefined; + +/** + * @param {import('node-pg-migrate').MigrationBuilder} pgm + */ +export const up = (pgm) => { + pgm.sql(` + -- 16 columns on users (union of add-user-profile-columns.js + add-user-profile-fields.js) + ALTER TABLE users + ADD COLUMN IF NOT EXISTS first_name VARCHAR(255), + ADD COLUMN IF NOT EXISTS last_name VARCHAR(255), + ADD COLUMN IF NOT EXISTS username VARCHAR(255) UNIQUE, + ADD COLUMN IF NOT EXISTS profile_image_url TEXT, + ADD COLUMN IF NOT EXISTS bio TEXT, + ADD COLUMN IF NOT EXISTS avatar_url TEXT, + ADD COLUMN IF NOT EXISTS favorite_games JSONB DEFAULT '["MTG"]', + ADD COLUMN IF NOT EXISTS collection_visibility VARCHAR(20) DEFAULT 'private', + ADD COLUMN IF NOT EXISTS preferred_currency VARCHAR(3) DEFAULT 'USD', + ADD COLUMN IF NOT EXISTS cards_per_page INTEGER DEFAULT 50, + ADD COLUMN IF NOT EXISTS default_view VARCHAR(10) DEFAULT 'grid', + ADD COLUMN IF NOT EXISTS notifications_email BOOLEAN DEFAULT true, + ADD COLUMN IF NOT EXISTS notifications_marketing BOOLEAN DEFAULT false, + ADD COLUMN IF NOT EXISTS two_factor_enabled BOOLEAN DEFAULT false, + ADD COLUMN IF NOT EXISTS theme VARCHAR(10) DEFAULT 'system', + ADD COLUMN IF NOT EXISTS language VARCHAR(5) DEFAULT 'en'; + + -- user_settings table + CREATE TABLE IF NOT EXISTS user_settings ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + setting_key VARCHAR(100) NOT NULL, + setting_value JSONB NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(user_id, setting_key) + ); + + -- user_avatars table + CREATE TABLE IF NOT EXISTS user_avatars ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(255) NOT NULL, + original_name VARCHAR(255), + mime_type VARCHAR(100), + file_size INTEGER, + file_path TEXT NOT NULL, + is_active BOOLEAN DEFAULT true, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + -- 6 indexes + CREATE INDEX IF NOT EXISTS idx_users_username ON users (username); + CREATE INDEX IF NOT EXISTS idx_users_email ON users (email); + CREATE INDEX IF NOT EXISTS idx_user_settings_user_id ON user_settings (user_id); + CREATE INDEX IF NOT EXISTS idx_user_settings_key ON user_settings (setting_key); + CREATE INDEX IF NOT EXISTS idx_user_avatars_user_id ON user_avatars (user_id); + CREATE INDEX IF NOT EXISTS idx_user_avatars_active ON user_avatars (user_id, is_active); + + -- 6 CHECK constraints (each in its own DO block so one failure doesn't block the rest) + DO $$ BEGIN + ALTER TABLE users ADD CONSTRAINT check_collection_visibility + CHECK (collection_visibility IN ('private', 'public', 'unlisted')); + EXCEPTION WHEN duplicate_object THEN NULL; END $$; + + DO $$ BEGIN + ALTER TABLE users ADD CONSTRAINT check_preferred_currency + CHECK (preferred_currency IN ('USD', 'EUR', 'GBP', 'CAD', 'JPY')); + EXCEPTION WHEN duplicate_object THEN NULL; END $$; + + DO $$ BEGIN + ALTER TABLE users ADD CONSTRAINT check_cards_per_page + CHECK (cards_per_page IN (25, 50, 100)); + EXCEPTION WHEN duplicate_object THEN NULL; END $$; + + DO $$ BEGIN + ALTER TABLE users ADD CONSTRAINT check_default_view + CHECK (default_view IN ('grid', 'list')); + EXCEPTION WHEN duplicate_object THEN NULL; END $$; + + DO $$ BEGIN + ALTER TABLE users ADD CONSTRAINT check_theme + CHECK (theme IN ('light', 'dark', 'system')); + EXCEPTION WHEN duplicate_object THEN NULL; END $$; + + DO $$ BEGIN + ALTER TABLE users ADD CONSTRAINT check_language + CHECK (language IN ('en', 'es', 'fr', 'de', 'ja')); + EXCEPTION WHEN duplicate_object THEN NULL; END $$; + `); +}; + +/** + * Down-migration intentionally throws. Dropping 16 user-profile columns + * + user_settings + user_avatars on a long-lived env would erase every + * user's profile data, preferences, avatar history, and settings, and + * break runtime reads in pages/api/user/{settings,profile,avatar}.js. + * + * @returns {void} + */ +export const down = () => { + throw new Error( + '[migration:1781000000005_reconcile-user-profile] Down not supported. ' + + 'Dropping these columns + tables would erase every user profile, preference, ' + + 'avatar history, and settings row, and break runtime reads in ' + + 'pages/api/user/{settings,profile,avatar}.js. Write a new dated migration ' + + 'for any future schema correction.' + ); +}; +``` + +## Conventions to follow + +- `.cursor/rules/db-and-schema.mdc` § "Schema source of truth" — `migrations/` is canonical. +- `.cursor/rules/no-go-zones.mdc` — both source scripts are append-only history. +- Style: raw `pgm.sql(...)` template literals. Per D2. +- ESM exports; `"type": "module"`. +- `IF NOT EXISTS` on every ALTER + CREATE INDEX + CREATE TABLE. +- CHECK constraints in `DO $$ ... EXCEPTION WHEN duplicate_object THEN NULL END $$;` blocks, one block per constraint. +- Match the source scripts' column types, defaults, FK rules, and CHECK values verbatim. + +## Acceptance criteria + +- [ ] `migrations/1781000000005_reconcile-user-profile.js` exists with the exact filename above. +- [ ] The file's `up()` adds exactly **16 columns** to `users` (per the dedup count in the source-scripts note above), creates **2 tables** (`user_settings`, `user_avatars`), creates **6 indexes**, and adds **6 CHECK constraints** each in its own `DO $$ EXCEPTION` block. +- [ ] BOTH `profile_image_url` AND `avatar_url` are present (parity with prod; redundancy is documented). +- [ ] The file's `down()` throws with a clear message. +- [ ] The file's docstring cites both source scripts + the convoy file + the `unify-user-avatar-column` follow-up. +- [ ] Per-row UPDATE backfill DML is NOT in the migration. +- [ ] `node --check migrations/1781000000005_reconcile-user-profile.js` passes. +- [ ] `node -e "import('./migrations/1781000000005_reconcile-user-profile.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined`. +- [ ] `npm run lint` matches baseline. +- [ ] `npm run test:run` reports 21/21 passing. + +## Verification + +```bash +node --check migrations/1781000000005_reconcile-user-profile.js +node -e "import('./migrations/1781000000005_reconcile-user-profile.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))" +npm run lint +npm run test:run +``` + +Expected: `node --check` silent, `node -e` prints `function function undefined`, lint baseline, 21/21 tests pass. + +**Do NOT** run `npm run migrate up` against any environment. + +## Commit message + +``` +feat(migrations): reconcile user profile fields into migration history (brief 5/7) + +Captures the deduplicated union of scripts/add-user-profile-columns.js +and scripts/add-user-profile-fields.js into one new node-pg-migrate +migration: + + - 16 columns on users (basic profile + preferences + notifications + display) + - 2 new tables (user_settings, user_avatars) + - 6 CHECK constraints (each in its own DO $$ EXCEPTION block) + - 6 indexes + +Both profile_image_url (from script #1) and avatar_url (from script #2) +are added for parity with prod. The redundancy is flagged in +docs/SCHEMA_MAP.md § "Known schema smells" #1 and queued for cleanup as +the `unify-user-avatar-column` follow-up. + +Per-row UPDATE backfill DML from both scripts is intentionally NOT +folded in — column DEFAULTs handle new rows; fresh envs have no rows +to backfill. + +Per .convoys/reconcile-historical-add-scripts.md § Brief outline → B5. +Idempotent re-apply (IF NOT EXISTS guards + per-CHECK DO blocks). +``` + +## PR shape + +**Title:** `feat(migrations): reconcile user profile fields into migration history (brief 5/7)` + +**Body template:** + +```markdown +Brief 5 of the `reconcile-historical-add-scripts` convoy. See +[`.convoys/reconcile-historical-add-scripts.md`](../.convoys/reconcile-historical-add-scripts.md) +for the full plan and rationale. + +## What this PR does + +Adds `migrations/1781000000005_reconcile-user-profile.js` — captures +the deduplicated union of `scripts/add-user-profile-columns.js` +(earlier, narrower) and `scripts/add-user-profile-fields.js` (later, +broader) into a single new `node-pg-migrate` migration. + +- 16 new columns on `users` (first_name, last_name, username UNIQUE, + profile_image_url, bio, avatar_url, favorite_games JSONB, …, theme, + language) — all `ADD COLUMN IF NOT EXISTS` +- 2 new tables: `user_settings`, `user_avatars` — both + `CREATE TABLE IF NOT EXISTS` +- 6 indexes — `CREATE INDEX IF NOT EXISTS` +- 6 CHECK constraints (each in its own + `DO $$ ... EXCEPTION WHEN duplicate_object THEN NULL END $$;` block) + +## Why both `profile_image_url` and `avatar_url`? + +Script #1 added `profile_image_url`, script #2 added `avatar_url` — +both exist on prod and both are in this migration for fresh-env +parity. The redundancy is flagged in +`docs/SCHEMA_MAP.md` § "Known schema smells" #1 and queued for cleanup +as the `unify-user-avatar-column` follow-up. + +## What this PR does NOT do + +- Does **NOT** edit either source script (no-go-zones). +- Does **NOT** fold in the per-row UPDATE backfill DML from either + source script — column DEFAULTs handle new rows. +- Does **NOT** unify `profile_image_url` / `avatar_url` — that's the + `unify-user-avatar-column` follow-up. +- Does **NOT** edit `scripts/setup-neon-db.js`, `package.json`, README, + `AGENTS.md`, or `docs/SCHEMA_MAP.md` (B7 owns SCHEMA_MAP). +- Does **NOT** run `npm run migrate up` against any environment. + +## Verification checklist + +- [ ] `node --check migrations/1781000000005_reconcile-user-profile.js` exits 0 +- [ ] `node -e "import('./migrations/1781000000005_reconcile-user-profile.js').then(m => console.log(typeof m.up, typeof m.down, typeof m.shorthands))"` prints `function function undefined` +- [ ] `npm run lint` matches baseline +- [ ] `npm run test:run` reports 21/21 passing +- [ ] Did not run `npm run migrate up` against any environment in this PR + +## Cross-references + +- Convoy file: `.convoys/reconcile-historical-add-scripts.md` +- Source scripts (no-go-zones, audit reference only): + `scripts/add-user-profile-columns.js`, `scripts/add-user-profile-fields.js` +``` + +## DO NOT + +- DO NOT edit either source script or any file under `scripts/`. +- DO NOT edit any other file under `migrations/`. +- DO NOT edit `scripts/setup-neon-db.js`, `package.json`, README, `AGENTS.md`, or `docs/SCHEMA_MAP.md`. +- DO NOT run `npm run migrate up` against any environment. +- DO NOT pick one of `profile_image_url` / `avatar_url` to omit — both must be present for parity. +- DO NOT collapse the 6 CHECK constraints into a single `DO $$ EXCEPTION` block — one block per constraint so one duplicate doesn't swallow the others. +- DO NOT include the per-row UPDATE backfill DML. +- DO NOT call `npm run migrate create`. + +## Rationale (≤3 sentences) + +This is the largest single migration in the convoy because both historical scripts are tightly coupled to the `users` table surface and splitting them would create artificial boundaries (e.g., separating "users columns" from "CHECK constraints on users columns" makes no sense). Per-CHECK `DO $$ EXCEPTION` blocks mirror the historical scripts' per-statement try/catch pattern and ensure one duplicate-constraint failure doesn't block the rest. Keeping both avatar-style columns matches prod-as-is and explicitly defers the cleanup to a scoped follow-up convoy. diff --git a/.convoys/reconcile-historical-add-scripts/brief-7-documentation-and-verification.md b/.convoys/reconcile-historical-add-scripts/brief-7-documentation-and-verification.md new file mode 100644 index 0000000..3366403 --- /dev/null +++ b/.convoys/reconcile-historical-add-scripts/brief-7-documentation-and-verification.md @@ -0,0 +1,457 @@ +--- +convoy: reconcile-historical-add-scripts +brief_number: 7 +depends_on: [1, 2, 3, 4, 5] +files: + - docs/SCHEMA_MAP.md + - docs/MIGRATION_VERIFICATION_RUNBOOK.md + - AGENTS.md + - .convoys/ship-readiness.md +--- + +# Brief 7: Documentation + verification runbook + +## Goal (1 sentence) + +Update the four documentation surfaces that describe the post-convoy schema and onboarding state — `docs/SCHEMA_MAP.md` (correct doc bugs + cross-reference new migrations), `docs/MIGRATION_VERIFICATION_RUNBOOK.md` (new — manual operator runbook from D5), `AGENTS.md` Gotcha #6 (cross-reference this convoy as the closing follow-up), `.convoys/ship-readiness.md` (flip this convoy's queued entry to RESOLVED + unblock `retire-graveyard-scripts-after-audit` + add new follow-ups) — so the next operator onboarding a fresh Neon branch can do so by `npm install` → `npm run setup-db` alone and verify the result. + +## Scope (files in scope — do not edit anything else) + +- `docs/SCHEMA_MAP.md` — **modified** +- `docs/MIGRATION_VERIFICATION_RUNBOOK.md` — **new** +- `AGENTS.md` — **modified** (small Gotcha #6 cross-reference update only) +- `.convoys/ship-readiness.md` — **modified** (queued convoys section) + +This brief is **sequenced last** because it cross-references the 5 new migration files that B1-B5 add; it cannot land before B1-B5 are merged. The architect's `slice_dependencies` block in the convoy file declares `depends_on: [1, 2, 3, 4, 5]`. + +## Per-file scope + +### 1. `docs/SCHEMA_MAP.md` — modified + +Targeted edits (keep all other content as-is): + +1. **Preamble update.** The current preamble (lines 1-18) notes that *"the initial backfill captures only the post-`setup-neon-db.js` shape. … A follow-up convoy (`reconcile-historical-add-scripts`) will fold the historical effects into the migration history; until then this file remains the curated reference for the full prod shape."* Replace with a sentence noting the convoy has **landed**, the migration history now captures the full prod shape, and the operator verification runbook lives at `docs/MIGRATION_VERIFICATION_RUNBOOK.md`. Bump the "Last reviewed" date. + +2. **`### users` notes column updates** (lines 43-57): each row references `add-user-profile-columns.js` / `add-user-profile-fields.js` — leave those references in place (they're historical context); ADD a single line at the end of the table noting *"All columns above are now captured by `migrations/1781000000005_reconcile-user-profile.js` (B5 of `reconcile-historical-add-scripts`, 2026-06-14)."* + +3. **`### cards` notes column updates** (lines 78-79): for the `quantity` and `favorited` rows, change the "Unused; consider dropping" annotation to *"Unused; captured by `migrations/1781000000001_reconcile-cards-columns.js` for fresh-env parity. Drop tracked as queued `drop-dead-cards-columns` follow-up."* + +4. **`### user_cards` index/constraint notes** (lines 90-102): add a brief note clarifying the canonical constraint is `UNIQUE(user_id, card_id, is_foil)` (3-col) from initial-schema, and that `scripts/fix-user-cards-constraints.js`'s stricter 2-col variant was either never-applied or reverted (Finding 1 of the reconcile convoy → Outcome A, 2026-06-14). This is the SCHEMA_MAP equivalent of the Drift Finding 1 resolution. + +5. **`### user_favorites` table** (lines 104-111): **FIX THE DOC BUG.** Current shape lists only `(user_id, card_id)`. Replace with the actual polymorphic shape per the runtime in `pages/api/favorites.js`: + + ```markdown + ### user_favorites + + | Column | Type | Notes | + | --- | --- | --- | + | `id` | `SERIAL PK` | | + | `user_id` | `INTEGER FK users(id) ON DELETE CASCADE` | | + | `item_type` | `VARCHAR(50) NOT NULL` | `'card' | 'collection' | 'deck'` — polymorphic | + | `item_id` | `INTEGER NOT NULL` | FK depends on `item_type`; not enforced at DB level | + | `created_at` | `TIMESTAMP` | | + | | | **UNIQUE(user_id, item_type, item_id)** | + + Indexes: `idx_user_favorites_user_id`, `idx_user_favorites_item_type`, + `idx_user_favorites_item_id`, `idx_user_favorites_user_type + (user_id, item_type)` — all in `migrations/1781000000004_reconcile-favorites-system.js`. + ``` + +6. **`### collections` notes**: add a line at the bottom noting *"`visibility`, `tcg`, `tags`, `slug` (+ `idx_collections_slug` unique + `check_slug_format` CHECK), `image` are now captured by `migrations/1781000000002_reconcile-collections-columns.js`."* + +7. **`### collection_permissions` + `### collection_activity` sections**: add a line at the bottom of each noting *"Captured by `migrations/1781000000003_reconcile-collaboration-tables.js`."* + +8. **NEW SECTION: `### user_settings`** (currently a one-liner at line 217). Expand to a proper column table matching the actual schema: + + ```markdown + ### user_settings + + Per-user key/value store for settings that don't warrant a column on `users`. + Captured by `migrations/1781000000005_reconcile-user-profile.js`. + + | Column | Type | Notes | + | --- | --- | --- | + | `id` | `SERIAL PK` | | + | `user_id` | `INTEGER FK users(id) ON DELETE CASCADE` | | + | `setting_key` | `VARCHAR(100) NOT NULL` | | + | `setting_value` | `JSONB NOT NULL` | | + | `created_at`, `updated_at` | `TIMESTAMP` default now | | + | | | **UNIQUE(user_id, setting_key)** | + + Indexes: `idx_user_settings_user_id`, `idx_user_settings_key`. + ``` + +9. **`### user_avatars` expansion** (currently one paragraph at lines 222-223). Replace with: + + ```markdown + ### user_avatars + + Tracks uploaded avatar history. Captured by + `migrations/1781000000005_reconcile-user-profile.js`. Older avatars + are typically deleted from blob storage; verify the cleanup job runs. + + | Column | Type | Notes | + | --- | --- | --- | + | `id` | `SERIAL PK` | | + | `user_id` | `INTEGER FK users(id) ON DELETE CASCADE` | | + | `filename` | `VARCHAR(255) NOT NULL` | | + | `original_name` | `VARCHAR(255)` | | + | `mime_type` | `VARCHAR(100)` | | + | `file_size` | `INTEGER` | | + | `file_path` | `TEXT NOT NULL` | | + | `is_active` | `BOOLEAN DEFAULT true` | | + | `created_at`, `updated_at` | `TIMESTAMP` default now | | + + Indexes: `idx_user_avatars_user_id`, `idx_user_avatars_active (user_id, is_active)`. + ``` + +10. **`## Known schema smells` section updates** (lines 226-232): + + - **Smell #2** (`is_public` vs `visibility`): add a sentence noting both columns are now captured by separate migrations (initial-schema for `is_public`, B2 for `visibility`); resolution lives in a future `unify-collection-visibility` follow-up. + - **Smell #3** (`cards.quantity` + `cards.favorited`): add the cross-reference to the queued `drop-dead-cards-columns` follow-up. + - **NEW: Smell #7 — `users.profile_image_url` vs `users.avatar_url`** (the parity smell that B5 perpetuates intentionally). Both columns are present for prod parity; cleanup is the queued `unify-user-avatar-column` follow-up. + +11. **`## Regeneration` section update** (lines 234-245): the manual-regeneration instructions can be removed entirely since the migration history is now authoritative. Replace with: + + ```markdown + ## Regeneration + + The migration history under `migrations/` is the authoritative source + of truth. To verify this file matches a live env (prod, preview, or a + fresh Neon branch), use the operator runbook at + [`docs/MIGRATION_VERIFICATION_RUNBOOK.md`](MIGRATION_VERIFICATION_RUNBOOK.md). + + When you add a new migration, update the relevant table section here + in the same PR. New tables get a new `###` section with the same + column-table shape. + ``` + +### 2. `docs/MIGRATION_VERIFICATION_RUNBOOK.md` — new + +Lift verbatim from `.convoys/reconcile-historical-add-scripts.md` § Verification plan (D5), with a small intro framing it as the canonical operator runbook (not just a convoy artifact). Suggested skeleton: + +```markdown +# Migration verification runbook + +How to verify that the migration history under `migrations/` produces +the same schema as a long-lived environment (prod, preview, or +similar). Use this runbook: + +- **After this repo's `reconcile-historical-add-scripts` convoy** (the + initial reconciliation), to confirm a fresh Neon branch reaches + parity with prod via `npm install` → `npm run setup-db` alone. +- **After any new migration lands on `main`**, to spot-check that + applying the migration to prod (via the operator's + `POSTGRES_URL= npm run migrate up`) produced the intended + effect. +- **When suspecting drift** between an env's actual schema and the + migration history (rare; the migration history is authoritative). + +This runbook is the manual precursor to the automated check planned +in the queued `wire-migrate-into-ci` follow-up (see +`.convoys/ship-readiness.md` § "Queued convoys"). + +## Prerequisites + +- `pg_dump` (PostgreSQL 16+) installed locally. +- `POSTGRES_URL` for the env you're verifying. +- `ADMIN_INITIAL_PASSWORD` (for the fresh-branch onboarding step) — + see `AGENTS.md` § 5 "Running locally". +- A Neon account with permission to create a branch (or any other + way to spin up a fresh Postgres DB on the same major version as + prod). + +## Procedure + +### Step 1 — Snapshot the reference env's structural shape + +Run against the env you consider canonical (usually prod): + +\`\`\`bash +POSTGRES_URL= pg_dump --schema-only --no-owner --no-acl \ + --schema=public > /tmp/reference-schema.sql +\`\`\` + +### Step 2 — Create a fresh DB and onboard via `setup-db` + +Create a clean Neon branch from an **empty** parent (or any other +fresh Postgres DB on the same major version): + +\`\`\`bash +POSTGRES_URL= \ +ADMIN_INITIAL_PASSWORD=$(openssl rand -base64 24) \ + npm run setup-db +\`\`\` + +`setup-db` runs `npm run migrate up` (applying every migration in +`migrations/` in timestamp order) and seeds the admin user. + +### Step 3 — Snapshot the fresh DB's structural shape + +\`\`\`bash +POSTGRES_URL= pg_dump --schema-only --no-owner --no-acl \ + --schema=public > /tmp/fresh-schema.sql +\`\`\` + +### Step 4 — Diff + +\`\`\`bash +diff <(sort /tmp/reference-schema.sql) <(sort /tmp/fresh-schema.sql) +\`\`\` + +**Expected non-material differences** (acceptable; do not chase): + +- Constraint or index NAME differences. Prod constraints created via + the historical `scripts/add-*` / `scripts/fix-*` jobs may have + autogenerated tuple-UNIQUE names that differ from the migrations' + explicit names. +- Column-ORDER differences. Prod has columns in historical-script-ALTER + order; fresh envs have them in migration-order. + +**Material differences** (bug — fix before declaring verified): + +- A column type, default, or NULL/NOT NULL state that differs. +- A missing or extra table. +- A missing or extra CHECK constraint that changes accepted values. +- A missing or extra index that changes query plan shape. + +## Supplementary spot-checks + +For the highest-risk surfaces (frequently-edited tables), run these +information-schema queries against both envs and confirm the +column-count / constraint-count / index-count totals match exactly: + +\`\`\`sql +-- Every column on every table +SELECT table_name, column_name, data_type, is_nullable, column_default +FROM information_schema.columns +WHERE table_schema = 'public' +ORDER BY table_name, ordinal_position; + +-- Every constraint +SELECT table_name, constraint_name, constraint_type +FROM information_schema.table_constraints +WHERE table_schema = 'public' +ORDER BY table_name, constraint_name; + +-- Every index +SELECT tablename, indexname, indexdef +FROM pg_indexes +WHERE schemaname = 'public' +ORDER BY tablename, indexname; +\`\`\` + +A mismatch in column count, constraint count, or index count is a +material difference and indicates a bug. + +## What to do if you find a material difference + +1. **Identify which env is canonical.** Usually prod. If the diff + surfaces a missing column on prod that's in the migration history, + the migration was never applied to prod — run + `POSTGRES_URL= npm run migrate up` to catch up. +2. **If the diff surfaces a column on prod that's NOT in the + migration history**, you've found a drift bug. Write a new + reconciliation migration (under `migrations/`) that captures the + prod column, following the pattern in `.convoys/reconcile-historical-add-scripts/`. +3. **Do not edit existing migrations** — they're pinned by + `pgmigrations` rows. Always write a new dated migration to correct + schema. + +## Cross-references + +- `AGENTS.md` § 4 Gotcha #6 — migration tool adoption history. +- `.convoys/migration-tool.md` — the convoy that adopted `node-pg-migrate`. +- `.convoys/reconcile-historical-add-scripts.md` — the convoy that folded + the 13 historical scripts into the migration history. +- `.cursor/rules/db-and-schema.mdc` — schema-change conventions. +``` + +### 3. `AGENTS.md` — modified (small Gotcha #6 cross-reference) + +Gotcha #6 is already "RESOLVED" per `migration-tool` (2026-05-26). This brief adds **one sentence** to the end of Gotcha #6 cross-referencing this convoy as the closing follow-up. Verbatim addition (insert immediately before the `- **#7**` line at AGENTS.md line 120): + +> Post-`reconcile-historical-add-scripts` (2026-06-14), the +> migration history additionally captures the full effect of the 13 +> historical `scripts/add-*.js` / `scripts/fix-*.js` / +> `scripts/seed-*.js` jobs (where applicable — pure-DML seed scripts +> stay out per D4 of that convoy). A brand-new Neon branch can now be +> onboarded by `npm install` → `npm run setup-db` alone. Operator +> verification runbook at +> [`docs/MIGRATION_VERIFICATION_RUNBOOK.md`](docs/MIGRATION_VERIFICATION_RUNBOOK.md). +> The 13 historical scripts remain no-go-zones until the queued +> `retire-graveyard-scripts-after-audit` (P3) cleanup convoy lands; +> that convoy is now unblocked. + +Do NOT flip Gotcha #6's RESOLVED marker — it's already RESOLVED by the right convoy (`migration-tool`). Just append the cross-reference. + +### 4. `.convoys/ship-readiness.md` — modified (queued convoys section) + +Find the existing line under § "Queued convoys" that reads: + +> - **`reconcile-historical-add-scripts`** (priority: P1 quality — needed for fresh-env onboarding). Surfaced 2026-05-26 by `migration-tool` (PR #32). Fold the effects of the 27 historical `scripts/add-*.js` / `fix-*.js` / `seed-*.js` jobs … Documented in `.convoys/migration-tool.md` § R1. + +Replace with: + +> - **`reconcile-historical-add-scripts`** — **RESOLVED 2026-06-14** by [convoy](../.convoys/reconcile-historical-add-scripts.md) (PRs #XX-#XX). Captured 7 of the 13 historical scripts' effects into 5 new migrations under `migrations/` (B1-B5); 2 scripts were already captured by `initial-schema` + `1780378340194_system-collection-description`; 3 are DML-only and stay as dev fixtures; 1 (`fix-user-cards-constraints.js`) is a no-op on prod per Finding 1 (Outcome A — the canonical 3-col `UNIQUE(user_id, card_id, is_foil)` from `initial-schema` is what prod has, and `fix-user-cards-constraints.js`'s stricter 2-col variant is not present). A brand-new Neon branch now onboards via `npm install` → `npm run setup-db` alone. Operator verification at [`docs/MIGRATION_VERIFICATION_RUNBOOK.md`](../docs/MIGRATION_VERIFICATION_RUNBOOK.md). Entry kept (not deleted) for audit trail. + +Find the existing line: + +> - **`retire-graveyard-scripts-after-audit`** (priority: P3 polish; **blocked on `reconcile-historical-add-scripts`**). … + +Replace the `**blocked on `reconcile-historical-add-scripts`**` marker with `**UNBLOCKED 2026-06-14**` and leave the rest of the description intact. + +Add two new queued entries (driven by Findings 2 + 3 of the reconcile convoy): + +> - **`unify-user-avatar-column`** (priority: P3 hygiene). Surfaced 2026-06-14 by `reconcile-historical-add-scripts` Finding 2. `users.profile_image_url` (added by `scripts/add-user-profile-columns.js`) and `users.avatar_url` (added by `scripts/add-user-profile-fields.js`) coexist on prod and in the migration history (both columns are needed for parity per B5). Pick one canonical column, migrate the other's data to it, drop the loser, and sweep runtime readers in `pages/api/user/avatar*.js` + UI surfaces. Requires a query-trace audit first. +> - **`drop-dead-cards-columns`** (priority: P3 hygiene). Surfaced 2026-06-14 by `reconcile-historical-add-scripts` Finding 3. `cards.quantity INTEGER` and `cards.favorited BOOLEAN` are documented unused (per `docs/SCHEMA_MAP.md` § "Known schema smells" #3); reconciled into the migration history by B1 for parity, but the actual semantics live on `user_cards` / `user_favorites`. After a query-trace audit confirms zero runtime readers, ship a migration that DROPs both columns with a real `down()` that recreates them. + +(Do NOT add the withdrawn `add-system-collection-on-register` or `unify-user-cards-foil-tracking` entries — both were withdrawn during the reconcile convoy. See `.convoys/reconcile-historical-add-scripts.md` § Follow-ups.) + +## Conventions to follow + +- `.cursor/rules/no-go-zones.mdc` — no edits to `scripts/add-*` / `fix-*` / `seed-*`. +- `.cursor/rules/db-and-schema.mdc` — schema-change conventions; SCHEMA_MAP is updated alongside any migration. +- Markdown style: match the existing tone of `docs/SCHEMA_MAP.md` (compact tables, "Notes" column explains intent not type semantics) and `AGENTS.md` (numbered Gotchas, cross-reference convoy files). +- For `.convoys/ship-readiness.md`: match the existing entry style under § "Queued convoys" (one bullet per convoy, leading `- **`name`**`, RESOLVED entries get the "Entry kept (not deleted) for audit trail" closer when applicable). + +## Acceptance criteria + +- [ ] `docs/SCHEMA_MAP.md` preamble bumped (post-convoy state); "Last reviewed" date is 2026-06-14. +- [ ] `docs/SCHEMA_MAP.md` § `user_favorites` shape is the polymorphic `(item_type, item_id)` — NOT the previous `(card_id)` shape. +- [ ] Every reconciled table/column has a "captured by `migrations/`" cross-reference. +- [ ] `### user_settings` and `### user_avatars` sections are expanded from one-liners to full column tables matching B5's migration. +- [ ] New schema smell #7 (`profile_image_url` vs `avatar_url`) is added to § "Known schema smells". +- [ ] `docs/MIGRATION_VERIFICATION_RUNBOOK.md` exists with the 4-step procedure from D5 + the spot-check queries + the "material vs non-material differences" guidance. +- [ ] `AGENTS.md` Gotcha #6 gains a single appended paragraph cross-referencing this convoy + the new runbook + the unblocked `retire-graveyard-scripts-after-audit` follow-up. Gotcha #6's existing "RESOLVED by `migration-tool`" marker is NOT changed. +- [ ] `.convoys/ship-readiness.md` § "Queued convoys" → `reconcile-historical-add-scripts` entry is flipped to RESOLVED 2026-06-14 with a one-paragraph summary; `retire-graveyard-scripts-after-audit` is marked UNBLOCKED 2026-06-14; two new entries (`unify-user-avatar-column`, `drop-dead-cards-columns`) are added. +- [ ] No edits to any file under `scripts/`, `migrations/`, `pages/`, `lib/`, `components/`, `test/`, or `package.json`. +- [ ] `npm run lint` matches baseline (these are markdown-only edits; lint should be unaffected). +- [ ] `npm run test:run` reports 21/21 passing (these are markdown-only edits; no test surface). + +## Verification + +```bash +npm run lint +npm run test:run +``` + +Both must match baseline / be green. No additional verification for this brief — the migrations themselves are exercised by Brief 7's verification runbook AFTER an operator runs it manually post-merge. + +For the SCHEMA_MAP edits, eyeball the diff against the prior state and confirm every cross-reference to a migration file is accurate (the file actually exists, and the timestamp matches the assigned reservation). + +## Commit message + +``` +docs(reconcile): update SCHEMA_MAP, verification runbook, AGENTS, ship-readiness (brief 7/7) + +Closes the reconcile-historical-add-scripts convoy: + + - docs/SCHEMA_MAP.md: cross-reference 5 new migrations from B1-B5; + fix user_favorites doc bug (polymorphic shape, not (card_id)); + expand user_settings + user_avatars one-liners to full column tables; + add schema smell #7 (profile_image_url vs avatar_url redundancy). + - docs/MIGRATION_VERIFICATION_RUNBOOK.md (new): canonical operator + runbook for verifying fresh-env vs prod schema parity (lifted from + D5 of the convoy). + - AGENTS.md Gotcha #6: append cross-reference to this convoy + the + new runbook + the now-unblocked retire-graveyard-scripts-after-audit + follow-up. Existing "RESOLVED by migration-tool" marker unchanged. + - .convoys/ship-readiness.md: flip reconcile-historical-add-scripts + entry to RESOLVED 2026-06-14; mark retire-graveyard-scripts-after-audit + UNBLOCKED; add new queued unify-user-avatar-column (Finding 2) + + drop-dead-cards-columns (Finding 3). + +Per .convoys/reconcile-historical-add-scripts.md § Brief outline → B7. +Depends on B1-B5 having merged (this brief cross-references their files). +``` + +## PR shape + +**Title:** `docs(reconcile): update SCHEMA_MAP, verification runbook, AGENTS, ship-readiness (brief 7/7)` + +**Body template:** + +```markdown +Brief 7 (final) of the `reconcile-historical-add-scripts` convoy. +See [`.convoys/reconcile-historical-add-scripts.md`](../.convoys/reconcile-historical-add-scripts.md) +for the full plan and rationale. + +This is the closing brief — depends on B1-B5 having merged (PR +references below). + +## What this PR does + +- **`docs/SCHEMA_MAP.md`** — cross-reference all 5 new migrations from + B1-B5; fix the `user_favorites` doc bug (the table uses the + polymorphic `(item_type, item_id)` shape, not `(card_id)`); expand + `user_settings` + `user_avatars` from one-liners to full column + tables matching the B5 migration; add a new "Known schema smell" #7 + for the `profile_image_url` / `avatar_url` redundancy. +- **`docs/MIGRATION_VERIFICATION_RUNBOOK.md`** (new) — canonical + operator runbook for verifying fresh-env vs prod schema parity. + Pulls verbatim from D5 of the convoy file. Useful both for the + one-time reconciliation verification and for ongoing post-migration + spot-checks. +- **`AGENTS.md`** — append one paragraph to Gotcha #6 cross-referencing + this convoy as the closing follow-up. The "RESOLVED by `migration-tool`" + marker is unchanged (that's still the right resolution attribution + for the gotcha itself). +- **`.convoys/ship-readiness.md`** § "Queued convoys" — flip the + `reconcile-historical-add-scripts` entry to RESOLVED 2026-06-14; + flip `retire-graveyard-scripts-after-audit` from "blocked on + reconcile-historical-add-scripts" to UNBLOCKED 2026-06-14; add two + new queued entries (`unify-user-avatar-column` from Finding 2, + `drop-dead-cards-columns` from Finding 3). + +## Prerequisite PRs (all must be merged first) + +- B1 #XX — reconcile cards columns +- B2 #XX — reconcile collections columns +- B3 #XX — reconcile collaboration tables +- B4 #XX — reconcile favorites system +- B5 #XX — reconcile user profile + +## What this PR does NOT do + +- Does **NOT** edit any file under `migrations/`, `scripts/`, `pages/`, + `lib/`, `components/`, `test/`, or `package.json`. +- Does **NOT** run the verification runbook itself — that's the + operator's post-merge job; see the runbook's "Procedure" section. + +## Verification checklist + +- [ ] `npm run lint` matches baseline +- [ ] `npm run test:run` reports 21/21 passing +- [ ] Every "captured by `migrations/.js`" cross-reference in + SCHEMA_MAP points at a file that actually exists in `migrations/` + (eyeballed against `ls migrations/`) +- [ ] `AGENTS.md` Gotcha #6 still has its original RESOLVED marker + (we only APPEND to it, not rewrite it) +- [ ] `.convoys/ship-readiness.md` § "Queued convoys" has the flipped + `reconcile-historical-add-scripts` entry, the UNBLOCKED + `retire-graveyard-scripts-after-audit` marker, and the two new + follow-up entries +- [ ] Operator post-merge: run `docs/MIGRATION_VERIFICATION_RUNBOOK.md` + against prod + a fresh Neon branch and confirm the diff is + non-material + +## Cross-references + +- Convoy file: `.convoys/reconcile-historical-add-scripts.md` +- Per-brief files: `.convoys/reconcile-historical-add-scripts/brief-{1,2,3,4,5}-*.md` +``` + +## DO NOT + +- DO NOT edit any file under `scripts/`, `migrations/`, `pages/`, `lib/`, `components/`, `test/`. +- DO NOT edit `package.json`, README, `next.config.js`, or any rule under `.cursor/rules/`. +- DO NOT change `AGENTS.md` Gotcha #6's "RESOLVED by `migration-tool`" marker — append the new paragraph; don't rewrite the existing resolution attribution. +- DO NOT add a `B6` reference anywhere — that brief was removed (Finding 1 → Outcome A). +- DO NOT add `add-system-collection-on-register` or `unify-user-cards-foil-tracking` to the ship-readiness queued list — both were withdrawn during this convoy. +- DO NOT remove the existing `wire-migrate-into-ci` entry from ship-readiness — it's still queued and unrelated to this convoy. +- DO NOT run the verification runbook itself (that's the operator's post-merge step). + +## Rationale (≤3 sentences) + +Sequencing this brief last lets every documentation cross-reference point at a real file under `migrations/` rather than a placeholder. Bundling all four doc surfaces into one PR keeps the convoy's "as-shipped" record consistent (the migrations, the operator runbook, the AGENTS gotcha, and the ship-readiness ledger all flip together). The `docs/MIGRATION_VERIFICATION_RUNBOOK.md` extraction promotes a one-time convoy artifact into an evergreen operator tool that future migrations can reuse. diff --git a/.convoys/scan-visual-catalog-search.md b/.convoys/scan-visual-catalog-search.md new file mode 100644 index 0000000..5b21e34 --- /dev/null +++ b/.convoys/scan-visual-catalog-search.md @@ -0,0 +1,233 @@ +--- +name: scan-visual-catalog-search +classification: feature +success_metric: | + After catalog embeddings exist and the new kNN path is on, ≥50% of + legitimate card scans (scan_attempts excluding not_a_card) auto-match + a printing with no picker — the original add-real-ocr-layer 70% + target, measured honestly. Gemini L2 share of attempts falls vs the + Phase 1 post-ship baseline. +skip: + - ia + - ui-design + - visual + - a11y + - design + - flag +status: shipped +created: 2026-08-14 +depends_on: + - improve-scan-card-detection +umbrella: scanner-identify-upgrade +model_policy: + default_session: auto + roles: + role-conductor: composer-2.5-fast + role-architect: composer-2.5 + role-ia-architect: composer-2.5-fast + role-ux-reviewer: composer-2.5-fast + role-ui-designer: composer-2.5-fast + role-implementer: composer-2.5-fast + role-reviewer: cursor-grok-4.5-high + role-security-auditor: gpt-5.6-terra-medium + role-design-system-auditor: cursor-grok-4.5-high + role-a11y-auditor: cursor-grok-4.5-high + role-doc-writer: auto + escalate_to: claude-sonnet-5-thinking-medium + escalate_to_premium: claude-4.6-opus-high-thinking + never_premium: + - role-reviewer + - role-security-auditor + - role-design-system-auditor + - role-a11y-auditor + - role-ui-designer + - role-doc-writer +--- + +# Convoy: scan-visual-catalog-search + +Phase 3 of `scanner-identify-upgrade`. Identify cards by **what they +look like**, not by reading the name. This is how Manabox / Delver-style +scanners get printing-accurate matches on foil and alt-art. + +Blocked on `improve-scan-card-detection`: embeddings of unwarped +phone crops will not match catalog `image_url`s reliably. + +## Why + +Name-only `pg_trgm` cannot distinguish printings. Tesseract fails on +foil and stylized type even when it emits 8–20 characters (76.7% L1 +escalate with non-empty text). Gemini can read a name but still +returns a picker when set/number are missing, and costs 5/min. + +The catalog already stores `cards.image_url` for every imported +printing. Precompute an embedding per row; at scan time embed the +warped crop and take top-k cosine. OCR / Gemini become hints and +unknown-card fallback. + +## Scope + +### In scope + +- **Migration:** `pgvector` (confirm Neon availability) + + `cards.embedding` (or a side table `card_embeddings`) + ANN index. + Update `docs/SCHEMA_MAP.md`. +- **Offline embed job:** new dated script or `npm run` task that + reads `image_url`, writes vectors. Idempotent. Rate-limit the + embedding provider. Do **not** edit historical `scripts/add-*.js`. +- **Architect picks the embedder** (one): + 1. Gateway embedding model (same `AI_GATEWAY_API_KEY`, server-only). + 2. In-browser MobileCLIP-S2 / SigLIP ONNX for the *query* crop, with + catalog vectors baked or fetched — only if weight license + size + are acceptable. +- **Identify path:** new Layer-0 (or replace L1) — kNN then + auto-match / disambiguate / escalate to existing L2. Log + `scan_attempts.layer = 0` (or Architect-ratified value). +- **Similarity thresholds** analogous to 0.85 / 0.60, tuned on a + held-out set of scan captures if any exist in Blob. +- Auth + rate-limit on any new route. No client-side API keys + (`forbidden-client-side-llm-keys` must stay green). + +### Out of scope + +- Replacing Gemini entirely on day one — keep L2 for catalog misses + and low similarity. +- Training a custom card CNN. +- Python GPU service. +- Changing scanner chrome / cart. +- Auto-approving the 27 pending `card_submissions`. + +## Roles invoked + +1. `role-architect` — embedder, schema, layer numbering, thresholds, + brief split (migration / backfill / route). Security-sensitive: + escalate to Sonnet if schema + new route land together. +2. `role-implementer`. +3. Audit: `role-reviewer` + `role-security-auditor` (required). + +## Todos + +- [x] Architect: confirm `pgvector` on prod Neon tier +- [x] Brief 1 — migration + SCHEMA_MAP +- [x] Brief 2 — catalog backfill job (idempotent) +- [x] Brief 3 — identify kNN route + client escalate order +- [x] Threshold bake-off on real crops (initial: 0.82 match / 0.58 disambig — tune post-backfill) +- [ ] Re-measure auto-match % excluding `not_a_card` (blocked: L0 traffic + backfill) +- [ ] Operator: finish `npm run backfill-embeddings` — **14.9%** (9,865 / 66,211) as of 2026-08-15 + +## Post-ship (2026-08-15) + +**Merged:** PR #160 (Layer-0 visual catalog search), PR #162 (CI gates). + +**Pipeline roles (kickoff complete):** + +| Role | Outcome | +| --- | --- | +| Conductor | Convoy ratified; skip ia/ui-design/visual/a11y/design/flag | +| UX reviewer | No new screens; L0→L1→L2 order; silent escalate when index empty | +| Architect | D1–D4 decisions + 3 briefs (`.convoys/scan-visual-catalog-search/brief-*`) | +| Implementer | Briefs 1–3 shipped in #160 | +| Reviewer + security-auditor | #160 audit fan-out; #162 CI fix | +| Doc-writer | SCHEMA_MAP layer 0 documented | + +**CI fixes (#162):** `lib/card-embed.js` allowlist + pgvector migration graceful +skip on non-superuser homelab CI. + +**Neon:** pgvector v0.8.0; `cards.embedding vector(1024)` + HNSW index applied. +Backfill **in progress** (9,865 / 66,211 = 14.9%). **0** `scan_attempts` with +`layer = 0` yet — L0 path live but index too sparse / no post-#160 scans logged. + +**scan_attempts snapshot** (all-time n=250): + +| Signal | Value | Baseline | +| --- | --- | --- | +| L1 escalate | 78.4% (87/111) | 76.7% | +| L1 matched | 5.4% (6/111) | 5.8% | +| L2 not_a_card | 43.9% (61/139) | 44.6% | +| End-to-end auto-match | 13.2% excl. not_a_card | 13.5% | +| L0 attempts | 0 | — | + +**Aug 15 session (n=26):** See umbrella `.convoys/scanner-identify-upgrade.md` +§ Post-ship telemetry. + +Re-measure after backfill completes and preview scanning generates L0 rows. + +## Likely file ownership + +| Area | Files | +| --- | --- | +| Schema | `migrations/*_card-embeddings.js`, `docs/SCHEMA_MAP.md` | +| Backfill | new `scripts/` job (dated) or `lib/card-embed-backfill.js` | +| Query | new `lib/card-visual-match.js`, `pages/api/scan/identify-by-image.js` **or** fold into existing identify | +| Client | `lib/scanner-card-identify.js` (`tryLayer1TextIdentify` sibling) | + +Do not rewrite `lib/scanner-card-detection.js` here. + +## Multitask dispatch + +Brief 1 first. Brief 2 after 1. Brief 3 after 1 (can overlap 2 if the +route degrades to escalate-when-empty-index). + +``` +/multitask role-reviewer + role-security-auditor +``` + +Group id: `audit-scan-visual-catalog-search-`. + +## CI impact + +| Workflow / job | Behavior | +| --- | --- | +| `schema-map-fresh` | **Fires** — migration + SCHEMA_MAP | +| `ci.yml` migrate | Must apply `pgvector` on CT 102 CI Postgres — Architect must verify the extension is available there or gate the migration | +| `forbidden-client-side-llm-keys` | Blocking | + +## Operator action + +- Confirm Neon `pgvector` (or Neon’s equivalent) on the prod project. +- Budget: one embedding per catalog image, plus one per live scan if + the query embed is server-side. Architect publishes a cost note + before Brief 2 runs against prod images. +- No new browser secrets. + +## Conductor notes + +This is the accuracy leap. Do not start it to "try CLIP" before Phase 2 +crops are rectified — that wastes the backfill. If Architect finds +`pgvector` unavailable on CI Postgres, stop and write a fallback +(external index vs skip-CI-extension plan) rather than shipping an +untestable migration. + +## UX + +No new screens. Scan flow stays L0 → L1 → L2 with the same +disambiguation picker and error toasts. When the catalog index is empty +(backfill not run), L0 escalates silently with no embed cost. + +Gallery uploads now try visual match before OCR. + +## Architecture + +### Decision D1 — Gateway multimodal embedder (`cohere/embed-v4.0`) + +Server-only via `AI_GATEWAY_API_KEY`. 1024-dim vectors in +`cards.embedding`. Env: `SCAN_EMBED_MODEL`, `SCAN_EMBED_DIMENSION`. + +### Decision D2 — Layer numbering + +| Layer | Path | +| --- | --- | +| 0 | `POST /api/scan/identify-by-image` | +| 1 | Tesseract + `identify-by-text` | +| 2 | Gemini + `scan/identify` | + +### Decision D3 — Thresholds + +Match ≥ **0.82** (0.06 gap). Disambiguation ≥ **0.58**. + +### Decision D4 — Rate limit + +`checkScanRateLimit` on identify-by-image. L0 429 falls through to L1 +(not a hard stop). + +Audit group id: `audit-scan-visual-catalog-search-`. diff --git a/.convoys/scan-visual-catalog-search/brief-1-migration-schema.md b/.convoys/scan-visual-catalog-search/brief-1-migration-schema.md new file mode 100644 index 0000000..ce7cae0 --- /dev/null +++ b/.convoys/scan-visual-catalog-search/brief-1-migration-schema.md @@ -0,0 +1,23 @@ +--- +convoy: scan-visual-catalog-search +brief_number: 1 +depends_on: [] +recommended_model: composer-2.5 +model_tier: standard +files: + - migrations/1782000000001_add-card-embeddings.js + - docs/SCHEMA_MAP.md +--- + +# Brief 1: pgvector migration + SCHEMA_MAP + +## Goal + +Add `vector(1024)` embedding column + HNSW index on `cards`. + +## Acceptance criteria + +- [ ] `CREATE EXTENSION IF NOT EXISTS vector` +- [ ] `cards.embedding`, `cards.embedded_at` +- [ ] `idx_cards_embedding_hnsw` partial index +- [ ] SCHEMA_MAP documents layer 0 + pgvector diff --git a/.convoys/scan-visual-catalog-search/brief-2-backfill-embeddings.md b/.convoys/scan-visual-catalog-search/brief-2-backfill-embeddings.md new file mode 100644 index 0000000..295c1d5 --- /dev/null +++ b/.convoys/scan-visual-catalog-search/brief-2-backfill-embeddings.md @@ -0,0 +1,24 @@ +--- +convoy: scan-visual-catalog-search +brief_number: 2 +depends_on: [1] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/card-embed.js + - scripts/backfill-card-embeddings.js + - package.json + - test/lib/card-embed.test.js +--- + +# Brief 2: Catalog embedding backfill + +## Goal + +Server-side embed job using `cohere/embed-v4.0` via AI Gateway; idempotent backfill from `cards.image_url`. + +## Acceptance criteria + +- [ ] `lib/card-embed.js` exports `embedCardImage`, `formatEmbeddingForPg` +- [ ] `npm run backfill-embeddings` script (skips rows with embedding unless FORCE=1) +- [ ] Rate-limited (`SLEEP_MS`) and DRY_RUN support diff --git a/.convoys/scan-visual-catalog-search/brief-3-identify-knn-route.md b/.convoys/scan-visual-catalog-search/brief-3-identify-knn-route.md new file mode 100644 index 0000000..dd270c1 --- /dev/null +++ b/.convoys/scan-visual-catalog-search/brief-3-identify-knn-route.md @@ -0,0 +1,27 @@ +--- +convoy: scan-visual-catalog-search +brief_number: 3 +depends_on: [1] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/card-visual-match.js + - pages/api/scan/identify-by-image.js + - lib/scanner-card-identify.js + - lib/use-scanner-identification.js + - test/lib/card-visual-match.test.js +--- + +# Brief 3: Layer-0 identify route + client orchestration + +## Goal + +kNN visual match before L1/L2; log `scan_attempts.layer = 0`. + +## Acceptance criteria + +- [ ] `POST /api/scan/identify-by-image` — auth + `checkScanRateLimit` + embed + kNN +- [ ] `identifyTrackedCardCapture` order: L0 → L1 → L2 +- [ ] Gallery path runs L0 → L1 → L2 +- [ ] Skip auto Gemini refine when disambiguation from L0 (same as L1) +- [ ] Thresholds: match 0.82, disambiguation 0.58 diff --git a/.convoys/scanner-desktop-layout.md b/.convoys/scanner-desktop-layout.md new file mode 100644 index 0000000..bcac7d0 --- /dev/null +++ b/.convoys/scanner-desktop-layout.md @@ -0,0 +1,902 @@ +--- +name: scanner-desktop-layout +classification: feature +success_metric: | + On md+ viewports, /scanner keeps the desktop app chrome (sidebar + + top bar), shows a framed camera workstation with a real webcam + device picker, Upload Image, Batch Scan, Auto-detect, Scanner + Tips, a live match inspector, and a bottom strip with Recent + Scans / Scan Queue / Duplicates — without regressing the mobile + immersive checkout. +skip: [] +status: shipped +created: 2026-08-15 +depends_on: + - scanner-mobile-checkout + - scanner-rebuild +model_policy: + default_session: auto + roles: + role-conductor: composer-2.5-fast + role-architect: composer-2.5 + role-ia-architect: composer-2.5-fast + role-ux-reviewer: composer-2.5-fast + role-ui-designer: composer-2.5-fast + role-implementer: composer-2.5-fast + role-reviewer: cursor-grok-4.5-high + role-security-auditor: gpt-5.6-terra-medium + role-design-system-auditor: cursor-grok-4.5-high + role-a11y-auditor: cursor-grok-4.5-high + role-doc-writer: auto + escalate_to: claude-sonnet-5-thinking-medium + escalate_to_premium: claude-4.6-opus-high-thinking + never_premium: + - role-reviewer + - role-security-auditor + - role-design-system-auditor + - role-a11y-auditor + - role-ui-designer + - role-doc-writer +design_direction: + source: role-ui-designer + skill: ui-ux-pro-max + skill_version: "2.5.0" + version: 1 + locked_at: 2026-08-15 + product_type: desktop trading card scanner workstation + pattern: Feature-Rich Showcase (workstation variant) + style: Liquid Glass / glassmorphism + stack: nextjs + layout_reference: image-1eeebe23-9983-4a96-a3e7-4e3cdfdceb5b.png +--- + +# Convoy: scanner-desktop-layout + +Give `/scanner` a dedicated desktop workstation layout from the +attached dark/light mock (camera + live result + history strip), +while leaving the shipped mobile immersive checkout alone. + +Worktree: `tcg-vault-worktrees/scanner-desktop-layout` on +`convoy/scanner-desktop-layout` (branched from `origin/main` @ +`0d52858`). Layout reference: +`image-1eeebe23-9983-4a96-a3e7-4e3cdfdceb5b.png`. + +Do **not** land this on `dashboard-home-realignment` — that convoy +owns sidebar IA + top-bar chrome. Do **not** land this on +`scanner-identify-upgrade` — that epic owns detect/identify accuracy. + +## Why + +`scanner-mobile-checkout` shipped the phone job: full-bleed camera, +local cart, checkout sheet. Desktop (`md+`) got the leftover +composition — the same camera chrome plus a 360px cart side panel +(`ScannerReview` `variant="side-panel"`). That is not a desk +workstation. + +On a laptop the user wants to see the webcam, inspect the current +match (set, rarity, number, condition, foil, confidence), decide +what to do with it, and keep a history/queue in view — without +losing the app sidebar or search bar. The mock is that layout. +Today they get a phone overlay stretched into a column. + +## Scope + +### In scope + +- **Desktop-only composition (`md+`).** Keep Layout sidebar + + TopSearchBar. Stop treating desktop as an immersive camera page + with a bolted-on cart. Mobile (`max-md`) stays + `chrome="immersive"` + checkout sheet. +- **Framed camera viewport.** Large live feed with ember corner + brackets (already in `ScannerCamera`), Auto-detect status, and + desk controls under the frame. Camera is a panel in the page, + not a full-bleed overlay. +- **Real webcam device picker.** `enumerateDevices` + `deviceId` + in `useCameraScanner` (not facing-mode swap relabeled). Persist + the last-used device for the tab if cheap. Empty-list / denied- + permission fallback. Mobile keeps the existing facing-mode + toggle — do not replace the phone chrome with a device ``, Batch + Scan progress, and the three-tab strip. Do **not** skip + `ui-design`. +3. `role-ux-reviewer` — inspect-then-add vs scan-all-then-checkout + on desktop; Auto-detect off; Rescan; empty inspector; leave + with an uncommitted queue; sequential batch cancel; duplicate + tab actions (increment qty vs skip vs still add). +4. `role-architect` — briefs. Likely: (1) page composition + + Layout chrome split, (2) camera panel + `deviceId` picker, + (3) result inspector, (4) history/queue/duplicates strip, + (5) batch multi-file identify + Tips. `slice_dependencies` + must mark what can run in parallel. +5. `role-implementer` — per brief. +6. Audit fan-out: reviewer + security-auditor + design-system-auditor + + a11y-auditor. + +## Todos + +- [x] IA: desktop screen inventory + inspector-add vs cart-commit; + Duplicates membership; Batch Scan + Tips content +- [x] UI Designer: lock md+ workstation (tokens, not hex) from + the attached mock; light + dark; Tips, device picker, + batch progress, three-tab strip +- [x] UX: Auto-detect off, Rescan, empty state, leave-with-queue, + keyboard on desk controls, batch cancel, duplicate actions +- [x] Architect: briefs + `slice_dependencies`; confirm Layout + is `chrome="default"` on md+ only +- [ ] Desktop composition in `pages/scanner.js` (do not hide + sidebar / top bar at md+) +- [ ] Camera as a framed panel; `deviceId` picker + Upload + + Batch Scan + Auto-detect; hide mobile overlay chrome at md+ +- [ ] Live match inspector wired to the latest unprocessed + identify (condition / foil already on the cart entry) +- [ ] Bottom strip: Recent Scans + Scan Queue + Duplicates over + `useScannerQueue` / `ownershipMap` / `scanner-session` +- [ ] Scanner Tips popover/modal with convoy-authored copy +- [ ] Sequential multi-file Batch Scan through + `identifyFromGalleryFile` (cancellable, queue progress) +- [ ] Tests for desktop composition (inspector + queue + + duplicates + batch enqueue, no mobile sheet) and + no-regression on checkout sheet at `max-width: 767px` +- [ ] Visual-diff: desktop `/scanner` surface; refresh Linux + baselines if the page is in the visual suite + +## What exists today (conductor survey) + +`/scanner` is one route, two compositions, one engine. + +| Layer | Files | Today | +| --- | --- | --- | +| Page | `pages/scanner.js` | Auth gate; `Layout chrome="immersive"` on **all** viewports; camera column + `md:` 360px `ScannerReview` cart; mobile-only `ScannerCheckoutSheet` | +| Layout | `components/Layout.js` | Immersive hides **mobile** nav + top bar (`max-md` only). Desktop sidebar + TopSearchBar already stay visible. | +| Camera chrome | `components/scanner/ScannerCamera.js` | Full-bleed video, overlay top bar (back / title / gallery), bottom bar (flash / facing / status / Review N), scan peek, disambiguation. Same chrome on desktop. | +| Cart | `lib/use-scanner-queue.js`, `lib/scanner-session.js` | Identify enqueues locally (`processed: false`). Commit via `commitSelectedToOwned` / `commitSelectedToCollection`. `sessionStorage` persist. `addSingleCardToOwned` already exists. | +| Identify | `lib/use-scanner-identification.js`, `lib/use-camera-scanner.js` | Facing-mode swap only — **no** `deviceId` / `enumerateDevices`. Auto-detect is always on unless `verificationPausedRef` (checkout / list picker / disambiguation). | +| Review | `components/scanner/ScannerReview.js` | Thin wrapper: desktop side panel titled "Cart" that mounts `ScannerCheckoutContent`. | + +Mobile checkout decisions that still apply unless IA overturns them +for desktop only: stay on camera after commit (D1), skip Setup +(D3), My Collection + List only (D4), gallery in-scope (D5), +cart in `sessionStorage` (D7). + +## Conductor notes (build shape) + +Likely file ownership for Architect to refine: + +| Area | Files | +| --- | --- | +| Viewport split | `pages/scanner.js` — `chrome` default on md+, immersive on mobile; desktop grid vs mobile overlay | +| Camera panel | `components/scanner/ScannerCamera.js` (desktop variant or `variant="workstation"`), `lib/use-camera-scanner.js` (`enumerateDevices` + `deviceId`) | +| Inspector | new `components/scanner/ScannerResultPanel.js` — latest cart entry + condition/foil + add/rescan | +| History strip | new `components/scanner/ScannerHistoryStrip.js` — Recent / Queue / Duplicates over `queue.scannedCards` + `ownershipMap` | +| Batch Scan | `lib/use-scanner-identification.js` (`identifyFromGalleryFile` loop), queue progress UI | +| Tips | new `components/scanner/ScannerTips.js` — `` or popover, convoy copy | +| Cart reuse | `lib/use-scanner-queue.js`, `lib/scanner-session.js`, `ScannerCheckoutSheet.js` (mobile only) | +| Copy | `lib/collection-vocabulary.js` | + +Do not rewrite identification. Prefer a desktop layout shell that +**hides** mobile overlay chrome at `md+` rather than forking the +camera hook. + +## Decisions (post-conductor) + +Locked 2026-08-15 from the parent session. IA / UX / Architect +treat these as settled. + +| # | Decision | +| --- | --- | +| C1 | **Batch Scan is in.** Multi-file sequential identify via the existing gallery path. No new batch API, no parallel Gemini, no new pile detector. | +| C2 | **Duplicates tab is in.** Strip tab with a badge. Membership = already-owned (`ownershipMap`) and/or same-session name+set repeats. IA picks the exact rule and tab actions. | +| C3 | **Scanner Tips is in.** Header control → glass popover/modal. Copy in this convoy. | +| C4 | **Real webcam device picker is in.** `enumerateDevices` + `deviceId` on desktop. Mobile keeps facing-mode swap. | +| C5 | **Inspector can commit this card now** *and* the queue strip remains for multi-add (same cart, two commit surfaces). Overturn only if IA finds a conflict. | + +## Open questions (IA / product) + +1. **Duplicates membership + actions.** Owned-in-collection only, + session repeats only, or both? From the tab, can the user still + add (increment qty), skip, or jump the inspector to that row? +2. **Wishlist.** Out as a feature. Confirm "Save to Wishlist" → + Add to List on the inspector, or omit the third action. +3. **Auto-detect toggle.** User-facing pause of identification + (extend `verificationPausedRef`), or just a status badge? +4. **Batch Scan cancel / errors.** Mid-batch cancel: keep already- + identified rows? Per-file failure: continue the rest and flag + the row, or stop? +5. **Tips content.** Four or five short tips (lighting, frame the + card, foil glare, hold still, auto-detect). IA drafts; UI + Designer locks the surface. + +## Multitask dispatch + +Planning is serial: IA → UI Designer → UX → Architect. + +After architect: implementer fan-out only if briefs have +`depends_on: []` and disjoint `files:`. Device picker +(`use-camera-scanner.js`) and Tips (`ScannerTips.js`) are the +best candidates to parallelize with the inspector if they do +not both own `pages/scanner.js`. Page composition likely +blocks the strip and Batch Scan wiring. + +After PR draft: `/multitask` audit fan-out +`role-reviewer + role-security-auditor + role-design-system-auditor + role-a11y-auditor` +(group id: `audit-scanner-desktop-layout-`). + +## IA + +### Affected routes + +- `/scanner` — **[modified]** Single route, two viewport compositions. Desktop (`md+`) switches to `Layout chrome="default"` (sidebar + TopSearchBar visible), framed camera workstation, live match inspector (right rail), and bottom history strip. Mobile (`max-md`) stays `chrome="immersive"` with checkout sheet — no regression. +- `/login` — **[impacted]** Existing `returnUrl=/scanner` auth gate unchanged; desktop users land on the workstation after sign-in. +- `/collections`, `/my-cards` — **[impacted]** Post-commit navigation targets only (Add to List picker, success flows). No route or nav IA changes in this convoy. + +No new routes. No API route changes. + +### User flow + +```mermaid +flowchart LR + A["/scanner (auth)"] --> B{"md+?"} + B -->|Yes| C["Workstation"] + B -->|No| D["Immersive mobile"] + C --> E["Scan / Upload / Batch"] + E --> F["Match inspector"] + F --> G["Add or queue"] + C --> H["Strip tabs"] + H --> F +``` + +Desktop path: user opens `/scanner` with app chrome → scans via webcam, single Upload Image, or Batch Scan (sequential gallery identify) → latest match appears in the right-rail inspector → commits one card via inspector **or** batches via Scan Queue strip → Duplicates tab surfaces owned + session-repeat rows for review/increment. Mobile path unchanged: full-bleed camera → checkout sheet. + +### Screen inventory + +| Screen | Path | New/modified | Notes | +| --- | --- | --- | --- | +| Scanner Desktop Workstation | `/scanner` | modified | `md+` grid: framed camera panel (device picker, Upload, Batch Scan, Auto-detect toggle, Tips), right-rail inspector, bottom strip. Replaces 360px cart side panel as primary right-hand surface. | +| Scanner Mobile Immersive | `/scanner` | impacted (no regression) | `max-md`: `chrome="immersive"`, overlay camera chrome, `ScannerCheckoutSheet`. D1/D3/D4/D5/D7 decisions preserved. | +| Live Match Inspector | `/scanner` | new (sub-surface) | Right rail on desktop. Shows latest unprocessed identify: thumbnail, name, set, rarity, collector #, condition, foil, confidence. Actions: `VOCAB.ADD_TO_MY_COLLECTION`, `VOCAB.ADD_TO_LIST`, Rescan. Single-card commit without opening checkout sheet. | +| History / Queue Strip | `/scanner` | new (sub-surface) | Bottom strip on desktop. Tabs: **Recent Scans** (session history), **Scan Queue** (uncommitted cart, badge = unprocessed count), **Duplicates** (badge = duplicate row count). Row click focuses card in inspector. | +| Scanner Tips | `/scanner` | new (sub-surface) | Header control → glass `` or popover. Five convoy-authored tips (see Content deltas). No route change. | +| List Picker | `/scanner` | impacted | Existing "Choose a List" ``. Opened from inspector `VOCAB.ADD_TO_LIST` on desktop (and unchanged on mobile). | +| Leave Scanner | `/scanner` | impacted | Existing leave-with-uncommitted-queue modal. Applies to both viewports when navigating away with queue items. | + +### Content / data model deltas + +**Copy (ship from `lib/collection-vocabulary.js`):** + +- Primary add: `VOCAB.ADD_TO_MY_COLLECTION` ("Add to My Collection"). +- Secondary add: `VOCAB.ADD_TO_LIST` ("Add to List") — **not** "Save to Wishlist" (feature omitted). +- Strip tab labels: "Recent Scans", "Scan Queue", "Duplicates". +- Camera controls: "Upload Image", "Batch Scan", "Auto-detect" (toggle + status on/off), "Scanner Tips", "Rescan". +- Device picker: "Camera" or "Webcam" `` or styled native picker, label **Camera** | `enumerateDevices` video inputs; desktop only | +| Upload Image | `