From 7007eae3ba21c6724e075aec0131c2912aada25b Mon Sep 17 00:00:00 2001 From: varutasu <104105839+varutasu@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:53:34 -0500 Subject: [PATCH] fix(ci): Phase 3 post-merge CI gates (#162) * fix(ci): allow card-embed.js and skip pgvector on non-superuser CI Add lib/card-embed.js to the server-only LLM allowlist (forbidden-patterns Check 3). Make the pgvector migration degrade gracefully when CT 102 CI cannot CREATE EXTENSION vector so migrate up still passes. Co-authored-by: Cursor * docs(convoy): post-ship metrics and operator checklist for Phase 3 Record Neon migration applied, scan_attempts snapshot, and backfill gate. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- .convoys/scan-visual-catalog-search.md | 24 +++++++++++- .convoys/scanner-identify-upgrade.md | 9 +++-- .github/workflows/ci.yml | 1 + .../1782000000001_add-card-embeddings.js | 38 +++++++++++++++---- 4 files changed, 59 insertions(+), 13 deletions(-) diff --git a/.convoys/scan-visual-catalog-search.md b/.convoys/scan-visual-catalog-search.md index 19c97d7..8a7e154 100644 --- a/.convoys/scan-visual-catalog-search.md +++ b/.convoys/scan-visual-catalog-search.md @@ -14,7 +14,7 @@ skip: - a11y - design - flag -status: in-progress +status: shipped created: 2026-08-14 depends_on: - improve-scan-card-detection @@ -113,7 +113,27 @@ unknown-card fallback. - [x] Brief 3 — identify kNN route + client escalate order - [ ] Threshold bake-off on real crops - [ ] Re-measure auto-match % excluding `not_a_card` -- [ ] Operator: `npm run backfill-embeddings` on prod/staging after migrate +- [ ] Operator: `AI_GATEWAY_API_KEY` + `npm run backfill-embeddings` on Neon (migrate applied 2026-08-15) + +## Post-ship (2026-08-15) + +**CI fixes:** PR #162 — `lib/card-embed.js` allowlist + pgvector migration +graceful skip on non-superuser homelab CI. + +**Neon operator:** `1782000000001_add-card-embeddings` applied manually +(pgvector + columns + index + pgmigrations row). Full backfill pending +`AI_GATEWAY_API_KEY` in operator env. + +**scan_attempts snapshot** (90d, n=250, pre-backfill / no L0 traffic yet): + +| Signal | Value | Baseline | +| --- | --- | --- | +| L1 escalate | 78.4% (87/111) | 76.7% | +| L1 matched | 5.4% (6/111) | 5.8% | +| L2 not_a_card | 43.9% (61/139) | 44.6% | +| End-to-end auto-match | 10.0% (25/250) | 10.3% | + +Re-measure after backfill + preview scanning for L0 `layer=0` rows. ## Likely file ownership diff --git a/.convoys/scanner-identify-upgrade.md b/.convoys/scanner-identify-upgrade.md index cc3122e..f2a602a 100644 --- a/.convoys/scanner-identify-upgrade.md +++ b/.convoys/scanner-identify-upgrade.md @@ -169,10 +169,11 @@ on all three (no new routes or visual language). - [x] Pull `scan_attempts` baseline (2026-08-14) - [x] Open worktree `scanner-identify-upgrade` from `main` - [x] Seed sub-convoys #1–#3 -- [ ] Architect: pick up `tighten-scan-identify-hot-path` first -- [ ] Re-measure `scan_attempts` after #1 ships -- [ ] Architect: `improve-scan-card-detection` (or parallel if files stay disjoint) -- [ ] Architect: `scan-visual-catalog-search` after warped crops exist +- [x] Architect: pick up `tighten-scan-identify-hot-path` first +- [ ] Re-measure `scan_attempts` after all phases ship (post-backfill) +- [x] Architect: `improve-scan-card-detection` +- [x] Architect: `scan-visual-catalog-search` after warped crops exist +- [ ] Operator: merge #162 CI fix, `npm run migrate up`, `npm run backfill-embeddings` ## Worktree diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 270ed7f..99e56a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,6 +196,7 @@ jobs: fi SERVER_ONLY=( lib/scan-vision.js + lib/card-embed.js ) LLM_PATTERN='generativelanguage\.googleapis\.com|api\.openai\.com|ai-gateway\.vercel\.sh' LLM_FOUND=() diff --git a/migrations/1782000000001_add-card-embeddings.js b/migrations/1782000000001_add-card-embeddings.js index 3affc56..6e8cdbc 100644 --- a/migrations/1782000000001_add-card-embeddings.js +++ b/migrations/1782000000001_add-card-embeddings.js @@ -1,6 +1,11 @@ /** * pgvector + cards.embedding for Layer-0 visual catalog search. * + * Neon prod: CREATE EXTENSION vector succeeds for the project owner. + * Homelab CI (`deckhearth_ci`, non-superuser): extension create may fail; + * we skip embedding DDL so migrate still passes — L0 escalates until + * an operator enables pgvector on CT 102 or runs backfill on Neon. + * * @type {import('node-pg-migrate').ColumnDefinitions | undefined} */ export const shorthands = undefined; @@ -13,15 +18,34 @@ const EMBED_DIMENSION = 1024; */ export const up = (pgm) => { pgm.sql(` - CREATE EXTENSION IF NOT EXISTS vector; + DO $do$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') THEN + CREATE EXTENSION vector; + END IF; + EXCEPTION + WHEN insufficient_privilege OR OTHERS THEN + RAISE NOTICE 'vector extension unavailable on this database (%). Skipping embedding DDL.', SQLERRM; + RETURN; + END + $do$; - ALTER TABLE cards - ADD COLUMN IF NOT EXISTS embedding vector(${EMBED_DIMENSION}), - ADD COLUMN IF NOT EXISTS embedded_at TIMESTAMP; + DO $do$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') THEN + RAISE NOTICE 'vector extension not installed — skipping cards.embedding columns'; + RETURN; + END IF; - CREATE INDEX IF NOT EXISTS idx_cards_embedding_hnsw - ON cards USING hnsw (embedding vector_cosine_ops) - WHERE embedding IS NOT NULL; + ALTER TABLE cards + ADD COLUMN IF NOT EXISTS embedding vector(${EMBED_DIMENSION}), + ADD COLUMN IF NOT EXISTS embedded_at TIMESTAMP; + + CREATE INDEX IF NOT EXISTS idx_cards_embedding_hnsw + ON cards USING hnsw (embedding vector_cosine_ops) + WHERE embedding IS NOT NULL; + END + $do$; `); };