/** * Storage Migration: MinIO → Supabase Storage * * Copies all objects from a MinIO bucket to a Supabase Storage bucket, * preserving the same key paths so database references remain valid. * * Usage: * npx tsx scripts/migrate-storage.ts * * Required env vars: * MINIO_ENDPOINT - e.g. 192.168.68.105 * MINIO_PORT - e.g. 9000 * MINIO_ACCESS_KEY - e.g. minioadmin * MINIO_SECRET_KEY - your secret * MINIO_BUCKET - e.g. echos-ocr * * STORAGE_ENDPOINT - e.g. https://xxx.supabase.co/storage/v1/s3 * STORAGE_REGION - e.g. us-east-1 * STORAGE_ACCESS_KEY - Supabase S3 access key * STORAGE_SECRET_KEY - Supabase S3 secret key * STORAGE_BUCKET - e.g. echos-ocr */ import "dotenv/config"; import { S3Client, ListObjectsV2Command, GetObjectCommand, PutObjectCommand, HeadObjectCommand, } from "@aws-sdk/client-s3"; const sourceClient = new S3Client({ endpoint: `http://${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT || "9000"}`, region: "us-east-1", credentials: { accessKeyId: process.env.MINIO_ACCESS_KEY || "", secretAccessKey: process.env.MINIO_SECRET_KEY || "", }, forcePathStyle: true, }); const destClient = new S3Client({ endpoint: process.env.STORAGE_ENDPOINT || "", region: process.env.STORAGE_REGION || "us-east-1", credentials: { accessKeyId: process.env.STORAGE_ACCESS_KEY || "", secretAccessKey: process.env.STORAGE_SECRET_KEY || "", }, forcePathStyle: true, }); const SOURCE_BUCKET = process.env.MINIO_BUCKET || "echos-ocr"; const DEST_BUCKET = process.env.STORAGE_BUCKET || "echos-ocr"; async function listAllKeys(): Promise { const keys: string[] = []; let continuationToken: string | undefined; do { const res = await sourceClient.send( new ListObjectsV2Command({ Bucket: SOURCE_BUCKET, ContinuationToken: continuationToken, }) ); for (const obj of res.Contents ?? []) { if (obj.Key) keys.push(obj.Key); } continuationToken = res.NextContinuationToken; } while (continuationToken); return keys; } async function objectExists(key: string): Promise { try { await destClient.send( new HeadObjectCommand({ Bucket: DEST_BUCKET, Key: key }) ); return true; } catch { return false; } } async function copyObject(key: string): Promise { const res = await sourceClient.send( new GetObjectCommand({ Bucket: SOURCE_BUCKET, Key: key }) ); const stream = res.Body as ReadableStream; const buffer = Buffer.from(await new Response(stream).arrayBuffer()); await destClient.send( new PutObjectCommand({ Bucket: DEST_BUCKET, Key: key, Body: buffer, ContentType: res.ContentType || "application/octet-stream", }) ); } async function main() { console.log("=== Storage Migration: MinIO → Supabase Storage ==="); console.log(`Source: ${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT} / ${SOURCE_BUCKET}`); console.log(`Dest: ${process.env.STORAGE_ENDPOINT} / ${DEST_BUCKET}`); console.log(""); console.log("Listing all objects in source bucket..."); const keys = await listAllKeys(); console.log(`Found ${keys.length} object(s) to migrate`); if (keys.length === 0) { console.log("Nothing to migrate."); return; } const sourcesCount = keys.filter((k) => k.startsWith("sources/")).length; const imagesCount = keys.filter((k) => k.startsWith("images/")).length; const otherCount = keys.length - sourcesCount - imagesCount; console.log(` sources/: ${sourcesCount}, images/: ${imagesCount}, other: ${otherCount}`); console.log(""); let copied = 0; let skipped = 0; let errors = 0; for (let i = 0; i < keys.length; i++) { const key = keys[i]; const progress = `[${i + 1}/${keys.length}]`; try { const exists = await objectExists(key); if (exists) { skipped++; if ((i + 1) % 50 === 0) { console.log(`${progress} Skipped (already exists): ${key}`); } continue; } await copyObject(key); copied++; if (copied % 10 === 0 || (i + 1) === keys.length) { console.log(`${progress} Copied: ${key} (${copied} total)`); } } catch (err) { errors++; console.error(`${progress} ERROR copying ${key}:`, err instanceof Error ? err.message : err); } } console.log(""); console.log("=== Migration Summary ==="); console.log(`Total objects: ${keys.length}`); console.log(`Copied: ${copied}`); console.log(`Skipped: ${skipped} (already existed in destination)`); console.log(`Errors: ${errors}`); if (errors > 0) { console.log("\nSome objects failed to copy. Re-run this script to retry (it skips already-copied objects)."); process.exit(1); } console.log("\nAll objects migrated successfully."); } main().catch((err) => { console.error("Fatal error:", err); process.exit(1); });