echos-ocr/scripts/migrate-storage.ts

/**
 * Storage Migration: MinIO → Supabase Storage
 *
 * Copies all objects from a MinIO bucket to a Supabase Storage bucket,
 * preserving the same key paths so database references remain valid.
 *
 * Usage:
 *   npx tsx scripts/migrate-storage.ts
 *
 * Required env vars:
 *   MINIO_ENDPOINT       - e.g. 192.168.68.105
 *   MINIO_PORT           - e.g. 9000
 *   MINIO_ACCESS_KEY     - e.g. minioadmin
 *   MINIO_SECRET_KEY     - your secret
 *   MINIO_BUCKET         - e.g. echos-ocr
 *
 *   STORAGE_ENDPOINT     - e.g. https://xxx.supabase.co/storage/v1/s3
 *   STORAGE_REGION       - e.g. us-east-1
 *   STORAGE_ACCESS_KEY   - Supabase S3 access key
 *   STORAGE_SECRET_KEY   - Supabase S3 secret key
 *   STORAGE_BUCKET       - e.g. echos-ocr
 */

import "dotenv/config";
import {
  S3Client,
  ListObjectsV2Command,
  GetObjectCommand,
  PutObjectCommand,
  HeadObjectCommand,
} from "@aws-sdk/client-s3";

const sourceClient = new S3Client({
  endpoint: `http://${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT || "9000"}`,
  region: "us-east-1",
  credentials: {
    accessKeyId: process.env.MINIO_ACCESS_KEY || "",
    secretAccessKey: process.env.MINIO_SECRET_KEY || "",
  },
  forcePathStyle: true,
});

const destClient = new S3Client({
  endpoint: process.env.STORAGE_ENDPOINT || "",
  region: process.env.STORAGE_REGION || "us-east-1",
  credentials: {
    accessKeyId: process.env.STORAGE_ACCESS_KEY || "",
    secretAccessKey: process.env.STORAGE_SECRET_KEY || "",
  },
  forcePathStyle: true,
});

const SOURCE_BUCKET = process.env.MINIO_BUCKET || "echos-ocr";
const DEST_BUCKET = process.env.STORAGE_BUCKET || "echos-ocr";

async function listAllKeys(): Promise<string[]> {
  const keys: string[] = [];
  let continuationToken: string | undefined;

  do {
    const res = await sourceClient.send(
      new ListObjectsV2Command({
        Bucket: SOURCE_BUCKET,
        ContinuationToken: continuationToken,
      })
    );
    for (const obj of res.Contents ?? []) {
      if (obj.Key) keys.push(obj.Key);
    }
    continuationToken = res.NextContinuationToken;
  } while (continuationToken);

  return keys;
}

async function objectExists(key: string): Promise<boolean> {
  try {
    await destClient.send(
      new HeadObjectCommand({ Bucket: DEST_BUCKET, Key: key })
    );
    return true;
  } catch {
    return false;
  }
}

async function copyObject(key: string): Promise<void> {
  const res = await sourceClient.send(
    new GetObjectCommand({ Bucket: SOURCE_BUCKET, Key: key })
  );
  const stream = res.Body as ReadableStream;
  const buffer = Buffer.from(await new Response(stream).arrayBuffer());

  await destClient.send(
    new PutObjectCommand({
      Bucket: DEST_BUCKET,
      Key: key,
      Body: buffer,
      ContentType: res.ContentType || "application/octet-stream",
    })
  );
}

async function main() {
  console.log("=== Storage Migration: MinIO → Supabase Storage ===");
  console.log(`Source: ${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT} / ${SOURCE_BUCKET}`);
  console.log(`Dest:   ${process.env.STORAGE_ENDPOINT} / ${DEST_BUCKET}`);
  console.log("");

  console.log("Listing all objects in source bucket...");
  const keys = await listAllKeys();
  console.log(`Found ${keys.length} object(s) to migrate`);

  if (keys.length === 0) {
    console.log("Nothing to migrate.");
    return;
  }

  const sourcesCount = keys.filter((k) => k.startsWith("sources/")).length;
  const imagesCount = keys.filter((k) => k.startsWith("images/")).length;
  const otherCount = keys.length - sourcesCount - imagesCount;
  console.log(`  sources/: ${sourcesCount}, images/: ${imagesCount}, other: ${otherCount}`);
  console.log("");

  let copied = 0;
  let skipped = 0;
  let errors = 0;

  for (let i = 0; i < keys.length; i++) {
    const key = keys[i];
    const progress = `[${i + 1}/${keys.length}]`;

    try {
      const exists = await objectExists(key);
      if (exists) {
        skipped++;
        if ((i + 1) % 50 === 0) {
          console.log(`${progress} Skipped (already exists): ${key}`);
        }
        continue;
      }

      await copyObject(key);
      copied++;

      if (copied % 10 === 0 || (i + 1) === keys.length) {
        console.log(`${progress} Copied: ${key} (${copied} total)`);
      }
    } catch (err) {
      errors++;
      console.error(`${progress} ERROR copying ${key}:`, err instanceof Error ? err.message : err);
    }
  }

  console.log("");
  console.log("=== Migration Summary ===");
  console.log(`Total objects: ${keys.length}`);
  console.log(`Copied:        ${copied}`);
  console.log(`Skipped:       ${skipped} (already existed in destination)`);
  console.log(`Errors:        ${errors}`);

  if (errors > 0) {
    console.log("\nSome objects failed to copy. Re-run this script to retry (it skips already-copied objects).");
    process.exit(1);
  }

  console.log("\nAll objects migrated successfully.");
}

main().catch((err) => {
  console.error("Fatal error:", err);
  process.exit(1);
});