dcrouter/ts_migrations/index.ts

/**
 * dcrouter migration runner.
 *
 * Uses @push.rocks/smartmigration via dynamic import so smartmigration's type
 * chain (which pulls in mongodb 7.x and related types) doesn't leak into
 * compile-time type checking for this folder.
 */

/** Matches the subset of IMigrationRunResult we actually log. */
export interface IMigrationRunResult {
  stepsApplied: Array<unknown>;
  wasFreshInstall: boolean;
  currentVersionBefore: string | null;
  currentVersionAfter: string;
  totalDurationMs: number;
}

export interface IMigrationRunner {
  run(): Promise<IMigrationRunResult>;
}

type TMigrationSecurity = Record<string, any>;

const DEFAULT_SOURCE_PROFILES: Array<{
  name: string;
  description: string;
  security: TMigrationSecurity;
}> = [
  {
    name: 'TRUSTED NETWORKS',
    description: 'Trusted office, VPN, localhost, and private-network sources with high connection allowance',
    security: {
      ipAllowList: ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '127.0.0.1', '::1'],
      maxConnections: 5000,
    },
  },
  {
    name: 'AI CRAWLERS',
    description: 'Add verified crawler CIDRs before assigning this profile in a source policy',
    security: {
      ipAllowList: [],
      rateLimit: {
        enabled: true,
        maxRequests: 30,
        window: 60,
        keyBy: 'ip',
      },
    },
  },
  {
    name: 'PUBLIC',
    description: 'Public fallback source profile with per-IP request limiting',
    security: {
      ipAllowList: ['*'],
      rateLimit: {
        enabled: true,
        maxRequests: 120,
        window: 60,
        keyBy: 'ip',
      },
    },
  },
];

function mergeMigrationSecurityFields(
  base: TMigrationSecurity | undefined,
  override: TMigrationSecurity | undefined,
): TMigrationSecurity {
  if (!base && !override) return {};
  if (!base) return structuredClone(override || {});
  if (!override) return structuredClone(base || {});

  const merged: TMigrationSecurity = structuredClone(base);

  if (override.ipAllowList || base.ipAllowList) {
    merged.ipAllowList = [
      ...new Set([
        ...(base.ipAllowList || []),
        ...(override.ipAllowList || []),
      ]),
    ];
  }

  if (override.ipBlockList || base.ipBlockList) {
    merged.ipBlockList = [
      ...new Set([
        ...(base.ipBlockList || []),
        ...(override.ipBlockList || []),
      ]),
    ];
  }

  for (const key of ['maxConnections', 'rateLimit', 'authentication', 'basicAuth', 'jwtAuth', 'vpn']) {
    if (override[key] !== undefined) {
      merged[key] = structuredClone(override[key]);
    }
  }

  return merged;
}

function resolveMigrationSourceProfileSecurity(
  profileId: string,
  profiles: Map<string, any>,
  visited = new Set<string>(),
  depth = 0,
): TMigrationSecurity | null {
  if (depth > 5 || visited.has(profileId)) return null;

  const profile = profiles.get(profileId);
  if (!profile) return null;

  visited.add(profileId);
  let baseSecurity: TMigrationSecurity = {};
  const extendsProfiles = Array.isArray(profile.extendsProfiles) ? profile.extendsProfiles : [];
  for (const parentId of extendsProfiles) {
    if (typeof parentId !== 'string') continue;
    const parentSecurity = resolveMigrationSourceProfileSecurity(
      parentId,
      profiles,
      new Set(visited),
      depth + 1,
    );
    if (parentSecurity) {
      baseSecurity = mergeMigrationSecurityFields(baseSecurity, parentSecurity);
    }
  }

  return mergeMigrationSecurityFields(baseSecurity, profile.security || {});
}

async function rematerializeSourceProfileRouteSecurity(ctx: {
  mongo?: { collection: (name: string) => any };
  log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
  const profileCollection = ctx.mongo!.collection('SourceProfileDoc');
  const routeCollection = ctx.mongo!.collection('RouteDoc');
  const profiles = new Map<string, any>();

  for await (const profile of profileCollection.find({})) {
    if (typeof (profile as any).id === 'string') {
      profiles.set((profile as any).id, profile);
    }
  }

  let inspected = 0;
  let migrated = 0;
  let skippedMissingProfile = 0;
  const now = Date.now();

  for await (const routeDoc of routeCollection.find({})) {
    const sourceProfileRef = (routeDoc as any).metadata?.sourceProfileRef;
    if (typeof sourceProfileRef !== 'string' || sourceProfileRef.trim() === '') continue;
    inspected++;

    const resolvedSecurity = resolveMigrationSourceProfileSecurity(sourceProfileRef, profiles);
    const profile = profiles.get(sourceProfileRef);
    if (!resolvedSecurity || !profile) {
      skippedMissingProfile++;
      continue;
    }

    const currentSecurity = (routeDoc as any).route?.security || {};
    const securityChanged = JSON.stringify(currentSecurity) !== JSON.stringify(resolvedSecurity);
    const profileNameChanged = (routeDoc as any).metadata?.sourceProfileName !== profile.name;
    if (!securityChanged && !profileNameChanged) continue;

    const query = (routeDoc as any)._id
      ? { _id: (routeDoc as any)._id }
      : { id: (routeDoc as any).id };
    await routeCollection.updateOne(query, {
      $set: {
        'route.security': structuredClone(resolvedSecurity),
        'metadata.sourceProfileName': profile.name,
        'metadata.lastResolvedAt': now,
        updatedAt: now,
      },
    });
    migrated++;
  }

  ctx.log.log(
    'info',
    `rematerialize-source-profile-route-security: migrated ${migrated}/${inspected} route(s), skipped ${skippedMissingProfile} missing profile ref(s)`,
  );
}

async function migrateTargetProfileTargetHosts(ctx: {
  mongo?: { collection: (name: string) => any };
  log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
  const collection = ctx.mongo!.collection('TargetProfileDoc');
  const cursor = collection.find({ 'targets.host': { $exists: true } });
  let migrated = 0;

  for await (const doc of cursor) {
    const targets = ((doc as any).targets || []).map((target: any) => {
      if (target && typeof target === 'object' && 'host' in target && !('ip' in target)) {
        const { host, ...rest } = target;
        return { ...rest, ip: host };
      }
      return target;
    });

    await collection.updateOne({ _id: (doc as any)._id }, { $set: { targets } });
    migrated++;
  }

  ctx.log.log('info', `rename-target-profile-host-to-ip: migrated ${migrated} profile(s)`);
}

async function backfillSystemRouteKeys(ctx: {
  mongo?: { collection: (name: string) => any };
  log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
  const collection = ctx.mongo!.collection('RouteDoc');
  const cursor = collection.find({
    origin: { $in: ['config', 'email', 'dns'] },
    systemKey: { $exists: false },
    'route.name': { $type: 'string' },
  });
  let migrated = 0;

  for await (const doc of cursor) {
    const origin = typeof (doc as any).origin === 'string' ? (doc as any).origin : undefined;
    const routeName = typeof (doc as any).route?.name === 'string' ? (doc as any).route.name.trim() : '';
    if (!origin || !routeName) continue;

    await collection.updateOne(
      { _id: (doc as any)._id },
      { $set: { systemKey: `${origin}:${routeName}` } },
    );
    migrated++;
  }

  ctx.log.log('info', `backfill-system-route-keys: migrated ${migrated} route(s)`);
}

async function seedMissingDefaultSourceProfiles(ctx: {
  mongo?: { collection: (name: string) => any };
  log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
  const collection = ctx.mongo!.collection('SourceProfileDoc');
  const now = Date.now();
  let inserted = 0;
  let existing = 0;

  for (const profile of DEFAULT_SOURCE_PROFILES) {
    const existingProfile = await collection.findOne({ name: profile.name });
    if (existingProfile) {
      existing++;
      continue;
    }

    await collection.insertOne({
      id: globalThis.crypto.randomUUID(),
      name: profile.name,
      description: profile.description,
      security: structuredClone(profile.security),
      createdAt: now,
      updatedAt: now,
      createdBy: 'system',
    });
    inserted++;
  }

  ctx.log.log(
    'info',
    `seed-missing-default-source-profiles: inserted ${inserted}, already present ${existing}`,
  );
}

function normalizeMigrationSourceBinding(binding: any, profiles: Map<string, any>): any | undefined {
  if (!binding || typeof binding !== 'object') return undefined;
  const sourceProfileRef = typeof binding.sourceProfileRef === 'string'
    ? binding.sourceProfileRef.trim()
    : '';
  if (!sourceProfileRef) return undefined;

  const profile = profiles.get(sourceProfileRef);
  const normalizedBinding = structuredClone(binding);
  normalizedBinding.sourceProfileRef = sourceProfileRef;
  const sourceProfileName = typeof normalizedBinding.sourceProfileName === 'string'
    ? normalizedBinding.sourceProfileName.trim()
    : '';
  if (sourceProfileName) {
    normalizedBinding.sourceProfileName = sourceProfileName;
  } else if (typeof profile?.name === 'string' && profile.name.trim()) {
    normalizedBinding.sourceProfileName = profile.name.trim();
  } else {
    delete normalizedBinding.sourceProfileName;
  }

  return normalizedBinding;
}

async function convertRouteAccessMetadataToSourceBindings(ctx: {
  mongo?: { collection: (name: string) => any };
  log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
  const profileCollection = ctx.mongo!.collection('SourceProfileDoc');
  const routeCollection = ctx.mongo!.collection('RouteDoc');
  const profiles = new Map<string, any>();
  const now = Date.now();

  for await (const profile of profileCollection.find({})) {
    if (typeof (profile as any).id === 'string') {
      profiles.set((profile as any).id, profile);
    }
  }

  let inspected = 0;
  let migrated = 0;
  for await (const routeDoc of routeCollection.find({})) {
    const metadata = (routeDoc as any).metadata || {};
    const existingSourceBindings = Array.isArray(metadata.sourceBindings)
      ? metadata.sourceBindings
      : [];
    const legacyPolicyBindings = Array.isArray(metadata.sourcePolicy?.bindings)
      ? metadata.sourcePolicy.bindings
      : [];
    const legacySourceProfileRef = typeof metadata.sourceProfileRef === 'string'
      ? metadata.sourceProfileRef.trim()
      : '';
    const hasLegacyAccessFields = legacyPolicyBindings.length > 0
      || legacySourceProfileRef.length > 0
      || metadata.sourcePolicy !== undefined
      || metadata.sourceProfileRef !== undefined
      || metadata.sourceProfileName !== undefined;

    if (!hasLegacyAccessFields && existingSourceBindings.length === 0) {
      continue;
    }
    inspected++;

    const sourceBindings = existingSourceBindings.length > 0
      ? existingSourceBindings
          .map((binding: any) => normalizeMigrationSourceBinding(binding, profiles))
          .filter(Boolean)
      : legacyPolicyBindings.length > 0
        ? legacyPolicyBindings
            .map((binding: any) => normalizeMigrationSourceBinding(binding, profiles))
            .filter(Boolean)
        : legacySourceProfileRef
          ? [normalizeMigrationSourceBinding({
              sourceProfileRef: legacySourceProfileRef,
              sourceProfileName: metadata.sourceProfileName,
            }, profiles)].filter(Boolean)
          : [];

    const $set: Record<string, any> = { updatedAt: now };
    const $unset: Record<string, ''> = {
      'metadata.sourcePolicy': '',
      'metadata.sourceProfileRef': '',
      'metadata.sourceProfileName': '',
    };

    if (sourceBindings.length > 0) {
      $set['metadata.sourceBindings'] = sourceBindings;
      $set['metadata.lastResolvedAt'] = now;
    } else if (existingSourceBindings.length === 0) {
      $unset['metadata.sourceBindings'] = '';
    }

    if (existingSourceBindings.length === 0 && legacyPolicyBindings.length === 0 && legacySourceProfileRef) {
      $unset['route.security'] = '';
    }

    const query = (routeDoc as any)._id
      ? { _id: (routeDoc as any)._id }
      : { id: (routeDoc as any).id };
    await routeCollection.updateOne(query, { $set, $unset });
    migrated++;
  }

  ctx.log.log(
    'info',
    `convert-route-access-metadata-to-source-bindings: migrated ${migrated}/${inspected} route(s)`,
  );
}

/**
 * Create a configured SmartMigration runner with all dcrouter migration steps registered.
 *
 * Call `.run()` on the returned instance at startup (after DcRouterDb is ready,
 * before any service that reads migrated collections).
 *
 * @param db - The initialized SmartdataDb instance from DcRouterDb.getDb()
 * @param targetVersion - The current app version (from commitinfo.version)
 */
export async function createMigrationRunner(
  db: unknown,
  targetVersion: string,
): Promise<IMigrationRunner> {
  const sm = await import('@push.rocks/smartmigration');
  const migration = new sm.SmartMigration({
    targetVersion,
    db: db as any,
    // Brand-new installs skip all migrations and stamp directly to the current version.
    freshInstallVersion: targetVersion,
    // dcrouter uses the package version as targetVersion; bridge releases without DB changes.
    targetVersionStrategy: 'bridge',
  });

  // Register steps in execution order. Each step's .from() must match the
  // previous step's .to() to form a contiguous chain.
  migration
    .step('rename-target-profile-host-to-ip')
    .from('13.0.11').to('13.1.0')
    .description('Rename ITargetProfileTarget.host → ip on all target profiles')
    .up(async (ctx) => migrateTargetProfileTargetHosts(ctx))
    .step('rename-domain-source-manual-to-dcrouter')
    .from('13.1.0').to('13.8.1')
    .description('Rename DomainDoc.source value from "manual" to "dcrouter"')
    .up(async (ctx) => {
      const collection = ctx.mongo!.collection('domaindoc');
      const result = await collection.updateMany(
        { source: 'manual' },
        { $set: { source: 'dcrouter' } },
      );
      ctx.log.log(
        'info',
        `rename-domain-source-manual-to-dcrouter: migrated ${result.modifiedCount} domain(s)`,
      );
    })
    .step('rename-record-source-manual-to-local')
    .from('13.8.1').to('13.8.2')
    .description('Rename DnsRecordDoc.source value from "manual" to "local"')
    .up(async (ctx) => {
      const collection = ctx.mongo!.collection('dnsrecorddoc');
      const result = await collection.updateMany(
        { source: 'manual' },
        { $set: { source: 'local' } },
      );
      ctx.log.log(
        'info',
        `rename-record-source-manual-to-local: migrated ${result.modifiedCount} record(s)`,
      );
    })
    .step('unify-routes-rename-collection')
    .from('13.8.2').to('13.16.0')
    .description('Rename StoredRouteDoc → RouteDoc, add origin field, drop RouteOverrideDoc')
    .up(async (ctx) => {
      const db = ctx.mongo!;

      // 1. Rename StoredRouteDoc → RouteDoc (smartdata uses exact class names)
      const collections = await db.listCollections({ name: 'StoredRouteDoc' }).toArray();
      if (collections.length > 0) {
        await db.renameCollection('StoredRouteDoc', 'RouteDoc');
        ctx.log.log('info', 'Renamed StoredRouteDoc → RouteDoc');
      }

      // 2. Set origin='api' on all migrated docs (they were API-created)
      const routeCol = db.collection('RouteDoc');
      const result = await routeCol.updateMany(
        { origin: { $exists: false } },
        { $set: { origin: 'api' } },
      );
      ctx.log.log('info', `Set origin='api' on ${result.modifiedCount} migrated route(s)`);

      // 3. Drop RouteOverrideDoc collection
      const overrideCollections = await db.listCollections({ name: 'RouteOverrideDoc' }).toArray();
      if (overrideCollections.length > 0) {
        await db.collection('RouteOverrideDoc').drop();
        ctx.log.log('info', 'Dropped RouteOverrideDoc collection');
      }
    })
    .step('repair-target-profile-ip-migration')
    .from('13.16.0').to('13.17.4')
    .description('Repair TargetProfileDoc.targets host→ip migration for already-upgraded installs')
    .up(async (ctx) => {
      await migrateTargetProfileTargetHosts(ctx);
    })
    .step('backfill-system-route-keys')
    .from('13.17.4').to('13.18.0')
    .description('Backfill RouteDoc.systemKey for persisted config/email/dns routes')
    .up(async (ctx) => {
      await backfillSystemRouteKeys(ctx);
    })
    .step('rematerialize-source-profile-route-security')
    .from('13.18.0').to('13.40.2')
    .description('Replace stale route security with resolved source profile security')
    .up(async (ctx) => {
      await rematerializeSourceProfileRouteSecurity(ctx);
    })
    .step('seed-missing-default-source-profiles')
    .from('13.40.2').to('13.42.0')
    .description('Seed missing default source profiles for source-policy presets')
    .up(async (ctx) => {
      await seedMissingDefaultSourceProfiles(ctx);
    })
    .step('convert-route-access-metadata-to-source-bindings')
    .from('13.42.0').to('13.43.2')
    .description('Convert route sourceProfileRef/sourcePolicy metadata to canonical sourceBindings')
    .up(async (ctx) => {
      await convertRouteAccessMetadataToSourceBindings(ctx);
    });

  return migration;
}