Files
dcrouter/ts_migrations/index.ts
T

647 lines
22 KiB
TypeScript

/**
* dcrouter migration runner.
*
* Uses @push.rocks/smartmigration via dynamic import so smartmigration's type
* chain (which pulls in mongodb 7.x and related types) doesn't leak into
* compile-time type checking for this folder.
*/
/** Matches the subset of IMigrationRunResult we actually log. */
export interface IMigrationRunResult {
stepsApplied: Array<unknown>;
wasFreshInstall: boolean;
currentVersionBefore: string | null;
currentVersionAfter: string;
totalDurationMs: number;
}
export interface IMigrationRunner {
run(): Promise<IMigrationRunResult>;
}
type TMigrationSecurity = Record<string, any>;
export interface IDcRouterMigrationOptions {
remoteIngressHubSettings?: {
enabled?: boolean;
tunnelPort?: number;
hubDomain?: string | null;
performance?: Record<string, any> | null;
};
emailServerSettings?: {
enabled?: boolean;
emailConfig?: Record<string, any>;
emailPortConfig?: Record<string, any>;
};
}
const DEFAULT_SOURCE_PROFILES: Array<{
name: string;
description: string;
security: TMigrationSecurity;
}> = [
{
name: 'TRUSTED NETWORKS',
description: 'Trusted office, VPN, localhost, and private-network sources with high connection allowance',
security: {
ipAllowList: ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '127.0.0.1', '::1'],
maxConnections: 5000,
},
},
{
name: 'AI CRAWLERS',
description: 'Add verified crawler CIDRs before assigning this profile in a source policy',
security: {
ipAllowList: [],
rateLimit: {
enabled: true,
maxRequests: 30,
window: 60,
keyBy: 'ip',
},
},
},
{
name: 'PUBLIC',
description: 'Public fallback source profile with per-IP request limiting',
security: {
ipAllowList: ['*'],
rateLimit: {
enabled: true,
maxRequests: 120,
window: 60,
keyBy: 'ip',
},
},
},
];
const remoteIngressHubSettingsMigrationBaseVersion = '13.43.5';
function compareSemver(a: string, b: string): number {
const aParts = a.split('.').map((part) => Number.parseInt(part, 10) || 0);
const bParts = b.split('.').map((part) => Number.parseInt(part, 10) || 0);
const maxLength = Math.max(aParts.length, bParts.length);
for (let i = 0; i < maxLength; i++) {
const diff = (aParts[i] || 0) - (bParts[i] || 0);
if (diff !== 0) return diff;
}
return 0;
}
function mergeMigrationSecurityFields(
base: TMigrationSecurity | undefined,
override: TMigrationSecurity | undefined,
): TMigrationSecurity {
if (!base && !override) return {};
if (!base) return structuredClone(override || {});
if (!override) return structuredClone(base || {});
const merged: TMigrationSecurity = structuredClone(base);
if (override.ipAllowList || base.ipAllowList) {
merged.ipAllowList = [
...new Set([
...(base.ipAllowList || []),
...(override.ipAllowList || []),
]),
];
}
if (override.ipBlockList || base.ipBlockList) {
merged.ipBlockList = [
...new Set([
...(base.ipBlockList || []),
...(override.ipBlockList || []),
]),
];
}
for (const key of ['maxConnections', 'rateLimit', 'authentication', 'basicAuth', 'jwtAuth', 'vpn']) {
if (override[key] !== undefined) {
merged[key] = structuredClone(override[key]);
}
}
return merged;
}
function resolveMigrationSourceProfileSecurity(
profileId: string,
profiles: Map<string, any>,
visited = new Set<string>(),
depth = 0,
): TMigrationSecurity | null {
if (depth > 5 || visited.has(profileId)) return null;
const profile = profiles.get(profileId);
if (!profile) return null;
visited.add(profileId);
let baseSecurity: TMigrationSecurity = {};
const extendsProfiles = Array.isArray(profile.extendsProfiles) ? profile.extendsProfiles : [];
for (const parentId of extendsProfiles) {
if (typeof parentId !== 'string') continue;
const parentSecurity = resolveMigrationSourceProfileSecurity(
parentId,
profiles,
new Set(visited),
depth + 1,
);
if (parentSecurity) {
baseSecurity = mergeMigrationSecurityFields(baseSecurity, parentSecurity);
}
}
return mergeMigrationSecurityFields(baseSecurity, profile.security || {});
}
async function rematerializeSourceProfileRouteSecurity(ctx: {
mongo?: { collection: (name: string) => any };
log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
const profileCollection = ctx.mongo!.collection('SourceProfileDoc');
const routeCollection = ctx.mongo!.collection('RouteDoc');
const profiles = new Map<string, any>();
for await (const profile of profileCollection.find({})) {
if (typeof (profile as any).id === 'string') {
profiles.set((profile as any).id, profile);
}
}
let inspected = 0;
let migrated = 0;
let skippedMissingProfile = 0;
const now = Date.now();
for await (const routeDoc of routeCollection.find({})) {
const sourceProfileRef = (routeDoc as any).metadata?.sourceProfileRef;
if (typeof sourceProfileRef !== 'string' || sourceProfileRef.trim() === '') continue;
inspected++;
const resolvedSecurity = resolveMigrationSourceProfileSecurity(sourceProfileRef, profiles);
const profile = profiles.get(sourceProfileRef);
if (!resolvedSecurity || !profile) {
skippedMissingProfile++;
continue;
}
const currentSecurity = (routeDoc as any).route?.security || {};
const securityChanged = JSON.stringify(currentSecurity) !== JSON.stringify(resolvedSecurity);
const profileNameChanged = (routeDoc as any).metadata?.sourceProfileName !== profile.name;
if (!securityChanged && !profileNameChanged) continue;
const query = (routeDoc as any)._id
? { _id: (routeDoc as any)._id }
: { id: (routeDoc as any).id };
await routeCollection.updateOne(query, {
$set: {
'route.security': structuredClone(resolvedSecurity),
'metadata.sourceProfileName': profile.name,
'metadata.lastResolvedAt': now,
updatedAt: now,
},
});
migrated++;
}
ctx.log.log(
'info',
`rematerialize-source-profile-route-security: migrated ${migrated}/${inspected} route(s), skipped ${skippedMissingProfile} missing profile ref(s)`,
);
}
async function migrateTargetProfileTargetHosts(ctx: {
mongo?: { collection: (name: string) => any };
log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
const collection = ctx.mongo!.collection('TargetProfileDoc');
const cursor = collection.find({ 'targets.host': { $exists: true } });
let migrated = 0;
for await (const doc of cursor) {
const targets = ((doc as any).targets || []).map((target: any) => {
if (target && typeof target === 'object' && 'host' in target && !('ip' in target)) {
const { host, ...rest } = target;
return { ...rest, ip: host };
}
return target;
});
await collection.updateOne({ _id: (doc as any)._id }, { $set: { targets } });
migrated++;
}
ctx.log.log('info', `rename-target-profile-host-to-ip: migrated ${migrated} profile(s)`);
}
async function backfillSystemRouteKeys(ctx: {
mongo?: { collection: (name: string) => any };
log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
const collection = ctx.mongo!.collection('RouteDoc');
const cursor = collection.find({
origin: { $in: ['config', 'email', 'dns'] },
systemKey: { $exists: false },
'route.name': { $type: 'string' },
});
let migrated = 0;
for await (const doc of cursor) {
const origin = typeof (doc as any).origin === 'string' ? (doc as any).origin : undefined;
const routeName = typeof (doc as any).route?.name === 'string' ? (doc as any).route.name.trim() : '';
if (!origin || !routeName) continue;
await collection.updateOne(
{ _id: (doc as any)._id },
{ $set: { systemKey: `${origin}:${routeName}` } },
);
migrated++;
}
ctx.log.log('info', `backfill-system-route-keys: migrated ${migrated} route(s)`);
}
async function seedMissingDefaultSourceProfiles(ctx: {
mongo?: { collection: (name: string) => any };
log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
const collection = ctx.mongo!.collection('SourceProfileDoc');
const now = Date.now();
let inserted = 0;
let existing = 0;
for (const profile of DEFAULT_SOURCE_PROFILES) {
const existingProfile = await collection.findOne({ name: profile.name });
if (existingProfile) {
existing++;
continue;
}
await collection.insertOne({
id: globalThis.crypto.randomUUID(),
name: profile.name,
description: profile.description,
security: structuredClone(profile.security),
createdAt: now,
updatedAt: now,
createdBy: 'system',
});
inserted++;
}
ctx.log.log(
'info',
`seed-missing-default-source-profiles: inserted ${inserted}, already present ${existing}`,
);
}
function normalizeMigrationSourceBinding(binding: any, profiles: Map<string, any>): any | undefined {
if (!binding || typeof binding !== 'object') return undefined;
const sourceProfileRef = typeof binding.sourceProfileRef === 'string'
? binding.sourceProfileRef.trim()
: '';
if (!sourceProfileRef) return undefined;
const profile = profiles.get(sourceProfileRef);
const normalizedBinding = structuredClone(binding);
normalizedBinding.sourceProfileRef = sourceProfileRef;
const sourceProfileName = typeof normalizedBinding.sourceProfileName === 'string'
? normalizedBinding.sourceProfileName.trim()
: '';
if (sourceProfileName) {
normalizedBinding.sourceProfileName = sourceProfileName;
} else if (typeof profile?.name === 'string' && profile.name.trim()) {
normalizedBinding.sourceProfileName = profile.name.trim();
} else {
delete normalizedBinding.sourceProfileName;
}
return normalizedBinding;
}
async function convertRouteAccessMetadataToSourceBindings(ctx: {
mongo?: { collection: (name: string) => any };
log: { log: (level: 'info', message: string) => void };
}): Promise<void> {
const profileCollection = ctx.mongo!.collection('SourceProfileDoc');
const routeCollection = ctx.mongo!.collection('RouteDoc');
const profiles = new Map<string, any>();
const now = Date.now();
for await (const profile of profileCollection.find({})) {
if (typeof (profile as any).id === 'string') {
profiles.set((profile as any).id, profile);
}
}
let inspected = 0;
let migrated = 0;
for await (const routeDoc of routeCollection.find({})) {
const metadata = (routeDoc as any).metadata || {};
const existingSourceBindings = Array.isArray(metadata.sourceBindings)
? metadata.sourceBindings
: [];
const legacyPolicyBindings = Array.isArray(metadata.sourcePolicy?.bindings)
? metadata.sourcePolicy.bindings
: [];
const legacySourceProfileRef = typeof metadata.sourceProfileRef === 'string'
? metadata.sourceProfileRef.trim()
: '';
const hasLegacyAccessFields = legacyPolicyBindings.length > 0
|| legacySourceProfileRef.length > 0
|| metadata.sourcePolicy !== undefined
|| metadata.sourceProfileRef !== undefined
|| metadata.sourceProfileName !== undefined;
if (!hasLegacyAccessFields && existingSourceBindings.length === 0) {
continue;
}
inspected++;
const sourceBindings = existingSourceBindings.length > 0
? existingSourceBindings
.map((binding: any) => normalizeMigrationSourceBinding(binding, profiles))
.filter(Boolean)
: legacyPolicyBindings.length > 0
? legacyPolicyBindings
.map((binding: any) => normalizeMigrationSourceBinding(binding, profiles))
.filter(Boolean)
: legacySourceProfileRef
? [normalizeMigrationSourceBinding({
sourceProfileRef: legacySourceProfileRef,
sourceProfileName: metadata.sourceProfileName,
}, profiles)].filter(Boolean)
: [];
const $set: Record<string, any> = { updatedAt: now };
const $unset: Record<string, ''> = {
'metadata.sourcePolicy': '',
'metadata.sourceProfileRef': '',
'metadata.sourceProfileName': '',
};
if (sourceBindings.length > 0) {
$set['metadata.sourceBindings'] = sourceBindings;
$set['metadata.lastResolvedAt'] = now;
} else if (existingSourceBindings.length === 0) {
$unset['metadata.sourceBindings'] = '';
}
if (existingSourceBindings.length === 0 && legacyPolicyBindings.length === 0 && legacySourceProfileRef) {
$unset['route.security'] = '';
}
const query = (routeDoc as any)._id
? { _id: (routeDoc as any)._id }
: { id: (routeDoc as any).id };
await routeCollection.updateOne(query, { $set, $unset });
migrated++;
}
ctx.log.log(
'info',
`convert-route-access-metadata-to-source-bindings: migrated ${migrated}/${inspected} route(s)`,
);
}
async function backfillRemoteIngressHubSettings(ctx: {
mongo?: { collection: (name: string) => any };
log: { log: (level: 'info', message: string) => void };
}, options: IDcRouterMigrationOptions): Promise<void> {
const collection = ctx.mongo!.collection('RemoteIngressHubSettingsDoc');
const seed = options.remoteIngressHubSettings || {};
const now = Date.now();
const doc = await collection.findOne({ settingsId: 'remote-ingress-hub-settings' });
if (!doc) {
await collection.insertOne({
settingsId: 'remote-ingress-hub-settings',
enabled: seed.enabled ?? false,
tunnelPort: seed.tunnelPort ?? 8443,
hubDomain: seed.hubDomain || '',
performance: seed.performance || undefined,
updatedAt: now,
updatedBy: 'migration',
});
ctx.log.log('info', 'backfill-remote-ingress-hub-settings: inserted singleton settings document');
return;
}
const $set: Record<string, any> = {};
if ((doc as any).enabled === undefined) {
$set.enabled = seed.enabled ?? false;
}
if ((doc as any).tunnelPort === undefined) {
$set.tunnelPort = seed.tunnelPort ?? 8443;
}
if ((doc as any).hubDomain === undefined && seed.hubDomain) {
$set.hubDomain = seed.hubDomain;
}
if ((doc as any).performance === undefined && seed.performance) {
$set.performance = seed.performance;
}
if (Object.keys($set).length === 0) {
ctx.log.log('info', 'backfill-remote-ingress-hub-settings: no changes needed');
return;
}
$set.updatedAt = now;
$set.updatedBy = (doc as any).updatedBy || 'migration';
await collection.updateOne(
(doc as any)._id ? { _id: (doc as any)._id } : { settingsId: 'remote-ingress-hub-settings' },
{ $set },
);
ctx.log.log('info', `backfill-remote-ingress-hub-settings: set ${Object.keys($set).length - 2} missing field(s)`);
}
async function backfillEmailServerSettings(ctx: {
mongo?: { collection: (name: string) => any };
log: { log: (level: 'info', message: string) => void };
}, options: IDcRouterMigrationOptions): Promise<void> {
const collection = ctx.mongo!.collection('EmailServerSettingsDoc');
const seed = options.emailServerSettings || {};
const now = Date.now();
const doc = await collection.findOne({ settingsId: 'email-server-settings' });
if (!doc) {
await collection.insertOne({
settingsId: 'email-server-settings',
enabled: seed.enabled ?? Boolean(seed.emailConfig),
emailConfig: seed.emailConfig || undefined,
emailPortConfig: seed.emailPortConfig || undefined,
updatedAt: now,
updatedBy: 'migration',
});
ctx.log.log('info', 'backfill-email-server-settings: inserted singleton settings document');
return;
}
const $set: Record<string, any> = {};
if ((doc as any).enabled === undefined) {
$set.enabled = seed.enabled ?? Boolean(seed.emailConfig);
}
if ((doc as any).emailConfig === undefined && seed.emailConfig) {
$set.emailConfig = seed.emailConfig;
}
if ((doc as any).emailPortConfig === undefined && seed.emailPortConfig) {
$set.emailPortConfig = seed.emailPortConfig;
}
if (Object.keys($set).length === 0) {
ctx.log.log('info', 'backfill-email-server-settings: no changes needed');
return;
}
$set.updatedAt = now;
$set.updatedBy = (doc as any).updatedBy || 'migration';
await collection.updateOne(
(doc as any)._id ? { _id: (doc as any)._id } : { settingsId: 'email-server-settings' },
{ $set },
);
ctx.log.log('info', `backfill-email-server-settings: set ${Object.keys($set).length - 2} missing field(s)`);
}
/**
* Create a configured SmartMigration runner with all dcrouter migration steps registered.
*
* Call `.run()` on the returned instance at startup (after DcRouterDb is ready,
* before any service that reads migrated collections).
*
* @param db - The initialized SmartdataDb instance from DcRouterDb.getDb()
* @param targetVersion - The current app version (from commitinfo.version)
*/
export async function createMigrationRunner(
db: unknown,
targetVersion: string,
options: IDcRouterMigrationOptions = {},
): Promise<IMigrationRunner> {
const sm = await import('@push.rocks/smartmigration');
const migration = new sm.SmartMigration({
targetVersion,
db: db as any,
// Brand-new installs skip all migrations and stamp directly to the current version.
freshInstallVersion: targetVersion,
// dcrouter uses the package version as targetVersion; bridge releases without DB changes.
targetVersionStrategy: 'bridge',
});
// Register steps in execution order. Each step's .from() must match the
// previous step's .to() to form a contiguous chain.
migration
.step('rename-target-profile-host-to-ip')
.from('13.0.11').to('13.1.0')
.description('Rename ITargetProfileTarget.host → ip on all target profiles')
.up(async (ctx) => migrateTargetProfileTargetHosts(ctx))
.step('rename-domain-source-manual-to-dcrouter')
.from('13.1.0').to('13.8.1')
.description('Rename DomainDoc.source value from "manual" to "dcrouter"')
.up(async (ctx) => {
const collection = ctx.mongo!.collection('domaindoc');
const result = await collection.updateMany(
{ source: 'manual' },
{ $set: { source: 'dcrouter' } },
);
ctx.log.log(
'info',
`rename-domain-source-manual-to-dcrouter: migrated ${result.modifiedCount} domain(s)`,
);
})
.step('rename-record-source-manual-to-local')
.from('13.8.1').to('13.8.2')
.description('Rename DnsRecordDoc.source value from "manual" to "local"')
.up(async (ctx) => {
const collection = ctx.mongo!.collection('dnsrecorddoc');
const result = await collection.updateMany(
{ source: 'manual' },
{ $set: { source: 'local' } },
);
ctx.log.log(
'info',
`rename-record-source-manual-to-local: migrated ${result.modifiedCount} record(s)`,
);
})
.step('unify-routes-rename-collection')
.from('13.8.2').to('13.16.0')
.description('Rename StoredRouteDoc → RouteDoc, add origin field, drop RouteOverrideDoc')
.up(async (ctx) => {
const db = ctx.mongo!;
// 1. Rename StoredRouteDoc → RouteDoc (smartdata uses exact class names)
const collections = await db.listCollections({ name: 'StoredRouteDoc' }).toArray();
if (collections.length > 0) {
await db.renameCollection('StoredRouteDoc', 'RouteDoc');
ctx.log.log('info', 'Renamed StoredRouteDoc → RouteDoc');
}
// 2. Set origin='api' on all migrated docs (they were API-created)
const routeCol = db.collection('RouteDoc');
const result = await routeCol.updateMany(
{ origin: { $exists: false } },
{ $set: { origin: 'api' } },
);
ctx.log.log('info', `Set origin='api' on ${result.modifiedCount} migrated route(s)`);
// 3. Drop RouteOverrideDoc collection
const overrideCollections = await db.listCollections({ name: 'RouteOverrideDoc' }).toArray();
if (overrideCollections.length > 0) {
await db.collection('RouteOverrideDoc').drop();
ctx.log.log('info', 'Dropped RouteOverrideDoc collection');
}
})
.step('repair-target-profile-ip-migration')
.from('13.16.0').to('13.17.4')
.description('Repair TargetProfileDoc.targets host→ip migration for already-upgraded installs')
.up(async (ctx) => {
await migrateTargetProfileTargetHosts(ctx);
})
.step('backfill-system-route-keys')
.from('13.17.4').to('13.18.0')
.description('Backfill RouteDoc.systemKey for persisted config/email/dns routes')
.up(async (ctx) => {
await backfillSystemRouteKeys(ctx);
})
.step('rematerialize-source-profile-route-security')
.from('13.18.0').to('13.40.2')
.description('Replace stale route security with resolved source profile security')
.up(async (ctx) => {
await rematerializeSourceProfileRouteSecurity(ctx);
})
.step('seed-missing-default-source-profiles')
.from('13.40.2').to('13.42.0')
.description('Seed missing default source profiles for source-policy presets')
.up(async (ctx) => {
await seedMissingDefaultSourceProfiles(ctx);
})
.step('convert-route-access-metadata-to-source-bindings')
.from('13.42.0').to('13.43.2')
.description('Convert route sourceProfileRef/sourcePolicy metadata to canonical sourceBindings')
.up(async (ctx) => {
await convertRouteAccessMetadataToSourceBindings(ctx);
})
.step('backfill-remote-ingress-hub-settings-current')
.from('13.43.2').to(remoteIngressHubSettingsMigrationBaseVersion)
.description('Backfill RemoteIngress hub singleton settings for current dcrouter 13.43.5 installs')
.up(async (ctx) => {
await backfillRemoteIngressHubSettings(ctx, options);
await backfillEmailServerSettings(ctx, options);
});
if (compareSemver(targetVersion, remoteIngressHubSettingsMigrationBaseVersion) > 0) {
migration
.step('backfill-remote-ingress-hub-settings')
.from(remoteIngressHubSettingsMigrationBaseVersion).to(targetVersion)
.description('Backfill DB-backed singleton runtime settings from legacy bootstrap config')
.up(async (ctx) => {
await backfillRemoteIngressHubSettings(ctx, options);
await backfillEmailServerSettings(ctx, options);
});
}
return migration;
}