/** * Schema Manager * * Index mapping management, templates, and migrations */ import { ElasticsearchConnectionManager } from '../../core/connection/connection-manager.js'; import { Logger, defaultLogger } from '../../core/observability/logger.js'; import { MetricsCollector, defaultMetricsCollector } from '../../core/observability/metrics.js'; import type { IndexSchema, IndexSettings, IndexMapping, FieldDefinition, SchemaMigration, MigrationHistoryEntry, MigrationStatus, SchemaManagerConfig, SchemaValidationResult, SchemaDiff, IndexTemplate, ComponentTemplate, SchemaManagerStats, } from './types.js'; /** * Default configuration */ const DEFAULT_CONFIG: Required = { historyIndex: '.schema_migrations', dryRun: false, strict: false, timeout: 60000, // 1 minute enableLogging: true, enableMetrics: true, validateBeforeApply: true, }; /** * Schema Manager */ export class SchemaManager { private config: Required; private stats: SchemaManagerStats; private logger: Logger; private metrics: MetricsCollector; constructor(config: SchemaManagerConfig = {}) { this.config = { ...DEFAULT_CONFIG, ...config }; this.logger = defaultLogger; this.metrics = defaultMetricsCollector; this.stats = { totalMigrations: 0, successfulMigrations: 0, failedMigrations: 0, rolledBackMigrations: 0, totalIndices: 0, totalTemplates: 0, avgMigrationDuration: 0, }; } /** * Initialize schema manager */ async initialize(): Promise { await this.ensureHistoryIndex(); this.logger.info('SchemaManager initialized', { historyIndex: this.config.historyIndex, dryRun: this.config.dryRun, }); } // ============================================================================ // Index Management // ============================================================================ /** * Create an index with schema */ async createIndex(schema: IndexSchema): Promise { const startTime = Date.now(); if (this.config.validateBeforeApply) { const validation = this.validateSchema(schema); if (!validation.valid) { throw new Error(`Schema validation failed: ${validation.errors.map((e) => e.message).join(', ')}`); } } if (this.config.dryRun) { this.logger.info('[DRY RUN] Would create index', { index: schema.name }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); const body: any = { settings: schema.settings, mappings: schema.mappings, }; if (schema.aliases) { body.aliases = schema.aliases; } await client.indices.create({ index: schema.name, ...body, timeout: `${this.config.timeout}ms`, }); this.stats.totalIndices++; if (this.config.enableLogging) { this.logger.info('Index created', { index: schema.name, version: schema.version, duration: Date.now() - startTime, }); } if (this.config.enableMetrics) { this.metrics.recordCounter('schema.indices.created', 1); } } /** * Update index mapping */ async updateMapping(index: string, mapping: Partial): Promise { const startTime = Date.now(); if (this.config.dryRun) { this.logger.info('[DRY RUN] Would update mapping', { index }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); await client.indices.putMapping({ index, ...mapping, timeout: `${this.config.timeout}ms`, }); if (this.config.enableLogging) { this.logger.info('Mapping updated', { index, duration: Date.now() - startTime, }); } if (this.config.enableMetrics) { this.metrics.recordCounter('schema.mappings.updated', 1); } } /** * Update index settings */ async updateSettings(index: string, settings: Partial): Promise { const startTime = Date.now(); if (this.config.dryRun) { this.logger.info('[DRY RUN] Would update settings', { index }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); // Some settings require closing the index const requiresClose = this.settingsRequireClose(settings); if (requiresClose) { await client.indices.close({ index }); } try { await client.indices.putSettings({ index, settings, timeout: `${this.config.timeout}ms`, }); } finally { if (requiresClose) { await client.indices.open({ index }); } } if (this.config.enableLogging) { this.logger.info('Settings updated', { index, duration: Date.now() - startTime, }); } if (this.config.enableMetrics) { this.metrics.recordCounter('schema.settings.updated', 1); } } /** * Get index schema */ async getSchema(index: string): Promise { const client = ElasticsearchConnectionManager.getInstance().getClient(); try { const [mappingResult, settingsResult, aliasResult] = await Promise.all([ client.indices.getMapping({ index }), client.indices.getSettings({ index }), client.indices.getAlias({ index }), ]); const indexData = mappingResult[index]; const settingsData = settingsResult[index]; const aliasData = aliasResult[index]; return { name: index, version: 0, // Version not stored in ES by default settings: settingsData?.settings?.index as IndexSettings, mappings: indexData?.mappings as IndexMapping, aliases: aliasData?.aliases, }; } catch (error: any) { if (error.meta?.statusCode === 404) { return null; } throw error; } } /** * Delete an index */ async deleteIndex(index: string): Promise { if (this.config.dryRun) { this.logger.info('[DRY RUN] Would delete index', { index }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); await client.indices.delete({ index, timeout: `${this.config.timeout}ms`, }); this.stats.totalIndices--; if (this.config.enableLogging) { this.logger.info('Index deleted', { index }); } if (this.config.enableMetrics) { this.metrics.recordCounter('schema.indices.deleted', 1); } } /** * Check if index exists */ async indexExists(index: string): Promise { const client = ElasticsearchConnectionManager.getInstance().getClient(); return await client.indices.exists({ index }); } // ============================================================================ // Migration Management // ============================================================================ /** * Run migrations */ async migrate(migrations: SchemaMigration[]): Promise { const results: MigrationHistoryEntry[] = []; // Get current migration state const appliedVersions = await this.getAppliedMigrations(); const appliedSet = new Set(appliedVersions.map((m) => m.version)); // Sort migrations by version const sortedMigrations = [...migrations].sort((a, b) => a.version - b.version); // Filter pending migrations const pendingMigrations = sortedMigrations.filter((m) => !appliedSet.has(m.version)); if (pendingMigrations.length === 0) { this.logger.info('No pending migrations'); return results; } this.logger.info(`Running ${pendingMigrations.length} migrations`); for (const migration of pendingMigrations) { const result = await this.runMigration(migration); results.push(result); if (result.status === 'failed') { this.logger.error('Migration failed, stopping', { version: migration.version, name: migration.name, }); break; } } return results; } /** * Run a single migration */ private async runMigration(migration: SchemaMigration): Promise { const startTime = Date.now(); const entry: MigrationHistoryEntry = { version: migration.version, name: migration.name, status: 'running', startedAt: new Date(), }; this.stats.totalMigrations++; if (this.config.enableLogging) { this.logger.info('Running migration', { version: migration.version, name: migration.name, type: migration.type, }); } try { await this.applyMigration(migration); entry.status = 'completed'; entry.completedAt = new Date(); entry.duration = Date.now() - startTime; this.stats.successfulMigrations++; this.stats.lastMigrationTime = new Date(); this.updateAvgDuration(entry.duration); // Record in history await this.recordMigration(entry); if (this.config.enableLogging) { this.logger.info('Migration completed', { version: migration.version, name: migration.name, duration: entry.duration, }); } if (this.config.enableMetrics) { this.metrics.recordCounter('schema.migrations.success', 1); this.metrics.recordHistogram('schema.migrations.duration', entry.duration); } } catch (error: any) { entry.status = 'failed'; entry.completedAt = new Date(); entry.duration = Date.now() - startTime; entry.error = error.message; this.stats.failedMigrations++; // Record failure in history await this.recordMigration(entry); if (this.config.enableLogging) { this.logger.error('Migration failed', { version: migration.version, name: migration.name, error: error.message, }); } if (this.config.enableMetrics) { this.metrics.recordCounter('schema.migrations.failed', 1); } } return entry; } /** * Apply migration changes */ private async applyMigration(migration: SchemaMigration): Promise { if (this.config.dryRun) { this.logger.info('[DRY RUN] Would apply migration', { version: migration.version, name: migration.name, }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); switch (migration.type) { case 'create': await this.createIndex({ name: migration.index, version: migration.version, settings: migration.changes.settings, mappings: migration.changes.mappings as IndexMapping, }); break; case 'update': if (migration.changes.settings) { await this.updateSettings(migration.index, migration.changes.settings); } if (migration.changes.mappings) { await this.updateMapping(migration.index, migration.changes.mappings); } if (migration.changes.aliases) { await this.updateAliases(migration.index, migration.changes.aliases); } break; case 'delete': await this.deleteIndex(migration.index); break; case 'reindex': if (migration.changes.reindex) { await client.reindex({ source: { index: migration.changes.reindex.source }, dest: { index: migration.changes.reindex.dest }, script: migration.changes.reindex.script ? { source: migration.changes.reindex.script } : undefined, timeout: `${this.config.timeout}ms`, }); } break; case 'alias': if (migration.changes.aliases) { await this.updateAliases(migration.index, migration.changes.aliases); } break; } } /** * Rollback a migration */ async rollback(version: number, migrations: SchemaMigration[]): Promise { const migration = migrations.find((m) => m.version === version); if (!migration) { throw new Error(`Migration version ${version} not found`); } if (!migration.rollback) { throw new Error(`Migration version ${version} has no rollback defined`); } const startTime = Date.now(); const entry: MigrationHistoryEntry = { version, name: `rollback_${migration.name}`, status: 'running', startedAt: new Date(), }; try { // Apply rollback changes if (migration.rollback.settings) { await this.updateSettings(migration.index, migration.rollback.settings); } if (migration.rollback.mappings) { await this.updateMapping(migration.index, migration.rollback.mappings); } entry.status = 'rolled_back'; entry.completedAt = new Date(); entry.duration = Date.now() - startTime; this.stats.rolledBackMigrations++; // Update history await this.recordMigration(entry); if (this.config.enableLogging) { this.logger.info('Migration rolled back', { version, name: migration.name, duration: entry.duration, }); } if (this.config.enableMetrics) { this.metrics.recordCounter('schema.migrations.rollback', 1); } } catch (error: any) { entry.status = 'failed'; entry.error = error.message; this.logger.error('Rollback failed', { version, name: migration.name, error: error.message, }); } return entry; } /** * Get applied migrations */ async getAppliedMigrations(): Promise { const client = ElasticsearchConnectionManager.getInstance().getClient(); try { const result = await client.search({ index: this.config.historyIndex, size: 1000, sort: [{ version: 'asc' }], }); return result.hits.hits.map((hit) => hit._source as MigrationHistoryEntry); } catch (error: any) { if (error.meta?.statusCode === 404) { return []; } throw error; } } // ============================================================================ // Template Management // ============================================================================ /** * Create or update index template */ async putTemplate(template: IndexTemplate): Promise { if (this.config.dryRun) { this.logger.info('[DRY RUN] Would put template', { name: template.name }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); await client.indices.putIndexTemplate({ name: template.name, index_patterns: template.index_patterns, priority: template.priority, version: template.version, composed_of: template.composed_of, template: template.template, data_stream: template.data_stream, _meta: template._meta, }); this.stats.totalTemplates++; if (this.config.enableLogging) { this.logger.info('Template created/updated', { name: template.name }); } if (this.config.enableMetrics) { this.metrics.recordCounter('schema.templates.created', 1); } } /** * Get index template */ async getTemplate(name: string): Promise { const client = ElasticsearchConnectionManager.getInstance().getClient(); try { const result = await client.indices.getIndexTemplate({ name }); const template = result.index_templates[0]; if (!template) return null; return { name: template.name, index_patterns: template.index_template.index_patterns, priority: template.index_template.priority, version: template.index_template.version, composed_of: template.index_template.composed_of, template: template.index_template.template, data_stream: template.index_template.data_stream, _meta: template.index_template._meta, }; } catch (error: any) { if (error.meta?.statusCode === 404) { return null; } throw error; } } /** * Delete index template */ async deleteTemplate(name: string): Promise { if (this.config.dryRun) { this.logger.info('[DRY RUN] Would delete template', { name }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); await client.indices.deleteIndexTemplate({ name }); this.stats.totalTemplates--; if (this.config.enableLogging) { this.logger.info('Template deleted', { name }); } } /** * Create or update component template */ async putComponentTemplate(template: ComponentTemplate): Promise { if (this.config.dryRun) { this.logger.info('[DRY RUN] Would put component template', { name: template.name }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); await client.cluster.putComponentTemplate({ name: template.name, template: template.template, version: template.version, _meta: template._meta, }); if (this.config.enableLogging) { this.logger.info('Component template created/updated', { name: template.name }); } } // ============================================================================ // Alias Management // ============================================================================ /** * Update aliases */ async updateAliases( index: string, aliases: { add?: Record; remove?: string[] } ): Promise { if (this.config.dryRun) { this.logger.info('[DRY RUN] Would update aliases', { index }); return; } const client = ElasticsearchConnectionManager.getInstance().getClient(); const actions: any[] = []; if (aliases.add) { for (const [alias, config] of Object.entries(aliases.add)) { actions.push({ add: { index, alias, ...config } }); } } if (aliases.remove) { for (const alias of aliases.remove) { actions.push({ remove: { index, alias } }); } } if (actions.length > 0) { await client.indices.updateAliases({ actions }); } if (this.config.enableLogging) { this.logger.info('Aliases updated', { index, actions: actions.length }); } } /** * Add alias */ async addAlias(index: string, alias: string, config: Record = {}): Promise { await this.updateAliases(index, { add: { [alias]: config } }); } /** * Remove alias */ async removeAlias(index: string, alias: string): Promise { await this.updateAliases(index, { remove: [alias] }); } // ============================================================================ // Schema Validation and Comparison // ============================================================================ /** * Validate schema */ validateSchema(schema: IndexSchema): SchemaValidationResult { const errors: SchemaValidationResult['errors'] = []; const warnings: SchemaValidationResult['warnings'] = []; // Validate index name if (!schema.name || schema.name.length === 0) { errors.push({ field: 'name', message: 'Index name is required', severity: 'error' }); } if (schema.name && schema.name.startsWith('_')) { errors.push({ field: 'name', message: 'Index name cannot start with _', severity: 'error' }); } // Validate mappings if (!schema.mappings || !schema.mappings.properties) { errors.push({ field: 'mappings', message: 'Mappings with properties are required', severity: 'error' }); } else { this.validateFields(schema.mappings.properties, '', errors, warnings); } // Validate settings if (schema.settings) { if (schema.settings.number_of_shards !== undefined && schema.settings.number_of_shards < 1) { errors.push({ field: 'settings.number_of_shards', message: 'Must be at least 1', severity: 'error' }); } if (schema.settings.number_of_replicas !== undefined && schema.settings.number_of_replicas < 0) { errors.push({ field: 'settings.number_of_replicas', message: 'Cannot be negative', severity: 'error' }); } } return { valid: errors.length === 0, errors, warnings, }; } /** * Compare two schemas */ diffSchemas(oldSchema: IndexSchema, newSchema: IndexSchema): SchemaDiff { const added: string[] = []; const removed: string[] = []; const modified: SchemaDiff['modified'] = []; const breakingChanges: string[] = []; // Compare fields const oldFields = this.flattenFields(oldSchema.mappings?.properties || {}); const newFields = this.flattenFields(newSchema.mappings?.properties || {}); for (const field of newFields.keys()) { if (!oldFields.has(field)) { added.push(field); } } for (const field of oldFields.keys()) { if (!newFields.has(field)) { removed.push(field); breakingChanges.push(`Removed field: ${field}`); } } for (const [field, newDef] of newFields) { const oldDef = oldFields.get(field); if (oldDef && JSON.stringify(oldDef) !== JSON.stringify(newDef)) { modified.push({ field, from: oldDef, to: newDef }); // Check for breaking changes if (oldDef.type !== newDef.type) { breakingChanges.push(`Field type changed: ${field} (${oldDef.type} -> ${newDef.type})`); } } } return { identical: added.length === 0 && removed.length === 0 && modified.length === 0, added, removed, modified, settingsChanged: JSON.stringify(oldSchema.settings) !== JSON.stringify(newSchema.settings), aliasesChanged: JSON.stringify(oldSchema.aliases) !== JSON.stringify(newSchema.aliases), breakingChanges, }; } /** * Get statistics */ getStats(): SchemaManagerStats { return { ...this.stats }; } // ============================================================================ // Private Methods // ============================================================================ /** * Ensure history index exists */ private async ensureHistoryIndex(): Promise { const client = ElasticsearchConnectionManager.getInstance().getClient(); const exists = await client.indices.exists({ index: this.config.historyIndex }); if (!exists) { await client.indices.create({ index: this.config.historyIndex, mappings: { properties: { version: { type: 'integer' }, name: { type: 'keyword' }, status: { type: 'keyword' }, startedAt: { type: 'date' }, completedAt: { type: 'date' }, duration: { type: 'long' }, error: { type: 'text' }, checksum: { type: 'keyword' }, }, }, }); } } /** * Record migration in history */ private async recordMigration(entry: MigrationHistoryEntry): Promise { const client = ElasticsearchConnectionManager.getInstance().getClient(); await client.index({ index: this.config.historyIndex, id: `migration-${entry.version}`, document: entry, }); } /** * Check if settings require closing the index */ private settingsRequireClose(settings: Partial): boolean { const closeRequired = ['number_of_shards', 'codec', 'routing_partition_size']; return Object.keys(settings).some((key) => closeRequired.includes(key)); } /** * Validate fields recursively */ private validateFields( fields: Record, prefix: string, errors: SchemaValidationResult['errors'], warnings: SchemaValidationResult['warnings'] ): void { for (const [name, def] of Object.entries(fields)) { const path = prefix ? `${prefix}.${name}` : name; if (!def.type) { errors.push({ field: path, message: 'Field type is required', severity: 'error' }); } // Check for text fields without keyword sub-field if (def.type === 'text' && !def.properties) { warnings.push({ field: path, message: 'Text field without keyword sub-field may limit aggregation capabilities', }); } // Validate nested properties if (def.properties) { this.validateFields(def.properties, path, errors, warnings); } } } /** * Flatten nested fields */ private flattenFields( fields: Record, prefix: string = '' ): Map { const result = new Map(); for (const [name, def] of Object.entries(fields)) { const path = prefix ? `${prefix}.${name}` : name; result.set(path, def); if (def.properties) { const nested = this.flattenFields(def.properties, path); for (const [nestedPath, nestedDef] of nested) { result.set(nestedPath, nestedDef); } } } return result; } /** * Update average migration duration */ private updateAvgDuration(duration: number): void { const total = this.stats.successfulMigrations + this.stats.failedMigrations; this.stats.avgMigrationDuration = (this.stats.avgMigrationDuration * (total - 1) + duration) / total; } } /** * Create a schema manager */ export function createSchemaManager(config?: SchemaManagerConfig): SchemaManager { return new SchemaManager(config); }