Juergen Kunz
820f84ee61
- Fix ES client v8+ API: use document/doc instead of body for index/update operations - Add type assertions (as any) for ES client ILM, template, and search APIs - Fix strict null checks with proper undefined handling (nullish coalescing) - Fix MetricsCollector interface to match required method signatures - Fix Logger.error signature compatibility in plugins - Resolve TermsQuery type index signature conflict - Remove sourceMap from tsconfig (handled by tsbuild with inlineSourceMap)
927 lines
25 KiB
TypeScript
927 lines
25 KiB
TypeScript
/**
|
|
* Schema Manager
|
|
*
|
|
* Index mapping management, templates, and migrations
|
|
*/
|
|
|
|
import { ElasticsearchConnectionManager } from '../../core/connection/connection-manager.js';
|
|
import { Logger, defaultLogger } from '../../core/observability/logger.js';
|
|
import { MetricsCollector, defaultMetricsCollector } from '../../core/observability/metrics.js';
|
|
import type {
|
|
IndexSchema,
|
|
IndexSettings,
|
|
IndexMapping,
|
|
FieldDefinition,
|
|
SchemaMigration,
|
|
MigrationHistoryEntry,
|
|
SchemaManagerConfig,
|
|
SchemaValidationResult,
|
|
SchemaDiff,
|
|
IndexTemplate,
|
|
ComponentTemplate,
|
|
SchemaManagerStats,
|
|
} from './types.js';
|
|
|
|
/**
|
|
* Default configuration
|
|
*/
|
|
const DEFAULT_CONFIG: Required<SchemaManagerConfig> = {
|
|
historyIndex: '.schema_migrations',
|
|
dryRun: false,
|
|
strict: false,
|
|
timeout: 60000, // 1 minute
|
|
enableLogging: true,
|
|
enableMetrics: true,
|
|
validateBeforeApply: true,
|
|
};
|
|
|
|
/**
|
|
* Schema Manager
|
|
*/
|
|
export class SchemaManager {
|
|
private config: Required<SchemaManagerConfig>;
|
|
private stats: SchemaManagerStats;
|
|
private logger: Logger;
|
|
private metrics: MetricsCollector;
|
|
|
|
constructor(config: SchemaManagerConfig = {}) {
|
|
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
this.logger = defaultLogger;
|
|
this.metrics = defaultMetricsCollector;
|
|
|
|
this.stats = {
|
|
totalMigrations: 0,
|
|
successfulMigrations: 0,
|
|
failedMigrations: 0,
|
|
rolledBackMigrations: 0,
|
|
totalIndices: 0,
|
|
totalTemplates: 0,
|
|
avgMigrationDuration: 0,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Initialize schema manager
|
|
*/
|
|
async initialize(): Promise<void> {
|
|
await this.ensureHistoryIndex();
|
|
|
|
this.logger.info('SchemaManager initialized', {
|
|
historyIndex: this.config.historyIndex,
|
|
dryRun: this.config.dryRun,
|
|
});
|
|
}
|
|
|
|
// ============================================================================
|
|
// Index Management
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Create an index with schema
|
|
*/
|
|
async createIndex(schema: IndexSchema): Promise<void> {
|
|
const startTime = Date.now();
|
|
|
|
if (this.config.validateBeforeApply) {
|
|
const validation = this.validateSchema(schema);
|
|
if (!validation.valid) {
|
|
throw new Error(`Schema validation failed: ${validation.errors.map((e) => e.message).join(', ')}`);
|
|
}
|
|
}
|
|
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would create index', { index: schema.name });
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
const body: any = {
|
|
settings: schema.settings,
|
|
mappings: schema.mappings,
|
|
};
|
|
|
|
if (schema.aliases) {
|
|
body.aliases = schema.aliases;
|
|
}
|
|
|
|
await client.indices.create({
|
|
index: schema.name,
|
|
...body,
|
|
timeout: `${this.config.timeout}ms`,
|
|
});
|
|
|
|
this.stats.totalIndices++;
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Index created', {
|
|
index: schema.name,
|
|
version: schema.version,
|
|
duration: Date.now() - startTime,
|
|
});
|
|
}
|
|
|
|
if (this.config.enableMetrics) {
|
|
this.metrics.recordCounter('schema.indices.created', 1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update index mapping
|
|
*/
|
|
async updateMapping(index: string, mapping: Partial<IndexMapping>): Promise<void> {
|
|
const startTime = Date.now();
|
|
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would update mapping', { index });
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
await client.indices.putMapping({
|
|
index,
|
|
...mapping,
|
|
timeout: `${this.config.timeout}ms`,
|
|
} as any);
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Mapping updated', {
|
|
index,
|
|
duration: Date.now() - startTime,
|
|
});
|
|
}
|
|
|
|
if (this.config.enableMetrics) {
|
|
this.metrics.recordCounter('schema.mappings.updated', 1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update index settings
|
|
*/
|
|
async updateSettings(index: string, settings: Partial<IndexSettings>): Promise<void> {
|
|
const startTime = Date.now();
|
|
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would update settings', { index });
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
// Some settings require closing the index
|
|
const requiresClose = this.settingsRequireClose(settings);
|
|
|
|
if (requiresClose) {
|
|
await client.indices.close({ index });
|
|
}
|
|
|
|
try {
|
|
await client.indices.putSettings({
|
|
index,
|
|
settings,
|
|
timeout: `${this.config.timeout}ms`,
|
|
} as any);
|
|
} finally {
|
|
if (requiresClose) {
|
|
await client.indices.open({ index });
|
|
}
|
|
}
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Settings updated', {
|
|
index,
|
|
duration: Date.now() - startTime,
|
|
});
|
|
}
|
|
|
|
if (this.config.enableMetrics) {
|
|
this.metrics.recordCounter('schema.settings.updated', 1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get index schema
|
|
*/
|
|
async getSchema(index: string): Promise<IndexSchema | null> {
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
try {
|
|
const [mappingResult, settingsResult, aliasResult] = await Promise.all([
|
|
client.indices.getMapping({ index }),
|
|
client.indices.getSettings({ index }),
|
|
client.indices.getAlias({ index }),
|
|
]);
|
|
|
|
const indexData = mappingResult[index];
|
|
const settingsData = settingsResult[index];
|
|
const aliasData = aliasResult[index];
|
|
|
|
return {
|
|
name: index,
|
|
version: 0, // Version not stored in ES by default
|
|
settings: settingsData?.settings?.index as IndexSettings,
|
|
mappings: indexData?.mappings as IndexMapping,
|
|
aliases: aliasData?.aliases,
|
|
};
|
|
} catch (error: any) {
|
|
if (error.meta?.statusCode === 404) {
|
|
return null;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Delete an index
|
|
*/
|
|
async deleteIndex(index: string): Promise<void> {
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would delete index', { index });
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
await client.indices.delete({
|
|
index,
|
|
timeout: `${this.config.timeout}ms`,
|
|
});
|
|
|
|
this.stats.totalIndices--;
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Index deleted', { index });
|
|
}
|
|
|
|
if (this.config.enableMetrics) {
|
|
this.metrics.recordCounter('schema.indices.deleted', 1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if index exists
|
|
*/
|
|
async indexExists(index: string): Promise<boolean> {
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
return await client.indices.exists({ index });
|
|
}
|
|
|
|
// ============================================================================
|
|
// Migration Management
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Run migrations
|
|
*/
|
|
async migrate(migrations: SchemaMigration[]): Promise<MigrationHistoryEntry[]> {
|
|
const results: MigrationHistoryEntry[] = [];
|
|
|
|
// Get current migration state
|
|
const appliedVersions = await this.getAppliedMigrations();
|
|
const appliedSet = new Set(appliedVersions.map((m) => m.version));
|
|
|
|
// Sort migrations by version
|
|
const sortedMigrations = [...migrations].sort((a, b) => a.version - b.version);
|
|
|
|
// Filter pending migrations
|
|
const pendingMigrations = sortedMigrations.filter((m) => !appliedSet.has(m.version));
|
|
|
|
if (pendingMigrations.length === 0) {
|
|
this.logger.info('No pending migrations');
|
|
return results;
|
|
}
|
|
|
|
this.logger.info(`Running ${pendingMigrations.length} migrations`);
|
|
|
|
for (const migration of pendingMigrations) {
|
|
const result = await this.runMigration(migration);
|
|
results.push(result);
|
|
|
|
if (result.status === 'failed') {
|
|
this.logger.error('Migration failed, stopping', {
|
|
version: migration.version,
|
|
name: migration.name,
|
|
});
|
|
break;
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* Run a single migration
|
|
*/
|
|
private async runMigration(migration: SchemaMigration): Promise<MigrationHistoryEntry> {
|
|
const startTime = Date.now();
|
|
const entry: MigrationHistoryEntry = {
|
|
version: migration.version,
|
|
name: migration.name,
|
|
status: 'running',
|
|
startedAt: new Date(),
|
|
};
|
|
|
|
this.stats.totalMigrations++;
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Running migration', {
|
|
version: migration.version,
|
|
name: migration.name,
|
|
type: migration.type,
|
|
});
|
|
}
|
|
|
|
try {
|
|
await this.applyMigration(migration);
|
|
|
|
entry.status = 'completed';
|
|
entry.completedAt = new Date();
|
|
entry.duration = Date.now() - startTime;
|
|
|
|
this.stats.successfulMigrations++;
|
|
this.stats.lastMigrationTime = new Date();
|
|
this.updateAvgDuration(entry.duration);
|
|
|
|
// Record in history
|
|
await this.recordMigration(entry);
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Migration completed', {
|
|
version: migration.version,
|
|
name: migration.name,
|
|
duration: entry.duration,
|
|
});
|
|
}
|
|
|
|
if (this.config.enableMetrics) {
|
|
this.metrics.recordCounter('schema.migrations.success', 1);
|
|
this.metrics.recordHistogram('schema.migrations.duration', entry.duration);
|
|
}
|
|
} catch (error: any) {
|
|
entry.status = 'failed';
|
|
entry.completedAt = new Date();
|
|
entry.duration = Date.now() - startTime;
|
|
entry.error = error.message;
|
|
|
|
this.stats.failedMigrations++;
|
|
|
|
// Record failure in history
|
|
await this.recordMigration(entry);
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.error('Migration failed', {
|
|
version: migration.version,
|
|
name: migration.name,
|
|
error: error.message,
|
|
});
|
|
}
|
|
|
|
if (this.config.enableMetrics) {
|
|
this.metrics.recordCounter('schema.migrations.failed', 1);
|
|
}
|
|
}
|
|
|
|
return entry;
|
|
}
|
|
|
|
/**
|
|
* Apply migration changes
|
|
*/
|
|
private async applyMigration(migration: SchemaMigration): Promise<void> {
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would apply migration', {
|
|
version: migration.version,
|
|
name: migration.name,
|
|
});
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
switch (migration.type) {
|
|
case 'create':
|
|
await this.createIndex({
|
|
name: migration.index,
|
|
version: migration.version,
|
|
settings: migration.changes.settings,
|
|
mappings: migration.changes.mappings as IndexMapping,
|
|
});
|
|
break;
|
|
|
|
case 'update':
|
|
if (migration.changes.settings) {
|
|
await this.updateSettings(migration.index, migration.changes.settings);
|
|
}
|
|
if (migration.changes.mappings) {
|
|
await this.updateMapping(migration.index, migration.changes.mappings);
|
|
}
|
|
if (migration.changes.aliases) {
|
|
await this.updateAliases(migration.index, migration.changes.aliases);
|
|
}
|
|
break;
|
|
|
|
case 'delete':
|
|
await this.deleteIndex(migration.index);
|
|
break;
|
|
|
|
case 'reindex':
|
|
if (migration.changes.reindex) {
|
|
await client.reindex({
|
|
source: { index: migration.changes.reindex.source },
|
|
dest: { index: migration.changes.reindex.dest },
|
|
script: migration.changes.reindex.script
|
|
? { source: migration.changes.reindex.script }
|
|
: undefined,
|
|
timeout: `${this.config.timeout}ms`,
|
|
});
|
|
}
|
|
break;
|
|
|
|
case 'alias':
|
|
if (migration.changes.aliases) {
|
|
await this.updateAliases(migration.index, migration.changes.aliases);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Rollback a migration
|
|
*/
|
|
async rollback(version: number, migrations: SchemaMigration[]): Promise<MigrationHistoryEntry> {
|
|
const migration = migrations.find((m) => m.version === version);
|
|
|
|
if (!migration) {
|
|
throw new Error(`Migration version ${version} not found`);
|
|
}
|
|
|
|
if (!migration.rollback) {
|
|
throw new Error(`Migration version ${version} has no rollback defined`);
|
|
}
|
|
|
|
const startTime = Date.now();
|
|
const entry: MigrationHistoryEntry = {
|
|
version,
|
|
name: `rollback_${migration.name}`,
|
|
status: 'running',
|
|
startedAt: new Date(),
|
|
};
|
|
|
|
try {
|
|
// Apply rollback changes
|
|
if (migration.rollback.settings) {
|
|
await this.updateSettings(migration.index, migration.rollback.settings);
|
|
}
|
|
if (migration.rollback.mappings) {
|
|
await this.updateMapping(migration.index, migration.rollback.mappings);
|
|
}
|
|
|
|
entry.status = 'rolled_back';
|
|
entry.completedAt = new Date();
|
|
entry.duration = Date.now() - startTime;
|
|
|
|
this.stats.rolledBackMigrations++;
|
|
|
|
// Update history
|
|
await this.recordMigration(entry);
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Migration rolled back', {
|
|
version,
|
|
name: migration.name,
|
|
duration: entry.duration,
|
|
});
|
|
}
|
|
|
|
if (this.config.enableMetrics) {
|
|
this.metrics.recordCounter('schema.migrations.rollback', 1);
|
|
}
|
|
} catch (error: any) {
|
|
entry.status = 'failed';
|
|
entry.error = error.message;
|
|
|
|
this.logger.error('Rollback failed', {
|
|
version,
|
|
name: migration.name,
|
|
error: error.message,
|
|
});
|
|
}
|
|
|
|
return entry;
|
|
}
|
|
|
|
/**
|
|
* Get applied migrations
|
|
*/
|
|
async getAppliedMigrations(): Promise<MigrationHistoryEntry[]> {
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
try {
|
|
const result = await client.search({
|
|
index: this.config.historyIndex,
|
|
size: 1000,
|
|
sort: [{ version: 'asc' }],
|
|
});
|
|
|
|
return result.hits.hits.map((hit) => hit._source as MigrationHistoryEntry);
|
|
} catch (error: any) {
|
|
if (error.meta?.statusCode === 404) {
|
|
return [];
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// Template Management
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Create or update index template
|
|
*/
|
|
async putTemplate(template: IndexTemplate): Promise<void> {
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would put template', { name: template.name });
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
await client.indices.putIndexTemplate({
|
|
name: template.name,
|
|
index_patterns: template.index_patterns,
|
|
priority: template.priority,
|
|
version: template.version,
|
|
composed_of: template.composed_of,
|
|
template: template.template,
|
|
data_stream: template.data_stream,
|
|
_meta: template._meta,
|
|
} as any);
|
|
|
|
this.stats.totalTemplates++;
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Template created/updated', { name: template.name });
|
|
}
|
|
|
|
if (this.config.enableMetrics) {
|
|
this.metrics.recordCounter('schema.templates.created', 1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get index template
|
|
*/
|
|
async getTemplate(name: string): Promise<IndexTemplate | null> {
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
try {
|
|
const result = await client.indices.getIndexTemplate({ name });
|
|
const template = result.index_templates[0];
|
|
|
|
if (!template) return null;
|
|
|
|
return {
|
|
name: template.name,
|
|
index_patterns: template.index_template.index_patterns as string[],
|
|
priority: template.index_template.priority,
|
|
version: template.index_template.version,
|
|
composed_of: template.index_template.composed_of,
|
|
template: template.index_template.template as IndexTemplate['template'],
|
|
data_stream: template.index_template.data_stream,
|
|
_meta: template.index_template._meta,
|
|
};
|
|
} catch (error: any) {
|
|
if (error.meta?.statusCode === 404) {
|
|
return null;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Delete index template
|
|
*/
|
|
async deleteTemplate(name: string): Promise<void> {
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would delete template', { name });
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
await client.indices.deleteIndexTemplate({ name });
|
|
|
|
this.stats.totalTemplates--;
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Template deleted', { name });
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create or update component template
|
|
*/
|
|
async putComponentTemplate(template: ComponentTemplate): Promise<void> {
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would put component template', { name: template.name });
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
await client.cluster.putComponentTemplate({
|
|
name: template.name,
|
|
template: template.template,
|
|
version: template.version,
|
|
_meta: template._meta,
|
|
} as any);
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Component template created/updated', { name: template.name });
|
|
}
|
|
}
|
|
|
|
// ============================================================================
|
|
// Alias Management
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Update aliases
|
|
*/
|
|
async updateAliases(
|
|
index: string,
|
|
aliases: { add?: Record<string, unknown>; remove?: string[] }
|
|
): Promise<void> {
|
|
if (this.config.dryRun) {
|
|
this.logger.info('[DRY RUN] Would update aliases', { index });
|
|
return;
|
|
}
|
|
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
const actions: any[] = [];
|
|
|
|
if (aliases.add) {
|
|
for (const [alias, config] of Object.entries(aliases.add)) {
|
|
const configObj = config as Record<string, unknown>;
|
|
actions.push({ add: { index, alias, ...configObj } });
|
|
}
|
|
}
|
|
|
|
if (aliases.remove) {
|
|
for (const alias of aliases.remove) {
|
|
actions.push({ remove: { index, alias } });
|
|
}
|
|
}
|
|
|
|
if (actions.length > 0) {
|
|
await client.indices.updateAliases({ actions });
|
|
}
|
|
|
|
if (this.config.enableLogging) {
|
|
this.logger.info('Aliases updated', { index, actions: actions.length });
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Add alias
|
|
*/
|
|
async addAlias(index: string, alias: string, config: Record<string, unknown> = {}): Promise<void> {
|
|
await this.updateAliases(index, { add: { [alias]: config } });
|
|
}
|
|
|
|
/**
|
|
* Remove alias
|
|
*/
|
|
async removeAlias(index: string, alias: string): Promise<void> {
|
|
await this.updateAliases(index, { remove: [alias] });
|
|
}
|
|
|
|
// ============================================================================
|
|
// Schema Validation and Comparison
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Validate schema
|
|
*/
|
|
validateSchema(schema: IndexSchema): SchemaValidationResult {
|
|
const errors: SchemaValidationResult['errors'] = [];
|
|
const warnings: SchemaValidationResult['warnings'] = [];
|
|
|
|
// Validate index name
|
|
if (!schema.name || schema.name.length === 0) {
|
|
errors.push({ field: 'name', message: 'Index name is required', severity: 'error' });
|
|
}
|
|
|
|
if (schema.name && schema.name.startsWith('_')) {
|
|
errors.push({ field: 'name', message: 'Index name cannot start with _', severity: 'error' });
|
|
}
|
|
|
|
// Validate mappings
|
|
if (!schema.mappings || !schema.mappings.properties) {
|
|
errors.push({ field: 'mappings', message: 'Mappings with properties are required', severity: 'error' });
|
|
} else {
|
|
this.validateFields(schema.mappings.properties, '', errors, warnings);
|
|
}
|
|
|
|
// Validate settings
|
|
if (schema.settings) {
|
|
if (schema.settings.number_of_shards !== undefined && schema.settings.number_of_shards < 1) {
|
|
errors.push({ field: 'settings.number_of_shards', message: 'Must be at least 1', severity: 'error' });
|
|
}
|
|
|
|
if (schema.settings.number_of_replicas !== undefined && schema.settings.number_of_replicas < 0) {
|
|
errors.push({ field: 'settings.number_of_replicas', message: 'Cannot be negative', severity: 'error' });
|
|
}
|
|
}
|
|
|
|
return {
|
|
valid: errors.length === 0,
|
|
errors,
|
|
warnings,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Compare two schemas
|
|
*/
|
|
diffSchemas(oldSchema: IndexSchema, newSchema: IndexSchema): SchemaDiff {
|
|
const added: string[] = [];
|
|
const removed: string[] = [];
|
|
const modified: SchemaDiff['modified'] = [];
|
|
const breakingChanges: string[] = [];
|
|
|
|
// Compare fields
|
|
const oldFields = this.flattenFields(oldSchema.mappings?.properties || {});
|
|
const newFields = this.flattenFields(newSchema.mappings?.properties || {});
|
|
|
|
for (const field of newFields.keys()) {
|
|
if (!oldFields.has(field)) {
|
|
added.push(field);
|
|
}
|
|
}
|
|
|
|
for (const field of oldFields.keys()) {
|
|
if (!newFields.has(field)) {
|
|
removed.push(field);
|
|
breakingChanges.push(`Removed field: ${field}`);
|
|
}
|
|
}
|
|
|
|
for (const [field, newDef] of newFields) {
|
|
const oldDef = oldFields.get(field);
|
|
if (oldDef && JSON.stringify(oldDef) !== JSON.stringify(newDef)) {
|
|
modified.push({ field, from: oldDef, to: newDef });
|
|
|
|
// Check for breaking changes
|
|
if (oldDef.type !== newDef.type) {
|
|
breakingChanges.push(`Field type changed: ${field} (${oldDef.type} -> ${newDef.type})`);
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
identical: added.length === 0 && removed.length === 0 && modified.length === 0,
|
|
added,
|
|
removed,
|
|
modified,
|
|
settingsChanged: JSON.stringify(oldSchema.settings) !== JSON.stringify(newSchema.settings),
|
|
aliasesChanged: JSON.stringify(oldSchema.aliases) !== JSON.stringify(newSchema.aliases),
|
|
breakingChanges,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Get statistics
|
|
*/
|
|
getStats(): SchemaManagerStats {
|
|
return { ...this.stats };
|
|
}
|
|
|
|
// ============================================================================
|
|
// Private Methods
|
|
// ============================================================================
|
|
|
|
/**
|
|
* Ensure history index exists
|
|
*/
|
|
private async ensureHistoryIndex(): Promise<void> {
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
const exists = await client.indices.exists({ index: this.config.historyIndex });
|
|
|
|
if (!exists) {
|
|
await client.indices.create({
|
|
index: this.config.historyIndex,
|
|
mappings: {
|
|
properties: {
|
|
version: { type: 'integer' },
|
|
name: { type: 'keyword' },
|
|
status: { type: 'keyword' },
|
|
startedAt: { type: 'date' },
|
|
completedAt: { type: 'date' },
|
|
duration: { type: 'long' },
|
|
error: { type: 'text' },
|
|
checksum: { type: 'keyword' },
|
|
},
|
|
},
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Record migration in history
|
|
*/
|
|
private async recordMigration(entry: MigrationHistoryEntry): Promise<void> {
|
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
|
|
|
await client.index({
|
|
index: this.config.historyIndex,
|
|
id: `migration-${entry.version}`,
|
|
document: entry,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Check if settings require closing the index
|
|
*/
|
|
private settingsRequireClose(settings: Partial<IndexSettings>): boolean {
|
|
const closeRequired = ['number_of_shards', 'codec', 'routing_partition_size'];
|
|
return Object.keys(settings).some((key) => closeRequired.includes(key));
|
|
}
|
|
|
|
/**
|
|
* Validate fields recursively
|
|
*/
|
|
private validateFields(
|
|
fields: Record<string, FieldDefinition>,
|
|
prefix: string,
|
|
errors: SchemaValidationResult['errors'],
|
|
warnings: SchemaValidationResult['warnings']
|
|
): void {
|
|
for (const [name, def] of Object.entries(fields)) {
|
|
const path = prefix ? `${prefix}.${name}` : name;
|
|
|
|
if (!def.type) {
|
|
errors.push({ field: path, message: 'Field type is required', severity: 'error' });
|
|
}
|
|
|
|
// Check for text fields without keyword sub-field
|
|
if (def.type === 'text' && !def.properties) {
|
|
warnings.push({
|
|
field: path,
|
|
message: 'Text field without keyword sub-field may limit aggregation capabilities',
|
|
});
|
|
}
|
|
|
|
// Validate nested properties
|
|
if (def.properties) {
|
|
this.validateFields(def.properties, path, errors, warnings);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Flatten nested fields
|
|
*/
|
|
private flattenFields(
|
|
fields: Record<string, FieldDefinition>,
|
|
prefix: string = ''
|
|
): Map<string, FieldDefinition> {
|
|
const result = new Map<string, FieldDefinition>();
|
|
|
|
for (const [name, def] of Object.entries(fields)) {
|
|
const path = prefix ? `${prefix}.${name}` : name;
|
|
result.set(path, def);
|
|
|
|
if (def.properties) {
|
|
const nested = this.flattenFields(def.properties, path);
|
|
for (const [nestedPath, nestedDef] of nested) {
|
|
result.set(nestedPath, nestedDef);
|
|
}
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Update average migration duration
|
|
*/
|
|
private updateAvgDuration(duration: number): void {
|
|
const total = this.stats.successfulMigrations + this.stats.failedMigrations;
|
|
this.stats.avgMigrationDuration =
|
|
(this.stats.avgMigrationDuration * (total - 1) + duration) / total;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create a schema manager
|
|
*/
|
|
export function createSchemaManager(config?: SchemaManagerConfig): SchemaManager {
|
|
return new SchemaManager(config);
|
|
}
|