diff --git a/changelog.md b/changelog.md index 6f5d10b..8a233f2 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,12 @@ # Changelog +## 2025-11-29 - 3.1.0 - feat(schema-manager) +Add Schema Management module and expose it in public API; update README to mark Phase 3 complete and move priorities to Phase 4 + +- Add ts/domain/schema/index.ts to export SchemaManager and related types +- Re-export SchemaManager and schema types from top-level ts/index.ts so schema APIs are part of the public surface +- Update README hints: mark Phase 3 Advanced Features complete (Transaction Support and Schema Management), add schema/transaction example references, and update Next Priorities to Phase 4 (Comprehensive Test Suite, Migration Guide, README Update) + ## 2025-11-29 - 3.0.0 - BREAKING CHANGE(core) Refactor to v3: introduce modular core/domain architecture, plugin system, observability and strict TypeScript configuration; remove legacy classes diff --git a/readme.hints.md b/readme.hints.md index 6b0443c..d63bac1 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -42,7 +42,7 @@ - Cache hit/miss statistics - Example at `ts/examples/kv/kv-store-example.ts` -**Phase 3: Advanced Features** +**Phase 3: Advanced Features (100%)** - Complete! - **Plugin Architecture (100%)** - Extensible request/response middleware: - Plugin lifecycle hooks (beforeRequest, afterResponse, onError) - Plugin priority and execution ordering @@ -58,6 +58,30 @@ - Shared context between plugins - Timeout protection for plugin hooks - Example at `ts/examples/plugins/plugin-example.ts` +- **Transaction Support (100%)** - ACID-like distributed transactions: + - Optimistic concurrency control (seq_no/primary_term) + - Automatic rollback with compensation operations + - Savepoints for partial rollback + - Conflict detection and configurable resolution (retry, abort, skip, force) + - Multi-document atomic operations + - Isolation levels (read_uncommitted, read_committed, repeatable_read) + - Transaction callbacks (onBegin, onCommit, onRollback, onConflict) + - Transaction statistics and monitoring + - Timeout with automatic cleanup + - Example at `ts/examples/transactions/transaction-example.ts` +- **Schema Management (100%)** - Index mapping management and migrations: + - Index creation with mappings, settings, and aliases + - Schema validation before apply + - Versioned migrations with history tracking + - Schema diff and comparison (added/removed/modified fields) + - Breaking change detection + - Index templates (composable and legacy) + - Component templates + - Alias management with filters + - Dynamic settings updates + - Migration rollback support + - Dry run mode for testing + - Example at `ts/examples/schema/schema-example.ts` ### Query Builder Usage @@ -81,11 +105,10 @@ const stats = await createQuery('products') .execute(); ``` -### Next Priorities -1. **Transaction Support** (Phase 3) - Distributed transactions across multiple documents -2. **Schema Management** (Phase 3) - Index mapping management and migrations -3. **Comprehensive Test Suite** (Phase 4) - Full test coverage -4. **Migration Guide** (Phase 4) - Guide from v2 to v3 +### Next Priorities (Phase 4) +1. **Comprehensive Test Suite** - Full test coverage for all modules +2. **Migration Guide** - Guide from v2 to v3 API +3. **README Update** - Update main README with new v3 API documentation ### Structure - Single `ts/` directory (no parallel structures) diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 3d624e6..8adbfdc 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@apiclient.xyz/elasticsearch', - version: '3.0.0', + version: '3.1.0', description: 'log to elasticsearch in a kibana compatible format' } diff --git a/ts/domain/schema/index.ts b/ts/domain/schema/index.ts new file mode 100644 index 0000000..141466a --- /dev/null +++ b/ts/domain/schema/index.ts @@ -0,0 +1,26 @@ +/** + * Schema Management Module + * + * Index mapping management, templates, and migrations + */ + +// Main classes +export { SchemaManager, createSchemaManager } from './schema-manager.js'; + +// Types +export type { + FieldType, + FieldDefinition, + IndexSettings, + IndexMapping, + IndexSchema, + SchemaMigration, + MigrationStatus, + MigrationHistoryEntry, + SchemaManagerConfig, + SchemaValidationResult, + SchemaDiff, + IndexTemplate, + ComponentTemplate, + SchemaManagerStats, +} from './types.js'; diff --git a/ts/domain/schema/schema-manager.ts b/ts/domain/schema/schema-manager.ts new file mode 100644 index 0000000..46cc942 --- /dev/null +++ b/ts/domain/schema/schema-manager.ts @@ -0,0 +1,926 @@ +/** + * Schema Manager + * + * Index mapping management, templates, and migrations + */ + +import { ElasticsearchConnectionManager } from '../../core/connection/connection-manager.js'; +import { Logger, defaultLogger } from '../../core/observability/logger.js'; +import { MetricsCollector, defaultMetricsCollector } from '../../core/observability/metrics.js'; +import type { + IndexSchema, + IndexSettings, + IndexMapping, + FieldDefinition, + SchemaMigration, + MigrationHistoryEntry, + MigrationStatus, + SchemaManagerConfig, + SchemaValidationResult, + SchemaDiff, + IndexTemplate, + ComponentTemplate, + SchemaManagerStats, +} from './types.js'; + +/** + * Default configuration + */ +const DEFAULT_CONFIG: Required = { + historyIndex: '.schema_migrations', + dryRun: false, + strict: false, + timeout: 60000, // 1 minute + enableLogging: true, + enableMetrics: true, + validateBeforeApply: true, +}; + +/** + * Schema Manager + */ +export class SchemaManager { + private config: Required; + private stats: SchemaManagerStats; + private logger: Logger; + private metrics: MetricsCollector; + + constructor(config: SchemaManagerConfig = {}) { + this.config = { ...DEFAULT_CONFIG, ...config }; + this.logger = defaultLogger; + this.metrics = defaultMetricsCollector; + + this.stats = { + totalMigrations: 0, + successfulMigrations: 0, + failedMigrations: 0, + rolledBackMigrations: 0, + totalIndices: 0, + totalTemplates: 0, + avgMigrationDuration: 0, + }; + } + + /** + * Initialize schema manager + */ + async initialize(): Promise { + await this.ensureHistoryIndex(); + + this.logger.info('SchemaManager initialized', { + historyIndex: this.config.historyIndex, + dryRun: this.config.dryRun, + }); + } + + // ============================================================================ + // Index Management + // ============================================================================ + + /** + * Create an index with schema + */ + async createIndex(schema: IndexSchema): Promise { + const startTime = Date.now(); + + if (this.config.validateBeforeApply) { + const validation = this.validateSchema(schema); + if (!validation.valid) { + throw new Error(`Schema validation failed: ${validation.errors.map((e) => e.message).join(', ')}`); + } + } + + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would create index', { index: schema.name }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + const body: any = { + settings: schema.settings, + mappings: schema.mappings, + }; + + if (schema.aliases) { + body.aliases = schema.aliases; + } + + await client.indices.create({ + index: schema.name, + ...body, + timeout: `${this.config.timeout}ms`, + }); + + this.stats.totalIndices++; + + if (this.config.enableLogging) { + this.logger.info('Index created', { + index: schema.name, + version: schema.version, + duration: Date.now() - startTime, + }); + } + + if (this.config.enableMetrics) { + this.metrics.recordCounter('schema.indices.created', 1); + } + } + + /** + * Update index mapping + */ + async updateMapping(index: string, mapping: Partial): Promise { + const startTime = Date.now(); + + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would update mapping', { index }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + await client.indices.putMapping({ + index, + ...mapping, + timeout: `${this.config.timeout}ms`, + }); + + if (this.config.enableLogging) { + this.logger.info('Mapping updated', { + index, + duration: Date.now() - startTime, + }); + } + + if (this.config.enableMetrics) { + this.metrics.recordCounter('schema.mappings.updated', 1); + } + } + + /** + * Update index settings + */ + async updateSettings(index: string, settings: Partial): Promise { + const startTime = Date.now(); + + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would update settings', { index }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + // Some settings require closing the index + const requiresClose = this.settingsRequireClose(settings); + + if (requiresClose) { + await client.indices.close({ index }); + } + + try { + await client.indices.putSettings({ + index, + settings, + timeout: `${this.config.timeout}ms`, + }); + } finally { + if (requiresClose) { + await client.indices.open({ index }); + } + } + + if (this.config.enableLogging) { + this.logger.info('Settings updated', { + index, + duration: Date.now() - startTime, + }); + } + + if (this.config.enableMetrics) { + this.metrics.recordCounter('schema.settings.updated', 1); + } + } + + /** + * Get index schema + */ + async getSchema(index: string): Promise { + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + try { + const [mappingResult, settingsResult, aliasResult] = await Promise.all([ + client.indices.getMapping({ index }), + client.indices.getSettings({ index }), + client.indices.getAlias({ index }), + ]); + + const indexData = mappingResult[index]; + const settingsData = settingsResult[index]; + const aliasData = aliasResult[index]; + + return { + name: index, + version: 0, // Version not stored in ES by default + settings: settingsData?.settings?.index as IndexSettings, + mappings: indexData?.mappings as IndexMapping, + aliases: aliasData?.aliases, + }; + } catch (error: any) { + if (error.meta?.statusCode === 404) { + return null; + } + throw error; + } + } + + /** + * Delete an index + */ + async deleteIndex(index: string): Promise { + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would delete index', { index }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + await client.indices.delete({ + index, + timeout: `${this.config.timeout}ms`, + }); + + this.stats.totalIndices--; + + if (this.config.enableLogging) { + this.logger.info('Index deleted', { index }); + } + + if (this.config.enableMetrics) { + this.metrics.recordCounter('schema.indices.deleted', 1); + } + } + + /** + * Check if index exists + */ + async indexExists(index: string): Promise { + const client = ElasticsearchConnectionManager.getInstance().getClient(); + return await client.indices.exists({ index }); + } + + // ============================================================================ + // Migration Management + // ============================================================================ + + /** + * Run migrations + */ + async migrate(migrations: SchemaMigration[]): Promise { + const results: MigrationHistoryEntry[] = []; + + // Get current migration state + const appliedVersions = await this.getAppliedMigrations(); + const appliedSet = new Set(appliedVersions.map((m) => m.version)); + + // Sort migrations by version + const sortedMigrations = [...migrations].sort((a, b) => a.version - b.version); + + // Filter pending migrations + const pendingMigrations = sortedMigrations.filter((m) => !appliedSet.has(m.version)); + + if (pendingMigrations.length === 0) { + this.logger.info('No pending migrations'); + return results; + } + + this.logger.info(`Running ${pendingMigrations.length} migrations`); + + for (const migration of pendingMigrations) { + const result = await this.runMigration(migration); + results.push(result); + + if (result.status === 'failed') { + this.logger.error('Migration failed, stopping', { + version: migration.version, + name: migration.name, + }); + break; + } + } + + return results; + } + + /** + * Run a single migration + */ + private async runMigration(migration: SchemaMigration): Promise { + const startTime = Date.now(); + const entry: MigrationHistoryEntry = { + version: migration.version, + name: migration.name, + status: 'running', + startedAt: new Date(), + }; + + this.stats.totalMigrations++; + + if (this.config.enableLogging) { + this.logger.info('Running migration', { + version: migration.version, + name: migration.name, + type: migration.type, + }); + } + + try { + await this.applyMigration(migration); + + entry.status = 'completed'; + entry.completedAt = new Date(); + entry.duration = Date.now() - startTime; + + this.stats.successfulMigrations++; + this.stats.lastMigrationTime = new Date(); + this.updateAvgDuration(entry.duration); + + // Record in history + await this.recordMigration(entry); + + if (this.config.enableLogging) { + this.logger.info('Migration completed', { + version: migration.version, + name: migration.name, + duration: entry.duration, + }); + } + + if (this.config.enableMetrics) { + this.metrics.recordCounter('schema.migrations.success', 1); + this.metrics.recordHistogram('schema.migrations.duration', entry.duration); + } + } catch (error: any) { + entry.status = 'failed'; + entry.completedAt = new Date(); + entry.duration = Date.now() - startTime; + entry.error = error.message; + + this.stats.failedMigrations++; + + // Record failure in history + await this.recordMigration(entry); + + if (this.config.enableLogging) { + this.logger.error('Migration failed', { + version: migration.version, + name: migration.name, + error: error.message, + }); + } + + if (this.config.enableMetrics) { + this.metrics.recordCounter('schema.migrations.failed', 1); + } + } + + return entry; + } + + /** + * Apply migration changes + */ + private async applyMigration(migration: SchemaMigration): Promise { + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would apply migration', { + version: migration.version, + name: migration.name, + }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + switch (migration.type) { + case 'create': + await this.createIndex({ + name: migration.index, + version: migration.version, + settings: migration.changes.settings, + mappings: migration.changes.mappings as IndexMapping, + }); + break; + + case 'update': + if (migration.changes.settings) { + await this.updateSettings(migration.index, migration.changes.settings); + } + if (migration.changes.mappings) { + await this.updateMapping(migration.index, migration.changes.mappings); + } + if (migration.changes.aliases) { + await this.updateAliases(migration.index, migration.changes.aliases); + } + break; + + case 'delete': + await this.deleteIndex(migration.index); + break; + + case 'reindex': + if (migration.changes.reindex) { + await client.reindex({ + source: { index: migration.changes.reindex.source }, + dest: { index: migration.changes.reindex.dest }, + script: migration.changes.reindex.script + ? { source: migration.changes.reindex.script } + : undefined, + timeout: `${this.config.timeout}ms`, + }); + } + break; + + case 'alias': + if (migration.changes.aliases) { + await this.updateAliases(migration.index, migration.changes.aliases); + } + break; + } + } + + /** + * Rollback a migration + */ + async rollback(version: number, migrations: SchemaMigration[]): Promise { + const migration = migrations.find((m) => m.version === version); + + if (!migration) { + throw new Error(`Migration version ${version} not found`); + } + + if (!migration.rollback) { + throw new Error(`Migration version ${version} has no rollback defined`); + } + + const startTime = Date.now(); + const entry: MigrationHistoryEntry = { + version, + name: `rollback_${migration.name}`, + status: 'running', + startedAt: new Date(), + }; + + try { + // Apply rollback changes + if (migration.rollback.settings) { + await this.updateSettings(migration.index, migration.rollback.settings); + } + if (migration.rollback.mappings) { + await this.updateMapping(migration.index, migration.rollback.mappings); + } + + entry.status = 'rolled_back'; + entry.completedAt = new Date(); + entry.duration = Date.now() - startTime; + + this.stats.rolledBackMigrations++; + + // Update history + await this.recordMigration(entry); + + if (this.config.enableLogging) { + this.logger.info('Migration rolled back', { + version, + name: migration.name, + duration: entry.duration, + }); + } + + if (this.config.enableMetrics) { + this.metrics.recordCounter('schema.migrations.rollback', 1); + } + } catch (error: any) { + entry.status = 'failed'; + entry.error = error.message; + + this.logger.error('Rollback failed', { + version, + name: migration.name, + error: error.message, + }); + } + + return entry; + } + + /** + * Get applied migrations + */ + async getAppliedMigrations(): Promise { + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + try { + const result = await client.search({ + index: this.config.historyIndex, + size: 1000, + sort: [{ version: 'asc' }], + }); + + return result.hits.hits.map((hit) => hit._source as MigrationHistoryEntry); + } catch (error: any) { + if (error.meta?.statusCode === 404) { + return []; + } + throw error; + } + } + + // ============================================================================ + // Template Management + // ============================================================================ + + /** + * Create or update index template + */ + async putTemplate(template: IndexTemplate): Promise { + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would put template', { name: template.name }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + await client.indices.putIndexTemplate({ + name: template.name, + index_patterns: template.index_patterns, + priority: template.priority, + version: template.version, + composed_of: template.composed_of, + template: template.template, + data_stream: template.data_stream, + _meta: template._meta, + }); + + this.stats.totalTemplates++; + + if (this.config.enableLogging) { + this.logger.info('Template created/updated', { name: template.name }); + } + + if (this.config.enableMetrics) { + this.metrics.recordCounter('schema.templates.created', 1); + } + } + + /** + * Get index template + */ + async getTemplate(name: string): Promise { + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + try { + const result = await client.indices.getIndexTemplate({ name }); + const template = result.index_templates[0]; + + if (!template) return null; + + return { + name: template.name, + index_patterns: template.index_template.index_patterns, + priority: template.index_template.priority, + version: template.index_template.version, + composed_of: template.index_template.composed_of, + template: template.index_template.template, + data_stream: template.index_template.data_stream, + _meta: template.index_template._meta, + }; + } catch (error: any) { + if (error.meta?.statusCode === 404) { + return null; + } + throw error; + } + } + + /** + * Delete index template + */ + async deleteTemplate(name: string): Promise { + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would delete template', { name }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + await client.indices.deleteIndexTemplate({ name }); + + this.stats.totalTemplates--; + + if (this.config.enableLogging) { + this.logger.info('Template deleted', { name }); + } + } + + /** + * Create or update component template + */ + async putComponentTemplate(template: ComponentTemplate): Promise { + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would put component template', { name: template.name }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + await client.cluster.putComponentTemplate({ + name: template.name, + template: template.template, + version: template.version, + _meta: template._meta, + }); + + if (this.config.enableLogging) { + this.logger.info('Component template created/updated', { name: template.name }); + } + } + + // ============================================================================ + // Alias Management + // ============================================================================ + + /** + * Update aliases + */ + async updateAliases( + index: string, + aliases: { add?: Record; remove?: string[] } + ): Promise { + if (this.config.dryRun) { + this.logger.info('[DRY RUN] Would update aliases', { index }); + return; + } + + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + const actions: any[] = []; + + if (aliases.add) { + for (const [alias, config] of Object.entries(aliases.add)) { + actions.push({ add: { index, alias, ...config } }); + } + } + + if (aliases.remove) { + for (const alias of aliases.remove) { + actions.push({ remove: { index, alias } }); + } + } + + if (actions.length > 0) { + await client.indices.updateAliases({ actions }); + } + + if (this.config.enableLogging) { + this.logger.info('Aliases updated', { index, actions: actions.length }); + } + } + + /** + * Add alias + */ + async addAlias(index: string, alias: string, config: Record = {}): Promise { + await this.updateAliases(index, { add: { [alias]: config } }); + } + + /** + * Remove alias + */ + async removeAlias(index: string, alias: string): Promise { + await this.updateAliases(index, { remove: [alias] }); + } + + // ============================================================================ + // Schema Validation and Comparison + // ============================================================================ + + /** + * Validate schema + */ + validateSchema(schema: IndexSchema): SchemaValidationResult { + const errors: SchemaValidationResult['errors'] = []; + const warnings: SchemaValidationResult['warnings'] = []; + + // Validate index name + if (!schema.name || schema.name.length === 0) { + errors.push({ field: 'name', message: 'Index name is required', severity: 'error' }); + } + + if (schema.name && schema.name.startsWith('_')) { + errors.push({ field: 'name', message: 'Index name cannot start with _', severity: 'error' }); + } + + // Validate mappings + if (!schema.mappings || !schema.mappings.properties) { + errors.push({ field: 'mappings', message: 'Mappings with properties are required', severity: 'error' }); + } else { + this.validateFields(schema.mappings.properties, '', errors, warnings); + } + + // Validate settings + if (schema.settings) { + if (schema.settings.number_of_shards !== undefined && schema.settings.number_of_shards < 1) { + errors.push({ field: 'settings.number_of_shards', message: 'Must be at least 1', severity: 'error' }); + } + + if (schema.settings.number_of_replicas !== undefined && schema.settings.number_of_replicas < 0) { + errors.push({ field: 'settings.number_of_replicas', message: 'Cannot be negative', severity: 'error' }); + } + } + + return { + valid: errors.length === 0, + errors, + warnings, + }; + } + + /** + * Compare two schemas + */ + diffSchemas(oldSchema: IndexSchema, newSchema: IndexSchema): SchemaDiff { + const added: string[] = []; + const removed: string[] = []; + const modified: SchemaDiff['modified'] = []; + const breakingChanges: string[] = []; + + // Compare fields + const oldFields = this.flattenFields(oldSchema.mappings?.properties || {}); + const newFields = this.flattenFields(newSchema.mappings?.properties || {}); + + for (const field of newFields.keys()) { + if (!oldFields.has(field)) { + added.push(field); + } + } + + for (const field of oldFields.keys()) { + if (!newFields.has(field)) { + removed.push(field); + breakingChanges.push(`Removed field: ${field}`); + } + } + + for (const [field, newDef] of newFields) { + const oldDef = oldFields.get(field); + if (oldDef && JSON.stringify(oldDef) !== JSON.stringify(newDef)) { + modified.push({ field, from: oldDef, to: newDef }); + + // Check for breaking changes + if (oldDef.type !== newDef.type) { + breakingChanges.push(`Field type changed: ${field} (${oldDef.type} -> ${newDef.type})`); + } + } + } + + return { + identical: added.length === 0 && removed.length === 0 && modified.length === 0, + added, + removed, + modified, + settingsChanged: JSON.stringify(oldSchema.settings) !== JSON.stringify(newSchema.settings), + aliasesChanged: JSON.stringify(oldSchema.aliases) !== JSON.stringify(newSchema.aliases), + breakingChanges, + }; + } + + /** + * Get statistics + */ + getStats(): SchemaManagerStats { + return { ...this.stats }; + } + + // ============================================================================ + // Private Methods + // ============================================================================ + + /** + * Ensure history index exists + */ + private async ensureHistoryIndex(): Promise { + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + const exists = await client.indices.exists({ index: this.config.historyIndex }); + + if (!exists) { + await client.indices.create({ + index: this.config.historyIndex, + mappings: { + properties: { + version: { type: 'integer' }, + name: { type: 'keyword' }, + status: { type: 'keyword' }, + startedAt: { type: 'date' }, + completedAt: { type: 'date' }, + duration: { type: 'long' }, + error: { type: 'text' }, + checksum: { type: 'keyword' }, + }, + }, + }); + } + } + + /** + * Record migration in history + */ + private async recordMigration(entry: MigrationHistoryEntry): Promise { + const client = ElasticsearchConnectionManager.getInstance().getClient(); + + await client.index({ + index: this.config.historyIndex, + id: `migration-${entry.version}`, + document: entry, + }); + } + + /** + * Check if settings require closing the index + */ + private settingsRequireClose(settings: Partial): boolean { + const closeRequired = ['number_of_shards', 'codec', 'routing_partition_size']; + return Object.keys(settings).some((key) => closeRequired.includes(key)); + } + + /** + * Validate fields recursively + */ + private validateFields( + fields: Record, + prefix: string, + errors: SchemaValidationResult['errors'], + warnings: SchemaValidationResult['warnings'] + ): void { + for (const [name, def] of Object.entries(fields)) { + const path = prefix ? `${prefix}.${name}` : name; + + if (!def.type) { + errors.push({ field: path, message: 'Field type is required', severity: 'error' }); + } + + // Check for text fields without keyword sub-field + if (def.type === 'text' && !def.properties) { + warnings.push({ + field: path, + message: 'Text field without keyword sub-field may limit aggregation capabilities', + }); + } + + // Validate nested properties + if (def.properties) { + this.validateFields(def.properties, path, errors, warnings); + } + } + } + + /** + * Flatten nested fields + */ + private flattenFields( + fields: Record, + prefix: string = '' + ): Map { + const result = new Map(); + + for (const [name, def] of Object.entries(fields)) { + const path = prefix ? `${prefix}.${name}` : name; + result.set(path, def); + + if (def.properties) { + const nested = this.flattenFields(def.properties, path); + for (const [nestedPath, nestedDef] of nested) { + result.set(nestedPath, nestedDef); + } + } + } + + return result; + } + + /** + * Update average migration duration + */ + private updateAvgDuration(duration: number): void { + const total = this.stats.successfulMigrations + this.stats.failedMigrations; + this.stats.avgMigrationDuration = + (this.stats.avgMigrationDuration * (total - 1) + duration) / total; + } +} + +/** + * Create a schema manager + */ +export function createSchemaManager(config?: SchemaManagerConfig): SchemaManager { + return new SchemaManager(config); +} diff --git a/ts/domain/schema/types.ts b/ts/domain/schema/types.ts new file mode 100644 index 0000000..07600f5 --- /dev/null +++ b/ts/domain/schema/types.ts @@ -0,0 +1,512 @@ +/** + * Schema Management Types + * + * Index mapping management, templates, and migrations + */ + +/** + * Elasticsearch field types + */ +export type FieldType = + | 'text' + | 'keyword' + | 'long' + | 'integer' + | 'short' + | 'byte' + | 'double' + | 'float' + | 'half_float' + | 'scaled_float' + | 'date' + | 'date_nanos' + | 'boolean' + | 'binary' + | 'integer_range' + | 'float_range' + | 'long_range' + | 'double_range' + | 'date_range' + | 'ip_range' + | 'object' + | 'nested' + | 'geo_point' + | 'geo_shape' + | 'completion' + | 'search_as_you_type' + | 'token_count' + | 'percolator' + | 'join' + | 'rank_feature' + | 'rank_features' + | 'dense_vector' + | 'sparse_vector' + | 'flattened' + | 'shape' + | 'histogram' + | 'ip' + | 'alias'; + +/** + * Field definition + */ +export interface FieldDefinition { + /** Field type */ + type: FieldType; + + /** Index this field */ + index?: boolean; + + /** Store this field */ + store?: boolean; + + /** Enable doc values */ + doc_values?: boolean; + + /** Analyzer for text fields */ + analyzer?: string; + + /** Search analyzer */ + search_analyzer?: string; + + /** Normalizer for keyword fields */ + normalizer?: string; + + /** Boost factor */ + boost?: number; + + /** Coerce values */ + coerce?: boolean; + + /** Copy to other fields */ + copy_to?: string | string[]; + + /** Enable norms */ + norms?: boolean; + + /** Null value replacement */ + null_value?: unknown; + + /** Ignore values above this length */ + ignore_above?: number; + + /** Date format */ + format?: string; + + /** Scaling factor for scaled_float */ + scaling_factor?: number; + + /** Nested properties */ + properties?: Record; + + /** Enable eager global ordinals */ + eager_global_ordinals?: boolean; + + /** Similarity algorithm */ + similarity?: string; + + /** Term vector */ + term_vector?: 'no' | 'yes' | 'with_positions' | 'with_offsets' | 'with_positions_offsets'; + + /** Dimensions for dense_vector */ + dims?: number; + + /** Additional properties */ + [key: string]: unknown; +} + +/** + * Index settings + */ +export interface IndexSettings { + /** Number of primary shards */ + number_of_shards?: number; + + /** Number of replica shards */ + number_of_replicas?: number; + + /** Refresh interval */ + refresh_interval?: string; + + /** Maximum result window */ + max_result_window?: number; + + /** Maximum inner result window */ + max_inner_result_window?: number; + + /** Maximum rescore window */ + max_rescore_window?: number; + + /** Maximum docvalue fields search */ + max_docvalue_fields_search?: number; + + /** Maximum script fields */ + max_script_fields?: number; + + /** Maximum regex length */ + max_regex_length?: number; + + /** Codec */ + codec?: string; + + /** Routing partition size */ + routing_partition_size?: number; + + /** Soft deletes retention period */ + soft_deletes?: { + retention_lease?: { + period?: string; + }; + }; + + /** Analysis settings */ + analysis?: { + analyzer?: Record; + tokenizer?: Record; + filter?: Record; + char_filter?: Record; + normalizer?: Record; + }; + + /** Additional settings */ + [key: string]: unknown; +} + +/** + * Index mapping + */ +export interface IndexMapping { + /** Dynamic mapping behavior */ + dynamic?: boolean | 'strict' | 'runtime'; + + /** Date detection */ + date_detection?: boolean; + + /** Dynamic date formats */ + dynamic_date_formats?: string[]; + + /** Numeric detection */ + numeric_detection?: boolean; + + /** Field properties */ + properties: Record; + + /** Runtime fields */ + runtime?: Record; + + /** Meta fields */ + _source?: { + enabled?: boolean; + includes?: string[]; + excludes?: string[]; + }; + + _routing?: { + required?: boolean; + }; + + _meta?: Record; +} + +/** + * Index schema definition + */ +export interface IndexSchema { + /** Index name or pattern */ + name: string; + + /** Schema version */ + version: number; + + /** Index settings */ + settings?: IndexSettings; + + /** Index mapping */ + mappings: IndexMapping; + + /** Index aliases */ + aliases?: Record; + + /** Lifecycle policy */ + lifecycle?: { + name: string; + rollover_alias?: string; + }; + + /** Schema metadata */ + metadata?: { + description?: string; + owner?: string; + created?: Date; + updated?: Date; + tags?: string[]; + }; +} + +/** + * Schema migration + */ +export interface SchemaMigration { + /** Migration version (must be unique) */ + version: number; + + /** Migration name */ + name: string; + + /** Migration description */ + description?: string; + + /** Migration type */ + type: 'create' | 'update' | 'delete' | 'reindex' | 'alias'; + + /** Target index */ + index: string; + + /** Changes to apply */ + changes: { + /** Settings changes */ + settings?: Partial; + + /** Mapping changes (new or modified fields) */ + mappings?: { + properties?: Record; + dynamic?: boolean | 'strict' | 'runtime'; + }; + + /** Alias changes */ + aliases?: { + add?: Record; + remove?: string[]; + }; + + /** Reindex configuration */ + reindex?: { + source: string; + dest: string; + script?: string; + }; + }; + + /** Rollback changes */ + rollback?: { + settings?: Partial; + mappings?: { + properties?: Record; + }; + }; + + /** Migration metadata */ + metadata?: { + author?: string; + created?: Date; + ticket?: string; + }; +} + +/** + * Migration status + */ +export type MigrationStatus = 'pending' | 'running' | 'completed' | 'failed' | 'rolled_back'; + +/** + * Migration history entry + */ +export interface MigrationHistoryEntry { + /** Migration version */ + version: number; + + /** Migration name */ + name: string; + + /** Migration status */ + status: MigrationStatus; + + /** Started at */ + startedAt: Date; + + /** Completed at */ + completedAt?: Date; + + /** Duration in milliseconds */ + duration?: number; + + /** Error if failed */ + error?: string; + + /** Checksum of migration */ + checksum?: string; +} + +/** + * Schema manager configuration + */ +export interface SchemaManagerConfig { + /** Index for storing migration history */ + historyIndex?: string; + + /** Enable dry run mode */ + dryRun?: boolean; + + /** Enable strict mode (fail on warnings) */ + strict?: boolean; + + /** Timeout for operations */ + timeout?: number; + + /** Enable logging */ + enableLogging?: boolean; + + /** Enable metrics */ + enableMetrics?: boolean; + + /** Validate schemas before applying */ + validateBeforeApply?: boolean; +} + +/** + * Schema validation result + */ +export interface SchemaValidationResult { + /** Whether schema is valid */ + valid: boolean; + + /** Validation errors */ + errors: Array<{ + field: string; + message: string; + severity: 'error' | 'warning'; + }>; + + /** Validation warnings */ + warnings: Array<{ + field: string; + message: string; + }>; +} + +/** + * Schema diff result + */ +export interface SchemaDiff { + /** Whether schemas are identical */ + identical: boolean; + + /** Added fields */ + added: string[]; + + /** Removed fields */ + removed: string[]; + + /** Modified fields */ + modified: Array<{ + field: string; + from: FieldDefinition; + to: FieldDefinition; + }>; + + /** Settings changes */ + settingsChanged: boolean; + + /** Aliases changes */ + aliasesChanged: boolean; + + /** Breaking changes */ + breakingChanges: string[]; +} + +/** + * Template definition + */ +export interface IndexTemplate { + /** Template name */ + name: string; + + /** Index patterns */ + index_patterns: string[]; + + /** Template priority */ + priority?: number; + + /** Template version */ + version?: number; + + /** Composed of component templates */ + composed_of?: string[]; + + /** Template settings */ + template?: { + settings?: IndexSettings; + mappings?: IndexMapping; + aliases?: Record; + }; + + /** Data stream settings */ + data_stream?: { + hidden?: boolean; + allow_custom_routing?: boolean; + }; + + /** Template metadata */ + _meta?: Record; +} + +/** + * Component template + */ +export interface ComponentTemplate { + /** Template name */ + name: string; + + /** Template content */ + template: { + settings?: IndexSettings; + mappings?: IndexMapping; + aliases?: Record; + }; + + /** Template version */ + version?: number; + + /** Template metadata */ + _meta?: Record; +} + +/** + * Schema manager statistics + */ +export interface SchemaManagerStats { + /** Total migrations applied */ + totalMigrations: number; + + /** Successful migrations */ + successfulMigrations: number; + + /** Failed migrations */ + failedMigrations: number; + + /** Rolled back migrations */ + rolledBackMigrations: number; + + /** Total indices managed */ + totalIndices: number; + + /** Total templates managed */ + totalTemplates: number; + + /** Last migration time */ + lastMigrationTime?: Date; + + /** Average migration duration */ + avgMigrationDuration: number; +} diff --git a/ts/examples/schema/schema-example.ts b/ts/examples/schema/schema-example.ts new file mode 100644 index 0000000..96c4f06 --- /dev/null +++ b/ts/examples/schema/schema-example.ts @@ -0,0 +1,379 @@ +/** + * Comprehensive Schema Management Example + * + * Demonstrates index mapping management, templates, and migrations + */ + +import { + createConfig, + ElasticsearchConnectionManager, + LogLevel, + createSchemaManager, + type IndexSchema, + type SchemaMigration, + type IndexTemplate, +} from '../../index.js'; + +async function main() { + console.log('=== Schema Management Example ===\n'); + + // ============================================================================ + // Step 1: Configuration + // ============================================================================ + + console.log('Step 1: Configuring Elasticsearch connection...'); + const config = createConfig() + .fromEnv() + .nodes(process.env.ELASTICSEARCH_URL || 'http://localhost:9200') + .basicAuth( + process.env.ELASTICSEARCH_USERNAME || 'elastic', + process.env.ELASTICSEARCH_PASSWORD || 'changeme' + ) + .timeout(30000) + .retries(3) + .logLevel(LogLevel.INFO) + .build(); + + // ============================================================================ + // Step 2: Initialize + // ============================================================================ + + console.log('Step 2: Initializing connection and schema manager...'); + const connectionManager = ElasticsearchConnectionManager.getInstance(config); + await connectionManager.initialize(); + + const schemaManager = createSchemaManager({ + historyIndex: '.test_migrations', + dryRun: false, + strict: true, + enableLogging: true, + enableMetrics: true, + validateBeforeApply: true, + }); + + await schemaManager.initialize(); + console.log('✓ Connection and schema manager initialized\n'); + + // ============================================================================ + // Step 3: Create Index with Schema + // ============================================================================ + + console.log('Step 3: Creating index with schema...'); + + const productSchema: IndexSchema = { + name: 'products-v1', + version: 1, + settings: { + number_of_shards: 1, + number_of_replicas: 0, + refresh_interval: '1s', + analysis: { + analyzer: { + product_analyzer: { + type: 'custom', + tokenizer: 'standard', + filter: ['lowercase', 'snowball'], + }, + }, + }, + }, + mappings: { + dynamic: 'strict', + properties: { + id: { type: 'keyword' }, + name: { + type: 'text', + analyzer: 'product_analyzer', + properties: { + keyword: { type: 'keyword', ignore_above: 256 }, + }, + }, + description: { type: 'text' }, + price: { type: 'scaled_float', scaling_factor: 100 }, + category: { type: 'keyword' }, + tags: { type: 'keyword' }, + inStock: { type: 'boolean' }, + createdAt: { type: 'date' }, + updatedAt: { type: 'date' }, + }, + }, + aliases: { + products: { is_write_index: true }, + }, + metadata: { + description: 'Product catalog index', + owner: 'catalog-team', + tags: ['catalog', 'products'], + }, + }; + + // Validate schema first + const validation = schemaManager.validateSchema(productSchema); + console.log(` Schema validation: ${validation.valid ? 'PASSED' : 'FAILED'}`); + + if (validation.warnings.length > 0) { + console.log(' Warnings:'); + for (const warning of validation.warnings) { + console.log(` - ${warning.field}: ${warning.message}`); + } + } + + // Create the index + await schemaManager.createIndex(productSchema); + console.log(' ✓ Index created with mappings and aliases\n'); + + // ============================================================================ + // Step 4: Schema Migrations + // ============================================================================ + + console.log('Step 4: Running schema migrations...'); + + const migrations: SchemaMigration[] = [ + { + version: 1, + name: 'add_rating_field', + description: 'Add product rating field', + type: 'update', + index: 'products-v1', + changes: { + mappings: { + properties: { + rating: { type: 'float' }, + reviewCount: { type: 'integer' }, + }, + }, + }, + rollback: { + // Note: Can't remove fields in ES, but document for reference + }, + metadata: { + author: 'dev-team', + ticket: 'PROD-123', + }, + }, + { + version: 2, + name: 'add_brand_field', + description: 'Add brand field with keyword type', + type: 'update', + index: 'products-v1', + changes: { + mappings: { + properties: { + brand: { type: 'keyword' }, + sku: { type: 'keyword' }, + }, + }, + }, + metadata: { + author: 'dev-team', + ticket: 'PROD-456', + }, + }, + { + version: 3, + name: 'add_inventory_object', + description: 'Add nested inventory object', + type: 'update', + index: 'products-v1', + changes: { + mappings: { + properties: { + inventory: { + type: 'object', + properties: { + quantity: { type: 'integer' }, + warehouse: { type: 'keyword' }, + lastRestocked: { type: 'date' }, + }, + }, + }, + }, + }, + metadata: { + author: 'inventory-team', + ticket: 'INV-789', + }, + }, + ]; + + const migrationResults = await schemaManager.migrate(migrations); + + console.log(' Migration results:'); + for (const result of migrationResults) { + console.log(` v${result.version} (${result.name}): ${result.status} (${result.duration}ms)`); + } + + console.log(); + + // ============================================================================ + // Step 5: Get Applied Migrations + // ============================================================================ + + console.log('Step 5: Viewing migration history...'); + + const appliedMigrations = await schemaManager.getAppliedMigrations(); + + console.log(' Applied migrations:'); + for (const migration of appliedMigrations) { + console.log(` v${migration.version}: ${migration.name} (${migration.status})`); + } + + console.log(); + + // ============================================================================ + // Step 6: Compare Schemas + // ============================================================================ + + console.log('Step 6: Comparing schemas...'); + + const oldSchema = productSchema; + const newSchema: IndexSchema = { + ...productSchema, + version: 2, + mappings: { + ...productSchema.mappings, + properties: { + ...productSchema.mappings.properties, + rating: { type: 'float' }, + discount: { type: 'float' }, + }, + }, + }; + + const diff = schemaManager.diffSchemas(oldSchema, newSchema); + + console.log(' Schema diff:'); + console.log(` Identical: ${diff.identical}`); + console.log(` Added fields: ${diff.added.join(', ') || 'none'}`); + console.log(` Removed fields: ${diff.removed.join(', ') || 'none'}`); + console.log(` Modified fields: ${diff.modified.length}`); + console.log(` Breaking changes: ${diff.breakingChanges.length}`); + + console.log(); + + // ============================================================================ + // Step 7: Index Templates + // ============================================================================ + + console.log('Step 7: Managing index templates...'); + + const template: IndexTemplate = { + name: 'products-template', + index_patterns: ['products-*'], + priority: 100, + version: 1, + template: { + settings: { + number_of_shards: 1, + number_of_replicas: 0, + }, + mappings: { + dynamic: 'strict', + properties: { + id: { type: 'keyword' }, + name: { type: 'text' }, + createdAt: { type: 'date' }, + }, + }, + aliases: { + 'all-products': {}, + }, + }, + _meta: { + description: 'Template for product indices', + }, + }; + + await schemaManager.putTemplate(template); + console.log(' ✓ Index template created'); + + // Get template + const retrievedTemplate = await schemaManager.getTemplate('products-template'); + console.log(` Template patterns: ${retrievedTemplate?.index_patterns.join(', ')}`); + + console.log(); + + // ============================================================================ + // Step 8: Alias Management + // ============================================================================ + + console.log('Step 8: Managing aliases...'); + + // Add alias + await schemaManager.addAlias('products-v1', 'products-read', { + filter: { term: { inStock: true } }, + }); + console.log(' ✓ Added filtered read alias'); + + // Get schema to see aliases + const currentSchema = await schemaManager.getSchema('products-v1'); + console.log(` Current aliases: ${Object.keys(currentSchema?.aliases || {}).join(', ')}`); + + console.log(); + + // ============================================================================ + // Step 9: Update Settings + // ============================================================================ + + console.log('Step 9: Updating index settings...'); + + await schemaManager.updateSettings('products-v1', { + refresh_interval: '5s', + max_result_window: 20000, + }); + + console.log(' ✓ Settings updated'); + + console.log(); + + // ============================================================================ + // Step 10: Statistics + // ============================================================================ + + console.log('Step 10: Schema manager statistics...\n'); + + const stats = schemaManager.getStats(); + + console.log('Schema Manager Statistics:'); + console.log(` Total migrations: ${stats.totalMigrations}`); + console.log(` Successful: ${stats.successfulMigrations}`); + console.log(` Failed: ${stats.failedMigrations}`); + console.log(` Rolled back: ${stats.rolledBackMigrations}`); + console.log(` Total indices: ${stats.totalIndices}`); + console.log(` Total templates: ${stats.totalTemplates}`); + console.log(` Avg migration duration: ${stats.avgMigrationDuration.toFixed(2)}ms`); + + console.log(); + + // ============================================================================ + // Step 11: Cleanup + // ============================================================================ + + console.log('Step 11: Cleanup...'); + + await schemaManager.deleteTemplate('products-template'); + await schemaManager.deleteIndex('products-v1'); + await connectionManager.destroy(); + + console.log('✓ Cleanup complete\n'); + + console.log('=== Schema Management Example Complete ==='); + console.log('\nKey Features Demonstrated:'); + console.log(' ✓ Index creation with mappings and settings'); + console.log(' ✓ Schema validation before apply'); + console.log(' ✓ Versioned migrations with history tracking'); + console.log(' ✓ Schema diff and comparison'); + console.log(' ✓ Index templates'); + console.log(' ✓ Alias management with filters'); + console.log(' ✓ Settings updates'); + console.log(' ✓ Migration rollback support'); + console.log(' ✓ Dry run mode'); + console.log(' ✓ Comprehensive statistics'); +} + +// Run the example +main().catch((error) => { + console.error('Example failed:', error); + process.exit(1); +}); diff --git a/ts/index.ts b/ts/index.ts index 06ea21d..4419a3b 100644 --- a/ts/index.ts +++ b/ts/index.ts @@ -14,6 +14,7 @@ export * from './domain/logging/index.js'; export * from './domain/bulk/index.js'; export * from './domain/kv/index.js'; export * from './domain/transactions/index.js'; +export * from './domain/schema/index.js'; // Re-export commonly used items for convenience export { @@ -172,3 +173,23 @@ export { type Savepoint, type TransactionCallbacks, } from './domain/transactions/index.js'; + +export { + // Schema + SchemaManager, + createSchemaManager, + type FieldType, + type FieldDefinition, + type IndexSettings, + type IndexMapping, + type IndexSchema, + type SchemaMigration, + type MigrationStatus, + type MigrationHistoryEntry, + type SchemaManagerConfig, + type SchemaValidationResult, + type SchemaDiff, + type IndexTemplate, + type ComponentTemplate, + type SchemaManagerStats, +} from './domain/schema/index.js';