feat(schema-manager): Add Schema Management module and expose it in public API; update README to mark Phase 3 complete and move priorities to Phase 4
This commit is contained in:
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@apiclient.xyz/elasticsearch',
|
||||
version: '3.0.0',
|
||||
version: '3.1.0',
|
||||
description: 'log to elasticsearch in a kibana compatible format'
|
||||
}
|
||||
|
||||
26
ts/domain/schema/index.ts
Normal file
26
ts/domain/schema/index.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Schema Management Module
|
||||
*
|
||||
* Index mapping management, templates, and migrations
|
||||
*/
|
||||
|
||||
// Main classes
|
||||
export { SchemaManager, createSchemaManager } from './schema-manager.js';
|
||||
|
||||
// Types
|
||||
export type {
|
||||
FieldType,
|
||||
FieldDefinition,
|
||||
IndexSettings,
|
||||
IndexMapping,
|
||||
IndexSchema,
|
||||
SchemaMigration,
|
||||
MigrationStatus,
|
||||
MigrationHistoryEntry,
|
||||
SchemaManagerConfig,
|
||||
SchemaValidationResult,
|
||||
SchemaDiff,
|
||||
IndexTemplate,
|
||||
ComponentTemplate,
|
||||
SchemaManagerStats,
|
||||
} from './types.js';
|
||||
926
ts/domain/schema/schema-manager.ts
Normal file
926
ts/domain/schema/schema-manager.ts
Normal file
@@ -0,0 +1,926 @@
|
||||
/**
|
||||
* Schema Manager
|
||||
*
|
||||
* Index mapping management, templates, and migrations
|
||||
*/
|
||||
|
||||
import { ElasticsearchConnectionManager } from '../../core/connection/connection-manager.js';
|
||||
import { Logger, defaultLogger } from '../../core/observability/logger.js';
|
||||
import { MetricsCollector, defaultMetricsCollector } from '../../core/observability/metrics.js';
|
||||
import type {
|
||||
IndexSchema,
|
||||
IndexSettings,
|
||||
IndexMapping,
|
||||
FieldDefinition,
|
||||
SchemaMigration,
|
||||
MigrationHistoryEntry,
|
||||
MigrationStatus,
|
||||
SchemaManagerConfig,
|
||||
SchemaValidationResult,
|
||||
SchemaDiff,
|
||||
IndexTemplate,
|
||||
ComponentTemplate,
|
||||
SchemaManagerStats,
|
||||
} from './types.js';
|
||||
|
||||
/**
|
||||
* Default configuration
|
||||
*/
|
||||
const DEFAULT_CONFIG: Required<SchemaManagerConfig> = {
|
||||
historyIndex: '.schema_migrations',
|
||||
dryRun: false,
|
||||
strict: false,
|
||||
timeout: 60000, // 1 minute
|
||||
enableLogging: true,
|
||||
enableMetrics: true,
|
||||
validateBeforeApply: true,
|
||||
};
|
||||
|
||||
/**
|
||||
* Schema Manager
|
||||
*/
|
||||
export class SchemaManager {
|
||||
private config: Required<SchemaManagerConfig>;
|
||||
private stats: SchemaManagerStats;
|
||||
private logger: Logger;
|
||||
private metrics: MetricsCollector;
|
||||
|
||||
constructor(config: SchemaManagerConfig = {}) {
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
this.logger = defaultLogger;
|
||||
this.metrics = defaultMetricsCollector;
|
||||
|
||||
this.stats = {
|
||||
totalMigrations: 0,
|
||||
successfulMigrations: 0,
|
||||
failedMigrations: 0,
|
||||
rolledBackMigrations: 0,
|
||||
totalIndices: 0,
|
||||
totalTemplates: 0,
|
||||
avgMigrationDuration: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize schema manager
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
await this.ensureHistoryIndex();
|
||||
|
||||
this.logger.info('SchemaManager initialized', {
|
||||
historyIndex: this.config.historyIndex,
|
||||
dryRun: this.config.dryRun,
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Index Management
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Create an index with schema
|
||||
*/
|
||||
async createIndex(schema: IndexSchema): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
|
||||
if (this.config.validateBeforeApply) {
|
||||
const validation = this.validateSchema(schema);
|
||||
if (!validation.valid) {
|
||||
throw new Error(`Schema validation failed: ${validation.errors.map((e) => e.message).join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would create index', { index: schema.name });
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
const body: any = {
|
||||
settings: schema.settings,
|
||||
mappings: schema.mappings,
|
||||
};
|
||||
|
||||
if (schema.aliases) {
|
||||
body.aliases = schema.aliases;
|
||||
}
|
||||
|
||||
await client.indices.create({
|
||||
index: schema.name,
|
||||
...body,
|
||||
timeout: `${this.config.timeout}ms`,
|
||||
});
|
||||
|
||||
this.stats.totalIndices++;
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Index created', {
|
||||
index: schema.name,
|
||||
version: schema.version,
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
}
|
||||
|
||||
if (this.config.enableMetrics) {
|
||||
this.metrics.recordCounter('schema.indices.created', 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update index mapping
|
||||
*/
|
||||
async updateMapping(index: string, mapping: Partial<IndexMapping>): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would update mapping', { index });
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
await client.indices.putMapping({
|
||||
index,
|
||||
...mapping,
|
||||
timeout: `${this.config.timeout}ms`,
|
||||
});
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Mapping updated', {
|
||||
index,
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
}
|
||||
|
||||
if (this.config.enableMetrics) {
|
||||
this.metrics.recordCounter('schema.mappings.updated', 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update index settings
|
||||
*/
|
||||
async updateSettings(index: string, settings: Partial<IndexSettings>): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would update settings', { index });
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
// Some settings require closing the index
|
||||
const requiresClose = this.settingsRequireClose(settings);
|
||||
|
||||
if (requiresClose) {
|
||||
await client.indices.close({ index });
|
||||
}
|
||||
|
||||
try {
|
||||
await client.indices.putSettings({
|
||||
index,
|
||||
settings,
|
||||
timeout: `${this.config.timeout}ms`,
|
||||
});
|
||||
} finally {
|
||||
if (requiresClose) {
|
||||
await client.indices.open({ index });
|
||||
}
|
||||
}
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Settings updated', {
|
||||
index,
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
}
|
||||
|
||||
if (this.config.enableMetrics) {
|
||||
this.metrics.recordCounter('schema.settings.updated', 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get index schema
|
||||
*/
|
||||
async getSchema(index: string): Promise<IndexSchema | null> {
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
try {
|
||||
const [mappingResult, settingsResult, aliasResult] = await Promise.all([
|
||||
client.indices.getMapping({ index }),
|
||||
client.indices.getSettings({ index }),
|
||||
client.indices.getAlias({ index }),
|
||||
]);
|
||||
|
||||
const indexData = mappingResult[index];
|
||||
const settingsData = settingsResult[index];
|
||||
const aliasData = aliasResult[index];
|
||||
|
||||
return {
|
||||
name: index,
|
||||
version: 0, // Version not stored in ES by default
|
||||
settings: settingsData?.settings?.index as IndexSettings,
|
||||
mappings: indexData?.mappings as IndexMapping,
|
||||
aliases: aliasData?.aliases,
|
||||
};
|
||||
} catch (error: any) {
|
||||
if (error.meta?.statusCode === 404) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an index
|
||||
*/
|
||||
async deleteIndex(index: string): Promise<void> {
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would delete index', { index });
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
await client.indices.delete({
|
||||
index,
|
||||
timeout: `${this.config.timeout}ms`,
|
||||
});
|
||||
|
||||
this.stats.totalIndices--;
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Index deleted', { index });
|
||||
}
|
||||
|
||||
if (this.config.enableMetrics) {
|
||||
this.metrics.recordCounter('schema.indices.deleted', 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if index exists
|
||||
*/
|
||||
async indexExists(index: string): Promise<boolean> {
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
return await client.indices.exists({ index });
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Migration Management
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Run migrations
|
||||
*/
|
||||
async migrate(migrations: SchemaMigration[]): Promise<MigrationHistoryEntry[]> {
|
||||
const results: MigrationHistoryEntry[] = [];
|
||||
|
||||
// Get current migration state
|
||||
const appliedVersions = await this.getAppliedMigrations();
|
||||
const appliedSet = new Set(appliedVersions.map((m) => m.version));
|
||||
|
||||
// Sort migrations by version
|
||||
const sortedMigrations = [...migrations].sort((a, b) => a.version - b.version);
|
||||
|
||||
// Filter pending migrations
|
||||
const pendingMigrations = sortedMigrations.filter((m) => !appliedSet.has(m.version));
|
||||
|
||||
if (pendingMigrations.length === 0) {
|
||||
this.logger.info('No pending migrations');
|
||||
return results;
|
||||
}
|
||||
|
||||
this.logger.info(`Running ${pendingMigrations.length} migrations`);
|
||||
|
||||
for (const migration of pendingMigrations) {
|
||||
const result = await this.runMigration(migration);
|
||||
results.push(result);
|
||||
|
||||
if (result.status === 'failed') {
|
||||
this.logger.error('Migration failed, stopping', {
|
||||
version: migration.version,
|
||||
name: migration.name,
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single migration
|
||||
*/
|
||||
private async runMigration(migration: SchemaMigration): Promise<MigrationHistoryEntry> {
|
||||
const startTime = Date.now();
|
||||
const entry: MigrationHistoryEntry = {
|
||||
version: migration.version,
|
||||
name: migration.name,
|
||||
status: 'running',
|
||||
startedAt: new Date(),
|
||||
};
|
||||
|
||||
this.stats.totalMigrations++;
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Running migration', {
|
||||
version: migration.version,
|
||||
name: migration.name,
|
||||
type: migration.type,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
await this.applyMigration(migration);
|
||||
|
||||
entry.status = 'completed';
|
||||
entry.completedAt = new Date();
|
||||
entry.duration = Date.now() - startTime;
|
||||
|
||||
this.stats.successfulMigrations++;
|
||||
this.stats.lastMigrationTime = new Date();
|
||||
this.updateAvgDuration(entry.duration);
|
||||
|
||||
// Record in history
|
||||
await this.recordMigration(entry);
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Migration completed', {
|
||||
version: migration.version,
|
||||
name: migration.name,
|
||||
duration: entry.duration,
|
||||
});
|
||||
}
|
||||
|
||||
if (this.config.enableMetrics) {
|
||||
this.metrics.recordCounter('schema.migrations.success', 1);
|
||||
this.metrics.recordHistogram('schema.migrations.duration', entry.duration);
|
||||
}
|
||||
} catch (error: any) {
|
||||
entry.status = 'failed';
|
||||
entry.completedAt = new Date();
|
||||
entry.duration = Date.now() - startTime;
|
||||
entry.error = error.message;
|
||||
|
||||
this.stats.failedMigrations++;
|
||||
|
||||
// Record failure in history
|
||||
await this.recordMigration(entry);
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.error('Migration failed', {
|
||||
version: migration.version,
|
||||
name: migration.name,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
|
||||
if (this.config.enableMetrics) {
|
||||
this.metrics.recordCounter('schema.migrations.failed', 1);
|
||||
}
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply migration changes
|
||||
*/
|
||||
private async applyMigration(migration: SchemaMigration): Promise<void> {
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would apply migration', {
|
||||
version: migration.version,
|
||||
name: migration.name,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
switch (migration.type) {
|
||||
case 'create':
|
||||
await this.createIndex({
|
||||
name: migration.index,
|
||||
version: migration.version,
|
||||
settings: migration.changes.settings,
|
||||
mappings: migration.changes.mappings as IndexMapping,
|
||||
});
|
||||
break;
|
||||
|
||||
case 'update':
|
||||
if (migration.changes.settings) {
|
||||
await this.updateSettings(migration.index, migration.changes.settings);
|
||||
}
|
||||
if (migration.changes.mappings) {
|
||||
await this.updateMapping(migration.index, migration.changes.mappings);
|
||||
}
|
||||
if (migration.changes.aliases) {
|
||||
await this.updateAliases(migration.index, migration.changes.aliases);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'delete':
|
||||
await this.deleteIndex(migration.index);
|
||||
break;
|
||||
|
||||
case 'reindex':
|
||||
if (migration.changes.reindex) {
|
||||
await client.reindex({
|
||||
source: { index: migration.changes.reindex.source },
|
||||
dest: { index: migration.changes.reindex.dest },
|
||||
script: migration.changes.reindex.script
|
||||
? { source: migration.changes.reindex.script }
|
||||
: undefined,
|
||||
timeout: `${this.config.timeout}ms`,
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'alias':
|
||||
if (migration.changes.aliases) {
|
||||
await this.updateAliases(migration.index, migration.changes.aliases);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rollback a migration
|
||||
*/
|
||||
async rollback(version: number, migrations: SchemaMigration[]): Promise<MigrationHistoryEntry> {
|
||||
const migration = migrations.find((m) => m.version === version);
|
||||
|
||||
if (!migration) {
|
||||
throw new Error(`Migration version ${version} not found`);
|
||||
}
|
||||
|
||||
if (!migration.rollback) {
|
||||
throw new Error(`Migration version ${version} has no rollback defined`);
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
const entry: MigrationHistoryEntry = {
|
||||
version,
|
||||
name: `rollback_${migration.name}`,
|
||||
status: 'running',
|
||||
startedAt: new Date(),
|
||||
};
|
||||
|
||||
try {
|
||||
// Apply rollback changes
|
||||
if (migration.rollback.settings) {
|
||||
await this.updateSettings(migration.index, migration.rollback.settings);
|
||||
}
|
||||
if (migration.rollback.mappings) {
|
||||
await this.updateMapping(migration.index, migration.rollback.mappings);
|
||||
}
|
||||
|
||||
entry.status = 'rolled_back';
|
||||
entry.completedAt = new Date();
|
||||
entry.duration = Date.now() - startTime;
|
||||
|
||||
this.stats.rolledBackMigrations++;
|
||||
|
||||
// Update history
|
||||
await this.recordMigration(entry);
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Migration rolled back', {
|
||||
version,
|
||||
name: migration.name,
|
||||
duration: entry.duration,
|
||||
});
|
||||
}
|
||||
|
||||
if (this.config.enableMetrics) {
|
||||
this.metrics.recordCounter('schema.migrations.rollback', 1);
|
||||
}
|
||||
} catch (error: any) {
|
||||
entry.status = 'failed';
|
||||
entry.error = error.message;
|
||||
|
||||
this.logger.error('Rollback failed', {
|
||||
version,
|
||||
name: migration.name,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get applied migrations
|
||||
*/
|
||||
async getAppliedMigrations(): Promise<MigrationHistoryEntry[]> {
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
try {
|
||||
const result = await client.search({
|
||||
index: this.config.historyIndex,
|
||||
size: 1000,
|
||||
sort: [{ version: 'asc' }],
|
||||
});
|
||||
|
||||
return result.hits.hits.map((hit) => hit._source as MigrationHistoryEntry);
|
||||
} catch (error: any) {
|
||||
if (error.meta?.statusCode === 404) {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Template Management
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Create or update index template
|
||||
*/
|
||||
async putTemplate(template: IndexTemplate): Promise<void> {
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would put template', { name: template.name });
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
await client.indices.putIndexTemplate({
|
||||
name: template.name,
|
||||
index_patterns: template.index_patterns,
|
||||
priority: template.priority,
|
||||
version: template.version,
|
||||
composed_of: template.composed_of,
|
||||
template: template.template,
|
||||
data_stream: template.data_stream,
|
||||
_meta: template._meta,
|
||||
});
|
||||
|
||||
this.stats.totalTemplates++;
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Template created/updated', { name: template.name });
|
||||
}
|
||||
|
||||
if (this.config.enableMetrics) {
|
||||
this.metrics.recordCounter('schema.templates.created', 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get index template
|
||||
*/
|
||||
async getTemplate(name: string): Promise<IndexTemplate | null> {
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
try {
|
||||
const result = await client.indices.getIndexTemplate({ name });
|
||||
const template = result.index_templates[0];
|
||||
|
||||
if (!template) return null;
|
||||
|
||||
return {
|
||||
name: template.name,
|
||||
index_patterns: template.index_template.index_patterns,
|
||||
priority: template.index_template.priority,
|
||||
version: template.index_template.version,
|
||||
composed_of: template.index_template.composed_of,
|
||||
template: template.index_template.template,
|
||||
data_stream: template.index_template.data_stream,
|
||||
_meta: template.index_template._meta,
|
||||
};
|
||||
} catch (error: any) {
|
||||
if (error.meta?.statusCode === 404) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete index template
|
||||
*/
|
||||
async deleteTemplate(name: string): Promise<void> {
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would delete template', { name });
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
await client.indices.deleteIndexTemplate({ name });
|
||||
|
||||
this.stats.totalTemplates--;
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Template deleted', { name });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or update component template
|
||||
*/
|
||||
async putComponentTemplate(template: ComponentTemplate): Promise<void> {
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would put component template', { name: template.name });
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
await client.cluster.putComponentTemplate({
|
||||
name: template.name,
|
||||
template: template.template,
|
||||
version: template.version,
|
||||
_meta: template._meta,
|
||||
});
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Component template created/updated', { name: template.name });
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Alias Management
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Update aliases
|
||||
*/
|
||||
async updateAliases(
|
||||
index: string,
|
||||
aliases: { add?: Record<string, unknown>; remove?: string[] }
|
||||
): Promise<void> {
|
||||
if (this.config.dryRun) {
|
||||
this.logger.info('[DRY RUN] Would update aliases', { index });
|
||||
return;
|
||||
}
|
||||
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
const actions: any[] = [];
|
||||
|
||||
if (aliases.add) {
|
||||
for (const [alias, config] of Object.entries(aliases.add)) {
|
||||
actions.push({ add: { index, alias, ...config } });
|
||||
}
|
||||
}
|
||||
|
||||
if (aliases.remove) {
|
||||
for (const alias of aliases.remove) {
|
||||
actions.push({ remove: { index, alias } });
|
||||
}
|
||||
}
|
||||
|
||||
if (actions.length > 0) {
|
||||
await client.indices.updateAliases({ actions });
|
||||
}
|
||||
|
||||
if (this.config.enableLogging) {
|
||||
this.logger.info('Aliases updated', { index, actions: actions.length });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add alias
|
||||
*/
|
||||
async addAlias(index: string, alias: string, config: Record<string, unknown> = {}): Promise<void> {
|
||||
await this.updateAliases(index, { add: { [alias]: config } });
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove alias
|
||||
*/
|
||||
async removeAlias(index: string, alias: string): Promise<void> {
|
||||
await this.updateAliases(index, { remove: [alias] });
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Schema Validation and Comparison
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Validate schema
|
||||
*/
|
||||
validateSchema(schema: IndexSchema): SchemaValidationResult {
|
||||
const errors: SchemaValidationResult['errors'] = [];
|
||||
const warnings: SchemaValidationResult['warnings'] = [];
|
||||
|
||||
// Validate index name
|
||||
if (!schema.name || schema.name.length === 0) {
|
||||
errors.push({ field: 'name', message: 'Index name is required', severity: 'error' });
|
||||
}
|
||||
|
||||
if (schema.name && schema.name.startsWith('_')) {
|
||||
errors.push({ field: 'name', message: 'Index name cannot start with _', severity: 'error' });
|
||||
}
|
||||
|
||||
// Validate mappings
|
||||
if (!schema.mappings || !schema.mappings.properties) {
|
||||
errors.push({ field: 'mappings', message: 'Mappings with properties are required', severity: 'error' });
|
||||
} else {
|
||||
this.validateFields(schema.mappings.properties, '', errors, warnings);
|
||||
}
|
||||
|
||||
// Validate settings
|
||||
if (schema.settings) {
|
||||
if (schema.settings.number_of_shards !== undefined && schema.settings.number_of_shards < 1) {
|
||||
errors.push({ field: 'settings.number_of_shards', message: 'Must be at least 1', severity: 'error' });
|
||||
}
|
||||
|
||||
if (schema.settings.number_of_replicas !== undefined && schema.settings.number_of_replicas < 0) {
|
||||
errors.push({ field: 'settings.number_of_replicas', message: 'Cannot be negative', severity: 'error' });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
errors,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two schemas
|
||||
*/
|
||||
diffSchemas(oldSchema: IndexSchema, newSchema: IndexSchema): SchemaDiff {
|
||||
const added: string[] = [];
|
||||
const removed: string[] = [];
|
||||
const modified: SchemaDiff['modified'] = [];
|
||||
const breakingChanges: string[] = [];
|
||||
|
||||
// Compare fields
|
||||
const oldFields = this.flattenFields(oldSchema.mappings?.properties || {});
|
||||
const newFields = this.flattenFields(newSchema.mappings?.properties || {});
|
||||
|
||||
for (const field of newFields.keys()) {
|
||||
if (!oldFields.has(field)) {
|
||||
added.push(field);
|
||||
}
|
||||
}
|
||||
|
||||
for (const field of oldFields.keys()) {
|
||||
if (!newFields.has(field)) {
|
||||
removed.push(field);
|
||||
breakingChanges.push(`Removed field: ${field}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [field, newDef] of newFields) {
|
||||
const oldDef = oldFields.get(field);
|
||||
if (oldDef && JSON.stringify(oldDef) !== JSON.stringify(newDef)) {
|
||||
modified.push({ field, from: oldDef, to: newDef });
|
||||
|
||||
// Check for breaking changes
|
||||
if (oldDef.type !== newDef.type) {
|
||||
breakingChanges.push(`Field type changed: ${field} (${oldDef.type} -> ${newDef.type})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
identical: added.length === 0 && removed.length === 0 && modified.length === 0,
|
||||
added,
|
||||
removed,
|
||||
modified,
|
||||
settingsChanged: JSON.stringify(oldSchema.settings) !== JSON.stringify(newSchema.settings),
|
||||
aliasesChanged: JSON.stringify(oldSchema.aliases) !== JSON.stringify(newSchema.aliases),
|
||||
breakingChanges,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics
|
||||
*/
|
||||
getStats(): SchemaManagerStats {
|
||||
return { ...this.stats };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Private Methods
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Ensure history index exists
|
||||
*/
|
||||
private async ensureHistoryIndex(): Promise<void> {
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
const exists = await client.indices.exists({ index: this.config.historyIndex });
|
||||
|
||||
if (!exists) {
|
||||
await client.indices.create({
|
||||
index: this.config.historyIndex,
|
||||
mappings: {
|
||||
properties: {
|
||||
version: { type: 'integer' },
|
||||
name: { type: 'keyword' },
|
||||
status: { type: 'keyword' },
|
||||
startedAt: { type: 'date' },
|
||||
completedAt: { type: 'date' },
|
||||
duration: { type: 'long' },
|
||||
error: { type: 'text' },
|
||||
checksum: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record migration in history
|
||||
*/
|
||||
private async recordMigration(entry: MigrationHistoryEntry): Promise<void> {
|
||||
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||
|
||||
await client.index({
|
||||
index: this.config.historyIndex,
|
||||
id: `migration-${entry.version}`,
|
||||
document: entry,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if settings require closing the index
|
||||
*/
|
||||
private settingsRequireClose(settings: Partial<IndexSettings>): boolean {
|
||||
const closeRequired = ['number_of_shards', 'codec', 'routing_partition_size'];
|
||||
return Object.keys(settings).some((key) => closeRequired.includes(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate fields recursively
|
||||
*/
|
||||
private validateFields(
|
||||
fields: Record<string, FieldDefinition>,
|
||||
prefix: string,
|
||||
errors: SchemaValidationResult['errors'],
|
||||
warnings: SchemaValidationResult['warnings']
|
||||
): void {
|
||||
for (const [name, def] of Object.entries(fields)) {
|
||||
const path = prefix ? `${prefix}.${name}` : name;
|
||||
|
||||
if (!def.type) {
|
||||
errors.push({ field: path, message: 'Field type is required', severity: 'error' });
|
||||
}
|
||||
|
||||
// Check for text fields without keyword sub-field
|
||||
if (def.type === 'text' && !def.properties) {
|
||||
warnings.push({
|
||||
field: path,
|
||||
message: 'Text field without keyword sub-field may limit aggregation capabilities',
|
||||
});
|
||||
}
|
||||
|
||||
// Validate nested properties
|
||||
if (def.properties) {
|
||||
this.validateFields(def.properties, path, errors, warnings);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flatten nested fields
|
||||
*/
|
||||
private flattenFields(
|
||||
fields: Record<string, FieldDefinition>,
|
||||
prefix: string = ''
|
||||
): Map<string, FieldDefinition> {
|
||||
const result = new Map<string, FieldDefinition>();
|
||||
|
||||
for (const [name, def] of Object.entries(fields)) {
|
||||
const path = prefix ? `${prefix}.${name}` : name;
|
||||
result.set(path, def);
|
||||
|
||||
if (def.properties) {
|
||||
const nested = this.flattenFields(def.properties, path);
|
||||
for (const [nestedPath, nestedDef] of nested) {
|
||||
result.set(nestedPath, nestedDef);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update average migration duration
|
||||
*/
|
||||
private updateAvgDuration(duration: number): void {
|
||||
const total = this.stats.successfulMigrations + this.stats.failedMigrations;
|
||||
this.stats.avgMigrationDuration =
|
||||
(this.stats.avgMigrationDuration * (total - 1) + duration) / total;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a schema manager
|
||||
*/
|
||||
export function createSchemaManager(config?: SchemaManagerConfig): SchemaManager {
|
||||
return new SchemaManager(config);
|
||||
}
|
||||
512
ts/domain/schema/types.ts
Normal file
512
ts/domain/schema/types.ts
Normal file
@@ -0,0 +1,512 @@
|
||||
/**
|
||||
* Schema Management Types
|
||||
*
|
||||
* Index mapping management, templates, and migrations
|
||||
*/
|
||||
|
||||
/**
|
||||
* Elasticsearch field types
|
||||
*/
|
||||
export type FieldType =
|
||||
| 'text'
|
||||
| 'keyword'
|
||||
| 'long'
|
||||
| 'integer'
|
||||
| 'short'
|
||||
| 'byte'
|
||||
| 'double'
|
||||
| 'float'
|
||||
| 'half_float'
|
||||
| 'scaled_float'
|
||||
| 'date'
|
||||
| 'date_nanos'
|
||||
| 'boolean'
|
||||
| 'binary'
|
||||
| 'integer_range'
|
||||
| 'float_range'
|
||||
| 'long_range'
|
||||
| 'double_range'
|
||||
| 'date_range'
|
||||
| 'ip_range'
|
||||
| 'object'
|
||||
| 'nested'
|
||||
| 'geo_point'
|
||||
| 'geo_shape'
|
||||
| 'completion'
|
||||
| 'search_as_you_type'
|
||||
| 'token_count'
|
||||
| 'percolator'
|
||||
| 'join'
|
||||
| 'rank_feature'
|
||||
| 'rank_features'
|
||||
| 'dense_vector'
|
||||
| 'sparse_vector'
|
||||
| 'flattened'
|
||||
| 'shape'
|
||||
| 'histogram'
|
||||
| 'ip'
|
||||
| 'alias';
|
||||
|
||||
/**
|
||||
* Field definition
|
||||
*/
|
||||
export interface FieldDefinition {
|
||||
/** Field type */
|
||||
type: FieldType;
|
||||
|
||||
/** Index this field */
|
||||
index?: boolean;
|
||||
|
||||
/** Store this field */
|
||||
store?: boolean;
|
||||
|
||||
/** Enable doc values */
|
||||
doc_values?: boolean;
|
||||
|
||||
/** Analyzer for text fields */
|
||||
analyzer?: string;
|
||||
|
||||
/** Search analyzer */
|
||||
search_analyzer?: string;
|
||||
|
||||
/** Normalizer for keyword fields */
|
||||
normalizer?: string;
|
||||
|
||||
/** Boost factor */
|
||||
boost?: number;
|
||||
|
||||
/** Coerce values */
|
||||
coerce?: boolean;
|
||||
|
||||
/** Copy to other fields */
|
||||
copy_to?: string | string[];
|
||||
|
||||
/** Enable norms */
|
||||
norms?: boolean;
|
||||
|
||||
/** Null value replacement */
|
||||
null_value?: unknown;
|
||||
|
||||
/** Ignore values above this length */
|
||||
ignore_above?: number;
|
||||
|
||||
/** Date format */
|
||||
format?: string;
|
||||
|
||||
/** Scaling factor for scaled_float */
|
||||
scaling_factor?: number;
|
||||
|
||||
/** Nested properties */
|
||||
properties?: Record<string, FieldDefinition>;
|
||||
|
||||
/** Enable eager global ordinals */
|
||||
eager_global_ordinals?: boolean;
|
||||
|
||||
/** Similarity algorithm */
|
||||
similarity?: string;
|
||||
|
||||
/** Term vector */
|
||||
term_vector?: 'no' | 'yes' | 'with_positions' | 'with_offsets' | 'with_positions_offsets';
|
||||
|
||||
/** Dimensions for dense_vector */
|
||||
dims?: number;
|
||||
|
||||
/** Additional properties */
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index settings
|
||||
*/
|
||||
export interface IndexSettings {
|
||||
/** Number of primary shards */
|
||||
number_of_shards?: number;
|
||||
|
||||
/** Number of replica shards */
|
||||
number_of_replicas?: number;
|
||||
|
||||
/** Refresh interval */
|
||||
refresh_interval?: string;
|
||||
|
||||
/** Maximum result window */
|
||||
max_result_window?: number;
|
||||
|
||||
/** Maximum inner result window */
|
||||
max_inner_result_window?: number;
|
||||
|
||||
/** Maximum rescore window */
|
||||
max_rescore_window?: number;
|
||||
|
||||
/** Maximum docvalue fields search */
|
||||
max_docvalue_fields_search?: number;
|
||||
|
||||
/** Maximum script fields */
|
||||
max_script_fields?: number;
|
||||
|
||||
/** Maximum regex length */
|
||||
max_regex_length?: number;
|
||||
|
||||
/** Codec */
|
||||
codec?: string;
|
||||
|
||||
/** Routing partition size */
|
||||
routing_partition_size?: number;
|
||||
|
||||
/** Soft deletes retention period */
|
||||
soft_deletes?: {
|
||||
retention_lease?: {
|
||||
period?: string;
|
||||
};
|
||||
};
|
||||
|
||||
/** Analysis settings */
|
||||
analysis?: {
|
||||
analyzer?: Record<string, unknown>;
|
||||
tokenizer?: Record<string, unknown>;
|
||||
filter?: Record<string, unknown>;
|
||||
char_filter?: Record<string, unknown>;
|
||||
normalizer?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
/** Additional settings */
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index mapping
|
||||
*/
|
||||
export interface IndexMapping {
|
||||
/** Dynamic mapping behavior */
|
||||
dynamic?: boolean | 'strict' | 'runtime';
|
||||
|
||||
/** Date detection */
|
||||
date_detection?: boolean;
|
||||
|
||||
/** Dynamic date formats */
|
||||
dynamic_date_formats?: string[];
|
||||
|
||||
/** Numeric detection */
|
||||
numeric_detection?: boolean;
|
||||
|
||||
/** Field properties */
|
||||
properties: Record<string, FieldDefinition>;
|
||||
|
||||
/** Runtime fields */
|
||||
runtime?: Record<string, {
|
||||
type: string;
|
||||
script?: {
|
||||
source: string;
|
||||
lang?: string;
|
||||
};
|
||||
}>;
|
||||
|
||||
/** Meta fields */
|
||||
_source?: {
|
||||
enabled?: boolean;
|
||||
includes?: string[];
|
||||
excludes?: string[];
|
||||
};
|
||||
|
||||
_routing?: {
|
||||
required?: boolean;
|
||||
};
|
||||
|
||||
_meta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index schema definition
|
||||
*/
|
||||
export interface IndexSchema {
|
||||
/** Index name or pattern */
|
||||
name: string;
|
||||
|
||||
/** Schema version */
|
||||
version: number;
|
||||
|
||||
/** Index settings */
|
||||
settings?: IndexSettings;
|
||||
|
||||
/** Index mapping */
|
||||
mappings: IndexMapping;
|
||||
|
||||
/** Index aliases */
|
||||
aliases?: Record<string, {
|
||||
filter?: unknown;
|
||||
routing?: string;
|
||||
is_write_index?: boolean;
|
||||
is_hidden?: boolean;
|
||||
}>;
|
||||
|
||||
/** Lifecycle policy */
|
||||
lifecycle?: {
|
||||
name: string;
|
||||
rollover_alias?: string;
|
||||
};
|
||||
|
||||
/** Schema metadata */
|
||||
metadata?: {
|
||||
description?: string;
|
||||
owner?: string;
|
||||
created?: Date;
|
||||
updated?: Date;
|
||||
tags?: string[];
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Schema migration
|
||||
*/
|
||||
export interface SchemaMigration {
|
||||
/** Migration version (must be unique) */
|
||||
version: number;
|
||||
|
||||
/** Migration name */
|
||||
name: string;
|
||||
|
||||
/** Migration description */
|
||||
description?: string;
|
||||
|
||||
/** Migration type */
|
||||
type: 'create' | 'update' | 'delete' | 'reindex' | 'alias';
|
||||
|
||||
/** Target index */
|
||||
index: string;
|
||||
|
||||
/** Changes to apply */
|
||||
changes: {
|
||||
/** Settings changes */
|
||||
settings?: Partial<IndexSettings>;
|
||||
|
||||
/** Mapping changes (new or modified fields) */
|
||||
mappings?: {
|
||||
properties?: Record<string, FieldDefinition>;
|
||||
dynamic?: boolean | 'strict' | 'runtime';
|
||||
};
|
||||
|
||||
/** Alias changes */
|
||||
aliases?: {
|
||||
add?: Record<string, unknown>;
|
||||
remove?: string[];
|
||||
};
|
||||
|
||||
/** Reindex configuration */
|
||||
reindex?: {
|
||||
source: string;
|
||||
dest: string;
|
||||
script?: string;
|
||||
};
|
||||
};
|
||||
|
||||
/** Rollback changes */
|
||||
rollback?: {
|
||||
settings?: Partial<IndexSettings>;
|
||||
mappings?: {
|
||||
properties?: Record<string, FieldDefinition>;
|
||||
};
|
||||
};
|
||||
|
||||
/** Migration metadata */
|
||||
metadata?: {
|
||||
author?: string;
|
||||
created?: Date;
|
||||
ticket?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Migration status
|
||||
*/
|
||||
export type MigrationStatus = 'pending' | 'running' | 'completed' | 'failed' | 'rolled_back';
|
||||
|
||||
/**
|
||||
* Migration history entry
|
||||
*/
|
||||
export interface MigrationHistoryEntry {
|
||||
/** Migration version */
|
||||
version: number;
|
||||
|
||||
/** Migration name */
|
||||
name: string;
|
||||
|
||||
/** Migration status */
|
||||
status: MigrationStatus;
|
||||
|
||||
/** Started at */
|
||||
startedAt: Date;
|
||||
|
||||
/** Completed at */
|
||||
completedAt?: Date;
|
||||
|
||||
/** Duration in milliseconds */
|
||||
duration?: number;
|
||||
|
||||
/** Error if failed */
|
||||
error?: string;
|
||||
|
||||
/** Checksum of migration */
|
||||
checksum?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Schema manager configuration
|
||||
*/
|
||||
export interface SchemaManagerConfig {
|
||||
/** Index for storing migration history */
|
||||
historyIndex?: string;
|
||||
|
||||
/** Enable dry run mode */
|
||||
dryRun?: boolean;
|
||||
|
||||
/** Enable strict mode (fail on warnings) */
|
||||
strict?: boolean;
|
||||
|
||||
/** Timeout for operations */
|
||||
timeout?: number;
|
||||
|
||||
/** Enable logging */
|
||||
enableLogging?: boolean;
|
||||
|
||||
/** Enable metrics */
|
||||
enableMetrics?: boolean;
|
||||
|
||||
/** Validate schemas before applying */
|
||||
validateBeforeApply?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Schema validation result
|
||||
*/
|
||||
export interface SchemaValidationResult {
|
||||
/** Whether schema is valid */
|
||||
valid: boolean;
|
||||
|
||||
/** Validation errors */
|
||||
errors: Array<{
|
||||
field: string;
|
||||
message: string;
|
||||
severity: 'error' | 'warning';
|
||||
}>;
|
||||
|
||||
/** Validation warnings */
|
||||
warnings: Array<{
|
||||
field: string;
|
||||
message: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Schema diff result
|
||||
*/
|
||||
export interface SchemaDiff {
|
||||
/** Whether schemas are identical */
|
||||
identical: boolean;
|
||||
|
||||
/** Added fields */
|
||||
added: string[];
|
||||
|
||||
/** Removed fields */
|
||||
removed: string[];
|
||||
|
||||
/** Modified fields */
|
||||
modified: Array<{
|
||||
field: string;
|
||||
from: FieldDefinition;
|
||||
to: FieldDefinition;
|
||||
}>;
|
||||
|
||||
/** Settings changes */
|
||||
settingsChanged: boolean;
|
||||
|
||||
/** Aliases changes */
|
||||
aliasesChanged: boolean;
|
||||
|
||||
/** Breaking changes */
|
||||
breakingChanges: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Template definition
|
||||
*/
|
||||
export interface IndexTemplate {
|
||||
/** Template name */
|
||||
name: string;
|
||||
|
||||
/** Index patterns */
|
||||
index_patterns: string[];
|
||||
|
||||
/** Template priority */
|
||||
priority?: number;
|
||||
|
||||
/** Template version */
|
||||
version?: number;
|
||||
|
||||
/** Composed of component templates */
|
||||
composed_of?: string[];
|
||||
|
||||
/** Template settings */
|
||||
template?: {
|
||||
settings?: IndexSettings;
|
||||
mappings?: IndexMapping;
|
||||
aliases?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
/** Data stream settings */
|
||||
data_stream?: {
|
||||
hidden?: boolean;
|
||||
allow_custom_routing?: boolean;
|
||||
};
|
||||
|
||||
/** Template metadata */
|
||||
_meta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Component template
|
||||
*/
|
||||
export interface ComponentTemplate {
|
||||
/** Template name */
|
||||
name: string;
|
||||
|
||||
/** Template content */
|
||||
template: {
|
||||
settings?: IndexSettings;
|
||||
mappings?: IndexMapping;
|
||||
aliases?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
/** Template version */
|
||||
version?: number;
|
||||
|
||||
/** Template metadata */
|
||||
_meta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Schema manager statistics
|
||||
*/
|
||||
export interface SchemaManagerStats {
|
||||
/** Total migrations applied */
|
||||
totalMigrations: number;
|
||||
|
||||
/** Successful migrations */
|
||||
successfulMigrations: number;
|
||||
|
||||
/** Failed migrations */
|
||||
failedMigrations: number;
|
||||
|
||||
/** Rolled back migrations */
|
||||
rolledBackMigrations: number;
|
||||
|
||||
/** Total indices managed */
|
||||
totalIndices: number;
|
||||
|
||||
/** Total templates managed */
|
||||
totalTemplates: number;
|
||||
|
||||
/** Last migration time */
|
||||
lastMigrationTime?: Date;
|
||||
|
||||
/** Average migration duration */
|
||||
avgMigrationDuration: number;
|
||||
}
|
||||
379
ts/examples/schema/schema-example.ts
Normal file
379
ts/examples/schema/schema-example.ts
Normal file
@@ -0,0 +1,379 @@
|
||||
/**
|
||||
* Comprehensive Schema Management Example
|
||||
*
|
||||
* Demonstrates index mapping management, templates, and migrations
|
||||
*/
|
||||
|
||||
import {
|
||||
createConfig,
|
||||
ElasticsearchConnectionManager,
|
||||
LogLevel,
|
||||
createSchemaManager,
|
||||
type IndexSchema,
|
||||
type SchemaMigration,
|
||||
type IndexTemplate,
|
||||
} from '../../index.js';
|
||||
|
||||
async function main() {
|
||||
console.log('=== Schema Management Example ===\n');
|
||||
|
||||
// ============================================================================
|
||||
// Step 1: Configuration
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 1: Configuring Elasticsearch connection...');
|
||||
const config = createConfig()
|
||||
.fromEnv()
|
||||
.nodes(process.env.ELASTICSEARCH_URL || 'http://localhost:9200')
|
||||
.basicAuth(
|
||||
process.env.ELASTICSEARCH_USERNAME || 'elastic',
|
||||
process.env.ELASTICSEARCH_PASSWORD || 'changeme'
|
||||
)
|
||||
.timeout(30000)
|
||||
.retries(3)
|
||||
.logLevel(LogLevel.INFO)
|
||||
.build();
|
||||
|
||||
// ============================================================================
|
||||
// Step 2: Initialize
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 2: Initializing connection and schema manager...');
|
||||
const connectionManager = ElasticsearchConnectionManager.getInstance(config);
|
||||
await connectionManager.initialize();
|
||||
|
||||
const schemaManager = createSchemaManager({
|
||||
historyIndex: '.test_migrations',
|
||||
dryRun: false,
|
||||
strict: true,
|
||||
enableLogging: true,
|
||||
enableMetrics: true,
|
||||
validateBeforeApply: true,
|
||||
});
|
||||
|
||||
await schemaManager.initialize();
|
||||
console.log('✓ Connection and schema manager initialized\n');
|
||||
|
||||
// ============================================================================
|
||||
// Step 3: Create Index with Schema
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 3: Creating index with schema...');
|
||||
|
||||
const productSchema: IndexSchema = {
|
||||
name: 'products-v1',
|
||||
version: 1,
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
number_of_replicas: 0,
|
||||
refresh_interval: '1s',
|
||||
analysis: {
|
||||
analyzer: {
|
||||
product_analyzer: {
|
||||
type: 'custom',
|
||||
tokenizer: 'standard',
|
||||
filter: ['lowercase', 'snowball'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
mappings: {
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
id: { type: 'keyword' },
|
||||
name: {
|
||||
type: 'text',
|
||||
analyzer: 'product_analyzer',
|
||||
properties: {
|
||||
keyword: { type: 'keyword', ignore_above: 256 },
|
||||
},
|
||||
},
|
||||
description: { type: 'text' },
|
||||
price: { type: 'scaled_float', scaling_factor: 100 },
|
||||
category: { type: 'keyword' },
|
||||
tags: { type: 'keyword' },
|
||||
inStock: { type: 'boolean' },
|
||||
createdAt: { type: 'date' },
|
||||
updatedAt: { type: 'date' },
|
||||
},
|
||||
},
|
||||
aliases: {
|
||||
products: { is_write_index: true },
|
||||
},
|
||||
metadata: {
|
||||
description: 'Product catalog index',
|
||||
owner: 'catalog-team',
|
||||
tags: ['catalog', 'products'],
|
||||
},
|
||||
};
|
||||
|
||||
// Validate schema first
|
||||
const validation = schemaManager.validateSchema(productSchema);
|
||||
console.log(` Schema validation: ${validation.valid ? 'PASSED' : 'FAILED'}`);
|
||||
|
||||
if (validation.warnings.length > 0) {
|
||||
console.log(' Warnings:');
|
||||
for (const warning of validation.warnings) {
|
||||
console.log(` - ${warning.field}: ${warning.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Create the index
|
||||
await schemaManager.createIndex(productSchema);
|
||||
console.log(' ✓ Index created with mappings and aliases\n');
|
||||
|
||||
// ============================================================================
|
||||
// Step 4: Schema Migrations
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 4: Running schema migrations...');
|
||||
|
||||
const migrations: SchemaMigration[] = [
|
||||
{
|
||||
version: 1,
|
||||
name: 'add_rating_field',
|
||||
description: 'Add product rating field',
|
||||
type: 'update',
|
||||
index: 'products-v1',
|
||||
changes: {
|
||||
mappings: {
|
||||
properties: {
|
||||
rating: { type: 'float' },
|
||||
reviewCount: { type: 'integer' },
|
||||
},
|
||||
},
|
||||
},
|
||||
rollback: {
|
||||
// Note: Can't remove fields in ES, but document for reference
|
||||
},
|
||||
metadata: {
|
||||
author: 'dev-team',
|
||||
ticket: 'PROD-123',
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 2,
|
||||
name: 'add_brand_field',
|
||||
description: 'Add brand field with keyword type',
|
||||
type: 'update',
|
||||
index: 'products-v1',
|
||||
changes: {
|
||||
mappings: {
|
||||
properties: {
|
||||
brand: { type: 'keyword' },
|
||||
sku: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
},
|
||||
metadata: {
|
||||
author: 'dev-team',
|
||||
ticket: 'PROD-456',
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 3,
|
||||
name: 'add_inventory_object',
|
||||
description: 'Add nested inventory object',
|
||||
type: 'update',
|
||||
index: 'products-v1',
|
||||
changes: {
|
||||
mappings: {
|
||||
properties: {
|
||||
inventory: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
quantity: { type: 'integer' },
|
||||
warehouse: { type: 'keyword' },
|
||||
lastRestocked: { type: 'date' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
metadata: {
|
||||
author: 'inventory-team',
|
||||
ticket: 'INV-789',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const migrationResults = await schemaManager.migrate(migrations);
|
||||
|
||||
console.log(' Migration results:');
|
||||
for (const result of migrationResults) {
|
||||
console.log(` v${result.version} (${result.name}): ${result.status} (${result.duration}ms)`);
|
||||
}
|
||||
|
||||
console.log();
|
||||
|
||||
// ============================================================================
|
||||
// Step 5: Get Applied Migrations
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 5: Viewing migration history...');
|
||||
|
||||
const appliedMigrations = await schemaManager.getAppliedMigrations();
|
||||
|
||||
console.log(' Applied migrations:');
|
||||
for (const migration of appliedMigrations) {
|
||||
console.log(` v${migration.version}: ${migration.name} (${migration.status})`);
|
||||
}
|
||||
|
||||
console.log();
|
||||
|
||||
// ============================================================================
|
||||
// Step 6: Compare Schemas
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 6: Comparing schemas...');
|
||||
|
||||
const oldSchema = productSchema;
|
||||
const newSchema: IndexSchema = {
|
||||
...productSchema,
|
||||
version: 2,
|
||||
mappings: {
|
||||
...productSchema.mappings,
|
||||
properties: {
|
||||
...productSchema.mappings.properties,
|
||||
rating: { type: 'float' },
|
||||
discount: { type: 'float' },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const diff = schemaManager.diffSchemas(oldSchema, newSchema);
|
||||
|
||||
console.log(' Schema diff:');
|
||||
console.log(` Identical: ${diff.identical}`);
|
||||
console.log(` Added fields: ${diff.added.join(', ') || 'none'}`);
|
||||
console.log(` Removed fields: ${diff.removed.join(', ') || 'none'}`);
|
||||
console.log(` Modified fields: ${diff.modified.length}`);
|
||||
console.log(` Breaking changes: ${diff.breakingChanges.length}`);
|
||||
|
||||
console.log();
|
||||
|
||||
// ============================================================================
|
||||
// Step 7: Index Templates
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 7: Managing index templates...');
|
||||
|
||||
const template: IndexTemplate = {
|
||||
name: 'products-template',
|
||||
index_patterns: ['products-*'],
|
||||
priority: 100,
|
||||
version: 1,
|
||||
template: {
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
number_of_replicas: 0,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
id: { type: 'keyword' },
|
||||
name: { type: 'text' },
|
||||
createdAt: { type: 'date' },
|
||||
},
|
||||
},
|
||||
aliases: {
|
||||
'all-products': {},
|
||||
},
|
||||
},
|
||||
_meta: {
|
||||
description: 'Template for product indices',
|
||||
},
|
||||
};
|
||||
|
||||
await schemaManager.putTemplate(template);
|
||||
console.log(' ✓ Index template created');
|
||||
|
||||
// Get template
|
||||
const retrievedTemplate = await schemaManager.getTemplate('products-template');
|
||||
console.log(` Template patterns: ${retrievedTemplate?.index_patterns.join(', ')}`);
|
||||
|
||||
console.log();
|
||||
|
||||
// ============================================================================
|
||||
// Step 8: Alias Management
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 8: Managing aliases...');
|
||||
|
||||
// Add alias
|
||||
await schemaManager.addAlias('products-v1', 'products-read', {
|
||||
filter: { term: { inStock: true } },
|
||||
});
|
||||
console.log(' ✓ Added filtered read alias');
|
||||
|
||||
// Get schema to see aliases
|
||||
const currentSchema = await schemaManager.getSchema('products-v1');
|
||||
console.log(` Current aliases: ${Object.keys(currentSchema?.aliases || {}).join(', ')}`);
|
||||
|
||||
console.log();
|
||||
|
||||
// ============================================================================
|
||||
// Step 9: Update Settings
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 9: Updating index settings...');
|
||||
|
||||
await schemaManager.updateSettings('products-v1', {
|
||||
refresh_interval: '5s',
|
||||
max_result_window: 20000,
|
||||
});
|
||||
|
||||
console.log(' ✓ Settings updated');
|
||||
|
||||
console.log();
|
||||
|
||||
// ============================================================================
|
||||
// Step 10: Statistics
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 10: Schema manager statistics...\n');
|
||||
|
||||
const stats = schemaManager.getStats();
|
||||
|
||||
console.log('Schema Manager Statistics:');
|
||||
console.log(` Total migrations: ${stats.totalMigrations}`);
|
||||
console.log(` Successful: ${stats.successfulMigrations}`);
|
||||
console.log(` Failed: ${stats.failedMigrations}`);
|
||||
console.log(` Rolled back: ${stats.rolledBackMigrations}`);
|
||||
console.log(` Total indices: ${stats.totalIndices}`);
|
||||
console.log(` Total templates: ${stats.totalTemplates}`);
|
||||
console.log(` Avg migration duration: ${stats.avgMigrationDuration.toFixed(2)}ms`);
|
||||
|
||||
console.log();
|
||||
|
||||
// ============================================================================
|
||||
// Step 11: Cleanup
|
||||
// ============================================================================
|
||||
|
||||
console.log('Step 11: Cleanup...');
|
||||
|
||||
await schemaManager.deleteTemplate('products-template');
|
||||
await schemaManager.deleteIndex('products-v1');
|
||||
await connectionManager.destroy();
|
||||
|
||||
console.log('✓ Cleanup complete\n');
|
||||
|
||||
console.log('=== Schema Management Example Complete ===');
|
||||
console.log('\nKey Features Demonstrated:');
|
||||
console.log(' ✓ Index creation with mappings and settings');
|
||||
console.log(' ✓ Schema validation before apply');
|
||||
console.log(' ✓ Versioned migrations with history tracking');
|
||||
console.log(' ✓ Schema diff and comparison');
|
||||
console.log(' ✓ Index templates');
|
||||
console.log(' ✓ Alias management with filters');
|
||||
console.log(' ✓ Settings updates');
|
||||
console.log(' ✓ Migration rollback support');
|
||||
console.log(' ✓ Dry run mode');
|
||||
console.log(' ✓ Comprehensive statistics');
|
||||
}
|
||||
|
||||
// Run the example
|
||||
main().catch((error) => {
|
||||
console.error('Example failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
21
ts/index.ts
21
ts/index.ts
@@ -14,6 +14,7 @@ export * from './domain/logging/index.js';
|
||||
export * from './domain/bulk/index.js';
|
||||
export * from './domain/kv/index.js';
|
||||
export * from './domain/transactions/index.js';
|
||||
export * from './domain/schema/index.js';
|
||||
|
||||
// Re-export commonly used items for convenience
|
||||
export {
|
||||
@@ -172,3 +173,23 @@ export {
|
||||
type Savepoint,
|
||||
type TransactionCallbacks,
|
||||
} from './domain/transactions/index.js';
|
||||
|
||||
export {
|
||||
// Schema
|
||||
SchemaManager,
|
||||
createSchemaManager,
|
||||
type FieldType,
|
||||
type FieldDefinition,
|
||||
type IndexSettings,
|
||||
type IndexMapping,
|
||||
type IndexSchema,
|
||||
type SchemaMigration,
|
||||
type MigrationStatus,
|
||||
type MigrationHistoryEntry,
|
||||
type SchemaManagerConfig,
|
||||
type SchemaValidationResult,
|
||||
type SchemaDiff,
|
||||
type IndexTemplate,
|
||||
type ComponentTemplate,
|
||||
type SchemaManagerStats,
|
||||
} from './domain/schema/index.js';
|
||||
|
||||
Reference in New Issue
Block a user