feat(schema-manager): Add Schema Management module and expose it in public API; update README to mark Phase 3 complete and move priorities to Phase 4
This commit is contained in:
@@ -1,5 +1,12 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2025-11-29 - 3.1.0 - feat(schema-manager)
|
||||||
|
Add Schema Management module and expose it in public API; update README to mark Phase 3 complete and move priorities to Phase 4
|
||||||
|
|
||||||
|
- Add ts/domain/schema/index.ts to export SchemaManager and related types
|
||||||
|
- Re-export SchemaManager and schema types from top-level ts/index.ts so schema APIs are part of the public surface
|
||||||
|
- Update README hints: mark Phase 3 Advanced Features complete (Transaction Support and Schema Management), add schema/transaction example references, and update Next Priorities to Phase 4 (Comprehensive Test Suite, Migration Guide, README Update)
|
||||||
|
|
||||||
## 2025-11-29 - 3.0.0 - BREAKING CHANGE(core)
|
## 2025-11-29 - 3.0.0 - BREAKING CHANGE(core)
|
||||||
Refactor to v3: introduce modular core/domain architecture, plugin system, observability and strict TypeScript configuration; remove legacy classes
|
Refactor to v3: introduce modular core/domain architecture, plugin system, observability and strict TypeScript configuration; remove legacy classes
|
||||||
|
|
||||||
|
|||||||
@@ -42,7 +42,7 @@
|
|||||||
- Cache hit/miss statistics
|
- Cache hit/miss statistics
|
||||||
- Example at `ts/examples/kv/kv-store-example.ts`
|
- Example at `ts/examples/kv/kv-store-example.ts`
|
||||||
|
|
||||||
**Phase 3: Advanced Features**
|
**Phase 3: Advanced Features (100%)** - Complete!
|
||||||
- **Plugin Architecture (100%)** - Extensible request/response middleware:
|
- **Plugin Architecture (100%)** - Extensible request/response middleware:
|
||||||
- Plugin lifecycle hooks (beforeRequest, afterResponse, onError)
|
- Plugin lifecycle hooks (beforeRequest, afterResponse, onError)
|
||||||
- Plugin priority and execution ordering
|
- Plugin priority and execution ordering
|
||||||
@@ -58,6 +58,30 @@
|
|||||||
- Shared context between plugins
|
- Shared context between plugins
|
||||||
- Timeout protection for plugin hooks
|
- Timeout protection for plugin hooks
|
||||||
- Example at `ts/examples/plugins/plugin-example.ts`
|
- Example at `ts/examples/plugins/plugin-example.ts`
|
||||||
|
- **Transaction Support (100%)** - ACID-like distributed transactions:
|
||||||
|
- Optimistic concurrency control (seq_no/primary_term)
|
||||||
|
- Automatic rollback with compensation operations
|
||||||
|
- Savepoints for partial rollback
|
||||||
|
- Conflict detection and configurable resolution (retry, abort, skip, force)
|
||||||
|
- Multi-document atomic operations
|
||||||
|
- Isolation levels (read_uncommitted, read_committed, repeatable_read)
|
||||||
|
- Transaction callbacks (onBegin, onCommit, onRollback, onConflict)
|
||||||
|
- Transaction statistics and monitoring
|
||||||
|
- Timeout with automatic cleanup
|
||||||
|
- Example at `ts/examples/transactions/transaction-example.ts`
|
||||||
|
- **Schema Management (100%)** - Index mapping management and migrations:
|
||||||
|
- Index creation with mappings, settings, and aliases
|
||||||
|
- Schema validation before apply
|
||||||
|
- Versioned migrations with history tracking
|
||||||
|
- Schema diff and comparison (added/removed/modified fields)
|
||||||
|
- Breaking change detection
|
||||||
|
- Index templates (composable and legacy)
|
||||||
|
- Component templates
|
||||||
|
- Alias management with filters
|
||||||
|
- Dynamic settings updates
|
||||||
|
- Migration rollback support
|
||||||
|
- Dry run mode for testing
|
||||||
|
- Example at `ts/examples/schema/schema-example.ts`
|
||||||
|
|
||||||
### Query Builder Usage
|
### Query Builder Usage
|
||||||
|
|
||||||
@@ -81,11 +105,10 @@ const stats = await createQuery<Product>('products')
|
|||||||
.execute();
|
.execute();
|
||||||
```
|
```
|
||||||
|
|
||||||
### Next Priorities
|
### Next Priorities (Phase 4)
|
||||||
1. **Transaction Support** (Phase 3) - Distributed transactions across multiple documents
|
1. **Comprehensive Test Suite** - Full test coverage for all modules
|
||||||
2. **Schema Management** (Phase 3) - Index mapping management and migrations
|
2. **Migration Guide** - Guide from v2 to v3 API
|
||||||
3. **Comprehensive Test Suite** (Phase 4) - Full test coverage
|
3. **README Update** - Update main README with new v3 API documentation
|
||||||
4. **Migration Guide** (Phase 4) - Guide from v2 to v3
|
|
||||||
|
|
||||||
### Structure
|
### Structure
|
||||||
- Single `ts/` directory (no parallel structures)
|
- Single `ts/` directory (no parallel structures)
|
||||||
|
|||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@apiclient.xyz/elasticsearch',
|
name: '@apiclient.xyz/elasticsearch',
|
||||||
version: '3.0.0',
|
version: '3.1.0',
|
||||||
description: 'log to elasticsearch in a kibana compatible format'
|
description: 'log to elasticsearch in a kibana compatible format'
|
||||||
}
|
}
|
||||||
|
|||||||
26
ts/domain/schema/index.ts
Normal file
26
ts/domain/schema/index.ts
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
/**
|
||||||
|
* Schema Management Module
|
||||||
|
*
|
||||||
|
* Index mapping management, templates, and migrations
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Main classes
|
||||||
|
export { SchemaManager, createSchemaManager } from './schema-manager.js';
|
||||||
|
|
||||||
|
// Types
|
||||||
|
export type {
|
||||||
|
FieldType,
|
||||||
|
FieldDefinition,
|
||||||
|
IndexSettings,
|
||||||
|
IndexMapping,
|
||||||
|
IndexSchema,
|
||||||
|
SchemaMigration,
|
||||||
|
MigrationStatus,
|
||||||
|
MigrationHistoryEntry,
|
||||||
|
SchemaManagerConfig,
|
||||||
|
SchemaValidationResult,
|
||||||
|
SchemaDiff,
|
||||||
|
IndexTemplate,
|
||||||
|
ComponentTemplate,
|
||||||
|
SchemaManagerStats,
|
||||||
|
} from './types.js';
|
||||||
926
ts/domain/schema/schema-manager.ts
Normal file
926
ts/domain/schema/schema-manager.ts
Normal file
@@ -0,0 +1,926 @@
|
|||||||
|
/**
|
||||||
|
* Schema Manager
|
||||||
|
*
|
||||||
|
* Index mapping management, templates, and migrations
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { ElasticsearchConnectionManager } from '../../core/connection/connection-manager.js';
|
||||||
|
import { Logger, defaultLogger } from '../../core/observability/logger.js';
|
||||||
|
import { MetricsCollector, defaultMetricsCollector } from '../../core/observability/metrics.js';
|
||||||
|
import type {
|
||||||
|
IndexSchema,
|
||||||
|
IndexSettings,
|
||||||
|
IndexMapping,
|
||||||
|
FieldDefinition,
|
||||||
|
SchemaMigration,
|
||||||
|
MigrationHistoryEntry,
|
||||||
|
MigrationStatus,
|
||||||
|
SchemaManagerConfig,
|
||||||
|
SchemaValidationResult,
|
||||||
|
SchemaDiff,
|
||||||
|
IndexTemplate,
|
||||||
|
ComponentTemplate,
|
||||||
|
SchemaManagerStats,
|
||||||
|
} from './types.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Default configuration
|
||||||
|
*/
|
||||||
|
const DEFAULT_CONFIG: Required<SchemaManagerConfig> = {
|
||||||
|
historyIndex: '.schema_migrations',
|
||||||
|
dryRun: false,
|
||||||
|
strict: false,
|
||||||
|
timeout: 60000, // 1 minute
|
||||||
|
enableLogging: true,
|
||||||
|
enableMetrics: true,
|
||||||
|
validateBeforeApply: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema Manager
|
||||||
|
*/
|
||||||
|
export class SchemaManager {
|
||||||
|
private config: Required<SchemaManagerConfig>;
|
||||||
|
private stats: SchemaManagerStats;
|
||||||
|
private logger: Logger;
|
||||||
|
private metrics: MetricsCollector;
|
||||||
|
|
||||||
|
constructor(config: SchemaManagerConfig = {}) {
|
||||||
|
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||||
|
this.logger = defaultLogger;
|
||||||
|
this.metrics = defaultMetricsCollector;
|
||||||
|
|
||||||
|
this.stats = {
|
||||||
|
totalMigrations: 0,
|
||||||
|
successfulMigrations: 0,
|
||||||
|
failedMigrations: 0,
|
||||||
|
rolledBackMigrations: 0,
|
||||||
|
totalIndices: 0,
|
||||||
|
totalTemplates: 0,
|
||||||
|
avgMigrationDuration: 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize schema manager
|
||||||
|
*/
|
||||||
|
async initialize(): Promise<void> {
|
||||||
|
await this.ensureHistoryIndex();
|
||||||
|
|
||||||
|
this.logger.info('SchemaManager initialized', {
|
||||||
|
historyIndex: this.config.historyIndex,
|
||||||
|
dryRun: this.config.dryRun,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Index Management
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create an index with schema
|
||||||
|
*/
|
||||||
|
async createIndex(schema: IndexSchema): Promise<void> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
if (this.config.validateBeforeApply) {
|
||||||
|
const validation = this.validateSchema(schema);
|
||||||
|
if (!validation.valid) {
|
||||||
|
throw new Error(`Schema validation failed: ${validation.errors.map((e) => e.message).join(', ')}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would create index', { index: schema.name });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
const body: any = {
|
||||||
|
settings: schema.settings,
|
||||||
|
mappings: schema.mappings,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (schema.aliases) {
|
||||||
|
body.aliases = schema.aliases;
|
||||||
|
}
|
||||||
|
|
||||||
|
await client.indices.create({
|
||||||
|
index: schema.name,
|
||||||
|
...body,
|
||||||
|
timeout: `${this.config.timeout}ms`,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.stats.totalIndices++;
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Index created', {
|
||||||
|
index: schema.name,
|
||||||
|
version: schema.version,
|
||||||
|
duration: Date.now() - startTime,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableMetrics) {
|
||||||
|
this.metrics.recordCounter('schema.indices.created', 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update index mapping
|
||||||
|
*/
|
||||||
|
async updateMapping(index: string, mapping: Partial<IndexMapping>): Promise<void> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would update mapping', { index });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
await client.indices.putMapping({
|
||||||
|
index,
|
||||||
|
...mapping,
|
||||||
|
timeout: `${this.config.timeout}ms`,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Mapping updated', {
|
||||||
|
index,
|
||||||
|
duration: Date.now() - startTime,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableMetrics) {
|
||||||
|
this.metrics.recordCounter('schema.mappings.updated', 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update index settings
|
||||||
|
*/
|
||||||
|
async updateSettings(index: string, settings: Partial<IndexSettings>): Promise<void> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would update settings', { index });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
// Some settings require closing the index
|
||||||
|
const requiresClose = this.settingsRequireClose(settings);
|
||||||
|
|
||||||
|
if (requiresClose) {
|
||||||
|
await client.indices.close({ index });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await client.indices.putSettings({
|
||||||
|
index,
|
||||||
|
settings,
|
||||||
|
timeout: `${this.config.timeout}ms`,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
if (requiresClose) {
|
||||||
|
await client.indices.open({ index });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Settings updated', {
|
||||||
|
index,
|
||||||
|
duration: Date.now() - startTime,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableMetrics) {
|
||||||
|
this.metrics.recordCounter('schema.settings.updated', 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get index schema
|
||||||
|
*/
|
||||||
|
async getSchema(index: string): Promise<IndexSchema | null> {
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const [mappingResult, settingsResult, aliasResult] = await Promise.all([
|
||||||
|
client.indices.getMapping({ index }),
|
||||||
|
client.indices.getSettings({ index }),
|
||||||
|
client.indices.getAlias({ index }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const indexData = mappingResult[index];
|
||||||
|
const settingsData = settingsResult[index];
|
||||||
|
const aliasData = aliasResult[index];
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: index,
|
||||||
|
version: 0, // Version not stored in ES by default
|
||||||
|
settings: settingsData?.settings?.index as IndexSettings,
|
||||||
|
mappings: indexData?.mappings as IndexMapping,
|
||||||
|
aliases: aliasData?.aliases,
|
||||||
|
};
|
||||||
|
} catch (error: any) {
|
||||||
|
if (error.meta?.statusCode === 404) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete an index
|
||||||
|
*/
|
||||||
|
async deleteIndex(index: string): Promise<void> {
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would delete index', { index });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
await client.indices.delete({
|
||||||
|
index,
|
||||||
|
timeout: `${this.config.timeout}ms`,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.stats.totalIndices--;
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Index deleted', { index });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableMetrics) {
|
||||||
|
this.metrics.recordCounter('schema.indices.deleted', 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if index exists
|
||||||
|
*/
|
||||||
|
async indexExists(index: string): Promise<boolean> {
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
return await client.indices.exists({ index });
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Migration Management
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run migrations
|
||||||
|
*/
|
||||||
|
async migrate(migrations: SchemaMigration[]): Promise<MigrationHistoryEntry[]> {
|
||||||
|
const results: MigrationHistoryEntry[] = [];
|
||||||
|
|
||||||
|
// Get current migration state
|
||||||
|
const appliedVersions = await this.getAppliedMigrations();
|
||||||
|
const appliedSet = new Set(appliedVersions.map((m) => m.version));
|
||||||
|
|
||||||
|
// Sort migrations by version
|
||||||
|
const sortedMigrations = [...migrations].sort((a, b) => a.version - b.version);
|
||||||
|
|
||||||
|
// Filter pending migrations
|
||||||
|
const pendingMigrations = sortedMigrations.filter((m) => !appliedSet.has(m.version));
|
||||||
|
|
||||||
|
if (pendingMigrations.length === 0) {
|
||||||
|
this.logger.info('No pending migrations');
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.info(`Running ${pendingMigrations.length} migrations`);
|
||||||
|
|
||||||
|
for (const migration of pendingMigrations) {
|
||||||
|
const result = await this.runMigration(migration);
|
||||||
|
results.push(result);
|
||||||
|
|
||||||
|
if (result.status === 'failed') {
|
||||||
|
this.logger.error('Migration failed, stopping', {
|
||||||
|
version: migration.version,
|
||||||
|
name: migration.name,
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a single migration
|
||||||
|
*/
|
||||||
|
private async runMigration(migration: SchemaMigration): Promise<MigrationHistoryEntry> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
const entry: MigrationHistoryEntry = {
|
||||||
|
version: migration.version,
|
||||||
|
name: migration.name,
|
||||||
|
status: 'running',
|
||||||
|
startedAt: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
this.stats.totalMigrations++;
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Running migration', {
|
||||||
|
version: migration.version,
|
||||||
|
name: migration.name,
|
||||||
|
type: migration.type,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.applyMigration(migration);
|
||||||
|
|
||||||
|
entry.status = 'completed';
|
||||||
|
entry.completedAt = new Date();
|
||||||
|
entry.duration = Date.now() - startTime;
|
||||||
|
|
||||||
|
this.stats.successfulMigrations++;
|
||||||
|
this.stats.lastMigrationTime = new Date();
|
||||||
|
this.updateAvgDuration(entry.duration);
|
||||||
|
|
||||||
|
// Record in history
|
||||||
|
await this.recordMigration(entry);
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Migration completed', {
|
||||||
|
version: migration.version,
|
||||||
|
name: migration.name,
|
||||||
|
duration: entry.duration,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableMetrics) {
|
||||||
|
this.metrics.recordCounter('schema.migrations.success', 1);
|
||||||
|
this.metrics.recordHistogram('schema.migrations.duration', entry.duration);
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
entry.status = 'failed';
|
||||||
|
entry.completedAt = new Date();
|
||||||
|
entry.duration = Date.now() - startTime;
|
||||||
|
entry.error = error.message;
|
||||||
|
|
||||||
|
this.stats.failedMigrations++;
|
||||||
|
|
||||||
|
// Record failure in history
|
||||||
|
await this.recordMigration(entry);
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.error('Migration failed', {
|
||||||
|
version: migration.version,
|
||||||
|
name: migration.name,
|
||||||
|
error: error.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableMetrics) {
|
||||||
|
this.metrics.recordCounter('schema.migrations.failed', 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply migration changes
|
||||||
|
*/
|
||||||
|
private async applyMigration(migration: SchemaMigration): Promise<void> {
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would apply migration', {
|
||||||
|
version: migration.version,
|
||||||
|
name: migration.name,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
switch (migration.type) {
|
||||||
|
case 'create':
|
||||||
|
await this.createIndex({
|
||||||
|
name: migration.index,
|
||||||
|
version: migration.version,
|
||||||
|
settings: migration.changes.settings,
|
||||||
|
mappings: migration.changes.mappings as IndexMapping,
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'update':
|
||||||
|
if (migration.changes.settings) {
|
||||||
|
await this.updateSettings(migration.index, migration.changes.settings);
|
||||||
|
}
|
||||||
|
if (migration.changes.mappings) {
|
||||||
|
await this.updateMapping(migration.index, migration.changes.mappings);
|
||||||
|
}
|
||||||
|
if (migration.changes.aliases) {
|
||||||
|
await this.updateAliases(migration.index, migration.changes.aliases);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'delete':
|
||||||
|
await this.deleteIndex(migration.index);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'reindex':
|
||||||
|
if (migration.changes.reindex) {
|
||||||
|
await client.reindex({
|
||||||
|
source: { index: migration.changes.reindex.source },
|
||||||
|
dest: { index: migration.changes.reindex.dest },
|
||||||
|
script: migration.changes.reindex.script
|
||||||
|
? { source: migration.changes.reindex.script }
|
||||||
|
: undefined,
|
||||||
|
timeout: `${this.config.timeout}ms`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'alias':
|
||||||
|
if (migration.changes.aliases) {
|
||||||
|
await this.updateAliases(migration.index, migration.changes.aliases);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rollback a migration
|
||||||
|
*/
|
||||||
|
async rollback(version: number, migrations: SchemaMigration[]): Promise<MigrationHistoryEntry> {
|
||||||
|
const migration = migrations.find((m) => m.version === version);
|
||||||
|
|
||||||
|
if (!migration) {
|
||||||
|
throw new Error(`Migration version ${version} not found`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!migration.rollback) {
|
||||||
|
throw new Error(`Migration version ${version} has no rollback defined`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const startTime = Date.now();
|
||||||
|
const entry: MigrationHistoryEntry = {
|
||||||
|
version,
|
||||||
|
name: `rollback_${migration.name}`,
|
||||||
|
status: 'running',
|
||||||
|
startedAt: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Apply rollback changes
|
||||||
|
if (migration.rollback.settings) {
|
||||||
|
await this.updateSettings(migration.index, migration.rollback.settings);
|
||||||
|
}
|
||||||
|
if (migration.rollback.mappings) {
|
||||||
|
await this.updateMapping(migration.index, migration.rollback.mappings);
|
||||||
|
}
|
||||||
|
|
||||||
|
entry.status = 'rolled_back';
|
||||||
|
entry.completedAt = new Date();
|
||||||
|
entry.duration = Date.now() - startTime;
|
||||||
|
|
||||||
|
this.stats.rolledBackMigrations++;
|
||||||
|
|
||||||
|
// Update history
|
||||||
|
await this.recordMigration(entry);
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Migration rolled back', {
|
||||||
|
version,
|
||||||
|
name: migration.name,
|
||||||
|
duration: entry.duration,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableMetrics) {
|
||||||
|
this.metrics.recordCounter('schema.migrations.rollback', 1);
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
entry.status = 'failed';
|
||||||
|
entry.error = error.message;
|
||||||
|
|
||||||
|
this.logger.error('Rollback failed', {
|
||||||
|
version,
|
||||||
|
name: migration.name,
|
||||||
|
error: error.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get applied migrations
|
||||||
|
*/
|
||||||
|
async getAppliedMigrations(): Promise<MigrationHistoryEntry[]> {
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await client.search({
|
||||||
|
index: this.config.historyIndex,
|
||||||
|
size: 1000,
|
||||||
|
sort: [{ version: 'asc' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
return result.hits.hits.map((hit) => hit._source as MigrationHistoryEntry);
|
||||||
|
} catch (error: any) {
|
||||||
|
if (error.meta?.statusCode === 404) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Template Management
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create or update index template
|
||||||
|
*/
|
||||||
|
async putTemplate(template: IndexTemplate): Promise<void> {
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would put template', { name: template.name });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
await client.indices.putIndexTemplate({
|
||||||
|
name: template.name,
|
||||||
|
index_patterns: template.index_patterns,
|
||||||
|
priority: template.priority,
|
||||||
|
version: template.version,
|
||||||
|
composed_of: template.composed_of,
|
||||||
|
template: template.template,
|
||||||
|
data_stream: template.data_stream,
|
||||||
|
_meta: template._meta,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.stats.totalTemplates++;
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Template created/updated', { name: template.name });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableMetrics) {
|
||||||
|
this.metrics.recordCounter('schema.templates.created', 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get index template
|
||||||
|
*/
|
||||||
|
async getTemplate(name: string): Promise<IndexTemplate | null> {
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await client.indices.getIndexTemplate({ name });
|
||||||
|
const template = result.index_templates[0];
|
||||||
|
|
||||||
|
if (!template) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: template.name,
|
||||||
|
index_patterns: template.index_template.index_patterns,
|
||||||
|
priority: template.index_template.priority,
|
||||||
|
version: template.index_template.version,
|
||||||
|
composed_of: template.index_template.composed_of,
|
||||||
|
template: template.index_template.template,
|
||||||
|
data_stream: template.index_template.data_stream,
|
||||||
|
_meta: template.index_template._meta,
|
||||||
|
};
|
||||||
|
} catch (error: any) {
|
||||||
|
if (error.meta?.statusCode === 404) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete index template
|
||||||
|
*/
|
||||||
|
async deleteTemplate(name: string): Promise<void> {
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would delete template', { name });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
await client.indices.deleteIndexTemplate({ name });
|
||||||
|
|
||||||
|
this.stats.totalTemplates--;
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Template deleted', { name });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create or update component template
|
||||||
|
*/
|
||||||
|
async putComponentTemplate(template: ComponentTemplate): Promise<void> {
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would put component template', { name: template.name });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
await client.cluster.putComponentTemplate({
|
||||||
|
name: template.name,
|
||||||
|
template: template.template,
|
||||||
|
version: template.version,
|
||||||
|
_meta: template._meta,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Component template created/updated', { name: template.name });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Alias Management
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update aliases
|
||||||
|
*/
|
||||||
|
async updateAliases(
|
||||||
|
index: string,
|
||||||
|
aliases: { add?: Record<string, unknown>; remove?: string[] }
|
||||||
|
): Promise<void> {
|
||||||
|
if (this.config.dryRun) {
|
||||||
|
this.logger.info('[DRY RUN] Would update aliases', { index });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
const actions: any[] = [];
|
||||||
|
|
||||||
|
if (aliases.add) {
|
||||||
|
for (const [alias, config] of Object.entries(aliases.add)) {
|
||||||
|
actions.push({ add: { index, alias, ...config } });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (aliases.remove) {
|
||||||
|
for (const alias of aliases.remove) {
|
||||||
|
actions.push({ remove: { index, alias } });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (actions.length > 0) {
|
||||||
|
await client.indices.updateAliases({ actions });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.config.enableLogging) {
|
||||||
|
this.logger.info('Aliases updated', { index, actions: actions.length });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add alias
|
||||||
|
*/
|
||||||
|
async addAlias(index: string, alias: string, config: Record<string, unknown> = {}): Promise<void> {
|
||||||
|
await this.updateAliases(index, { add: { [alias]: config } });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove alias
|
||||||
|
*/
|
||||||
|
async removeAlias(index: string, alias: string): Promise<void> {
|
||||||
|
await this.updateAliases(index, { remove: [alias] });
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Schema Validation and Comparison
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate schema
|
||||||
|
*/
|
||||||
|
validateSchema(schema: IndexSchema): SchemaValidationResult {
|
||||||
|
const errors: SchemaValidationResult['errors'] = [];
|
||||||
|
const warnings: SchemaValidationResult['warnings'] = [];
|
||||||
|
|
||||||
|
// Validate index name
|
||||||
|
if (!schema.name || schema.name.length === 0) {
|
||||||
|
errors.push({ field: 'name', message: 'Index name is required', severity: 'error' });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (schema.name && schema.name.startsWith('_')) {
|
||||||
|
errors.push({ field: 'name', message: 'Index name cannot start with _', severity: 'error' });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate mappings
|
||||||
|
if (!schema.mappings || !schema.mappings.properties) {
|
||||||
|
errors.push({ field: 'mappings', message: 'Mappings with properties are required', severity: 'error' });
|
||||||
|
} else {
|
||||||
|
this.validateFields(schema.mappings.properties, '', errors, warnings);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate settings
|
||||||
|
if (schema.settings) {
|
||||||
|
if (schema.settings.number_of_shards !== undefined && schema.settings.number_of_shards < 1) {
|
||||||
|
errors.push({ field: 'settings.number_of_shards', message: 'Must be at least 1', severity: 'error' });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (schema.settings.number_of_replicas !== undefined && schema.settings.number_of_replicas < 0) {
|
||||||
|
errors.push({ field: 'settings.number_of_replicas', message: 'Cannot be negative', severity: 'error' });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
valid: errors.length === 0,
|
||||||
|
errors,
|
||||||
|
warnings,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compare two schemas
|
||||||
|
*/
|
||||||
|
diffSchemas(oldSchema: IndexSchema, newSchema: IndexSchema): SchemaDiff {
|
||||||
|
const added: string[] = [];
|
||||||
|
const removed: string[] = [];
|
||||||
|
const modified: SchemaDiff['modified'] = [];
|
||||||
|
const breakingChanges: string[] = [];
|
||||||
|
|
||||||
|
// Compare fields
|
||||||
|
const oldFields = this.flattenFields(oldSchema.mappings?.properties || {});
|
||||||
|
const newFields = this.flattenFields(newSchema.mappings?.properties || {});
|
||||||
|
|
||||||
|
for (const field of newFields.keys()) {
|
||||||
|
if (!oldFields.has(field)) {
|
||||||
|
added.push(field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const field of oldFields.keys()) {
|
||||||
|
if (!newFields.has(field)) {
|
||||||
|
removed.push(field);
|
||||||
|
breakingChanges.push(`Removed field: ${field}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [field, newDef] of newFields) {
|
||||||
|
const oldDef = oldFields.get(field);
|
||||||
|
if (oldDef && JSON.stringify(oldDef) !== JSON.stringify(newDef)) {
|
||||||
|
modified.push({ field, from: oldDef, to: newDef });
|
||||||
|
|
||||||
|
// Check for breaking changes
|
||||||
|
if (oldDef.type !== newDef.type) {
|
||||||
|
breakingChanges.push(`Field type changed: ${field} (${oldDef.type} -> ${newDef.type})`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
identical: added.length === 0 && removed.length === 0 && modified.length === 0,
|
||||||
|
added,
|
||||||
|
removed,
|
||||||
|
modified,
|
||||||
|
settingsChanged: JSON.stringify(oldSchema.settings) !== JSON.stringify(newSchema.settings),
|
||||||
|
aliasesChanged: JSON.stringify(oldSchema.aliases) !== JSON.stringify(newSchema.aliases),
|
||||||
|
breakingChanges,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get statistics
|
||||||
|
*/
|
||||||
|
getStats(): SchemaManagerStats {
|
||||||
|
return { ...this.stats };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Private Methods
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure history index exists
|
||||||
|
*/
|
||||||
|
private async ensureHistoryIndex(): Promise<void> {
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
const exists = await client.indices.exists({ index: this.config.historyIndex });
|
||||||
|
|
||||||
|
if (!exists) {
|
||||||
|
await client.indices.create({
|
||||||
|
index: this.config.historyIndex,
|
||||||
|
mappings: {
|
||||||
|
properties: {
|
||||||
|
version: { type: 'integer' },
|
||||||
|
name: { type: 'keyword' },
|
||||||
|
status: { type: 'keyword' },
|
||||||
|
startedAt: { type: 'date' },
|
||||||
|
completedAt: { type: 'date' },
|
||||||
|
duration: { type: 'long' },
|
||||||
|
error: { type: 'text' },
|
||||||
|
checksum: { type: 'keyword' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record migration in history
|
||||||
|
*/
|
||||||
|
private async recordMigration(entry: MigrationHistoryEntry): Promise<void> {
|
||||||
|
const client = ElasticsearchConnectionManager.getInstance().getClient();
|
||||||
|
|
||||||
|
await client.index({
|
||||||
|
index: this.config.historyIndex,
|
||||||
|
id: `migration-${entry.version}`,
|
||||||
|
document: entry,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if settings require closing the index
|
||||||
|
*/
|
||||||
|
private settingsRequireClose(settings: Partial<IndexSettings>): boolean {
|
||||||
|
const closeRequired = ['number_of_shards', 'codec', 'routing_partition_size'];
|
||||||
|
return Object.keys(settings).some((key) => closeRequired.includes(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate fields recursively
|
||||||
|
*/
|
||||||
|
private validateFields(
|
||||||
|
fields: Record<string, FieldDefinition>,
|
||||||
|
prefix: string,
|
||||||
|
errors: SchemaValidationResult['errors'],
|
||||||
|
warnings: SchemaValidationResult['warnings']
|
||||||
|
): void {
|
||||||
|
for (const [name, def] of Object.entries(fields)) {
|
||||||
|
const path = prefix ? `${prefix}.${name}` : name;
|
||||||
|
|
||||||
|
if (!def.type) {
|
||||||
|
errors.push({ field: path, message: 'Field type is required', severity: 'error' });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for text fields without keyword sub-field
|
||||||
|
if (def.type === 'text' && !def.properties) {
|
||||||
|
warnings.push({
|
||||||
|
field: path,
|
||||||
|
message: 'Text field without keyword sub-field may limit aggregation capabilities',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate nested properties
|
||||||
|
if (def.properties) {
|
||||||
|
this.validateFields(def.properties, path, errors, warnings);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flatten nested fields
|
||||||
|
*/
|
||||||
|
private flattenFields(
|
||||||
|
fields: Record<string, FieldDefinition>,
|
||||||
|
prefix: string = ''
|
||||||
|
): Map<string, FieldDefinition> {
|
||||||
|
const result = new Map<string, FieldDefinition>();
|
||||||
|
|
||||||
|
for (const [name, def] of Object.entries(fields)) {
|
||||||
|
const path = prefix ? `${prefix}.${name}` : name;
|
||||||
|
result.set(path, def);
|
||||||
|
|
||||||
|
if (def.properties) {
|
||||||
|
const nested = this.flattenFields(def.properties, path);
|
||||||
|
for (const [nestedPath, nestedDef] of nested) {
|
||||||
|
result.set(nestedPath, nestedDef);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update average migration duration
|
||||||
|
*/
|
||||||
|
private updateAvgDuration(duration: number): void {
|
||||||
|
const total = this.stats.successfulMigrations + this.stats.failedMigrations;
|
||||||
|
this.stats.avgMigrationDuration =
|
||||||
|
(this.stats.avgMigrationDuration * (total - 1) + duration) / total;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a schema manager
|
||||||
|
*/
|
||||||
|
export function createSchemaManager(config?: SchemaManagerConfig): SchemaManager {
|
||||||
|
return new SchemaManager(config);
|
||||||
|
}
|
||||||
512
ts/domain/schema/types.ts
Normal file
512
ts/domain/schema/types.ts
Normal file
@@ -0,0 +1,512 @@
|
|||||||
|
/**
|
||||||
|
* Schema Management Types
|
||||||
|
*
|
||||||
|
* Index mapping management, templates, and migrations
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Elasticsearch field types
|
||||||
|
*/
|
||||||
|
export type FieldType =
|
||||||
|
| 'text'
|
||||||
|
| 'keyword'
|
||||||
|
| 'long'
|
||||||
|
| 'integer'
|
||||||
|
| 'short'
|
||||||
|
| 'byte'
|
||||||
|
| 'double'
|
||||||
|
| 'float'
|
||||||
|
| 'half_float'
|
||||||
|
| 'scaled_float'
|
||||||
|
| 'date'
|
||||||
|
| 'date_nanos'
|
||||||
|
| 'boolean'
|
||||||
|
| 'binary'
|
||||||
|
| 'integer_range'
|
||||||
|
| 'float_range'
|
||||||
|
| 'long_range'
|
||||||
|
| 'double_range'
|
||||||
|
| 'date_range'
|
||||||
|
| 'ip_range'
|
||||||
|
| 'object'
|
||||||
|
| 'nested'
|
||||||
|
| 'geo_point'
|
||||||
|
| 'geo_shape'
|
||||||
|
| 'completion'
|
||||||
|
| 'search_as_you_type'
|
||||||
|
| 'token_count'
|
||||||
|
| 'percolator'
|
||||||
|
| 'join'
|
||||||
|
| 'rank_feature'
|
||||||
|
| 'rank_features'
|
||||||
|
| 'dense_vector'
|
||||||
|
| 'sparse_vector'
|
||||||
|
| 'flattened'
|
||||||
|
| 'shape'
|
||||||
|
| 'histogram'
|
||||||
|
| 'ip'
|
||||||
|
| 'alias';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Field definition
|
||||||
|
*/
|
||||||
|
export interface FieldDefinition {
|
||||||
|
/** Field type */
|
||||||
|
type: FieldType;
|
||||||
|
|
||||||
|
/** Index this field */
|
||||||
|
index?: boolean;
|
||||||
|
|
||||||
|
/** Store this field */
|
||||||
|
store?: boolean;
|
||||||
|
|
||||||
|
/** Enable doc values */
|
||||||
|
doc_values?: boolean;
|
||||||
|
|
||||||
|
/** Analyzer for text fields */
|
||||||
|
analyzer?: string;
|
||||||
|
|
||||||
|
/** Search analyzer */
|
||||||
|
search_analyzer?: string;
|
||||||
|
|
||||||
|
/** Normalizer for keyword fields */
|
||||||
|
normalizer?: string;
|
||||||
|
|
||||||
|
/** Boost factor */
|
||||||
|
boost?: number;
|
||||||
|
|
||||||
|
/** Coerce values */
|
||||||
|
coerce?: boolean;
|
||||||
|
|
||||||
|
/** Copy to other fields */
|
||||||
|
copy_to?: string | string[];
|
||||||
|
|
||||||
|
/** Enable norms */
|
||||||
|
norms?: boolean;
|
||||||
|
|
||||||
|
/** Null value replacement */
|
||||||
|
null_value?: unknown;
|
||||||
|
|
||||||
|
/** Ignore values above this length */
|
||||||
|
ignore_above?: number;
|
||||||
|
|
||||||
|
/** Date format */
|
||||||
|
format?: string;
|
||||||
|
|
||||||
|
/** Scaling factor for scaled_float */
|
||||||
|
scaling_factor?: number;
|
||||||
|
|
||||||
|
/** Nested properties */
|
||||||
|
properties?: Record<string, FieldDefinition>;
|
||||||
|
|
||||||
|
/** Enable eager global ordinals */
|
||||||
|
eager_global_ordinals?: boolean;
|
||||||
|
|
||||||
|
/** Similarity algorithm */
|
||||||
|
similarity?: string;
|
||||||
|
|
||||||
|
/** Term vector */
|
||||||
|
term_vector?: 'no' | 'yes' | 'with_positions' | 'with_offsets' | 'with_positions_offsets';
|
||||||
|
|
||||||
|
/** Dimensions for dense_vector */
|
||||||
|
dims?: number;
|
||||||
|
|
||||||
|
/** Additional properties */
|
||||||
|
[key: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Index settings
|
||||||
|
*/
|
||||||
|
export interface IndexSettings {
|
||||||
|
/** Number of primary shards */
|
||||||
|
number_of_shards?: number;
|
||||||
|
|
||||||
|
/** Number of replica shards */
|
||||||
|
number_of_replicas?: number;
|
||||||
|
|
||||||
|
/** Refresh interval */
|
||||||
|
refresh_interval?: string;
|
||||||
|
|
||||||
|
/** Maximum result window */
|
||||||
|
max_result_window?: number;
|
||||||
|
|
||||||
|
/** Maximum inner result window */
|
||||||
|
max_inner_result_window?: number;
|
||||||
|
|
||||||
|
/** Maximum rescore window */
|
||||||
|
max_rescore_window?: number;
|
||||||
|
|
||||||
|
/** Maximum docvalue fields search */
|
||||||
|
max_docvalue_fields_search?: number;
|
||||||
|
|
||||||
|
/** Maximum script fields */
|
||||||
|
max_script_fields?: number;
|
||||||
|
|
||||||
|
/** Maximum regex length */
|
||||||
|
max_regex_length?: number;
|
||||||
|
|
||||||
|
/** Codec */
|
||||||
|
codec?: string;
|
||||||
|
|
||||||
|
/** Routing partition size */
|
||||||
|
routing_partition_size?: number;
|
||||||
|
|
||||||
|
/** Soft deletes retention period */
|
||||||
|
soft_deletes?: {
|
||||||
|
retention_lease?: {
|
||||||
|
period?: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Analysis settings */
|
||||||
|
analysis?: {
|
||||||
|
analyzer?: Record<string, unknown>;
|
||||||
|
tokenizer?: Record<string, unknown>;
|
||||||
|
filter?: Record<string, unknown>;
|
||||||
|
char_filter?: Record<string, unknown>;
|
||||||
|
normalizer?: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Additional settings */
|
||||||
|
[key: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Index mapping
|
||||||
|
*/
|
||||||
|
export interface IndexMapping {
|
||||||
|
/** Dynamic mapping behavior */
|
||||||
|
dynamic?: boolean | 'strict' | 'runtime';
|
||||||
|
|
||||||
|
/** Date detection */
|
||||||
|
date_detection?: boolean;
|
||||||
|
|
||||||
|
/** Dynamic date formats */
|
||||||
|
dynamic_date_formats?: string[];
|
||||||
|
|
||||||
|
/** Numeric detection */
|
||||||
|
numeric_detection?: boolean;
|
||||||
|
|
||||||
|
/** Field properties */
|
||||||
|
properties: Record<string, FieldDefinition>;
|
||||||
|
|
||||||
|
/** Runtime fields */
|
||||||
|
runtime?: Record<string, {
|
||||||
|
type: string;
|
||||||
|
script?: {
|
||||||
|
source: string;
|
||||||
|
lang?: string;
|
||||||
|
};
|
||||||
|
}>;
|
||||||
|
|
||||||
|
/** Meta fields */
|
||||||
|
_source?: {
|
||||||
|
enabled?: boolean;
|
||||||
|
includes?: string[];
|
||||||
|
excludes?: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
_routing?: {
|
||||||
|
required?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
_meta?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Index schema definition
|
||||||
|
*/
|
||||||
|
export interface IndexSchema {
|
||||||
|
/** Index name or pattern */
|
||||||
|
name: string;
|
||||||
|
|
||||||
|
/** Schema version */
|
||||||
|
version: number;
|
||||||
|
|
||||||
|
/** Index settings */
|
||||||
|
settings?: IndexSettings;
|
||||||
|
|
||||||
|
/** Index mapping */
|
||||||
|
mappings: IndexMapping;
|
||||||
|
|
||||||
|
/** Index aliases */
|
||||||
|
aliases?: Record<string, {
|
||||||
|
filter?: unknown;
|
||||||
|
routing?: string;
|
||||||
|
is_write_index?: boolean;
|
||||||
|
is_hidden?: boolean;
|
||||||
|
}>;
|
||||||
|
|
||||||
|
/** Lifecycle policy */
|
||||||
|
lifecycle?: {
|
||||||
|
name: string;
|
||||||
|
rollover_alias?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Schema metadata */
|
||||||
|
metadata?: {
|
||||||
|
description?: string;
|
||||||
|
owner?: string;
|
||||||
|
created?: Date;
|
||||||
|
updated?: Date;
|
||||||
|
tags?: string[];
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema migration
|
||||||
|
*/
|
||||||
|
export interface SchemaMigration {
|
||||||
|
/** Migration version (must be unique) */
|
||||||
|
version: number;
|
||||||
|
|
||||||
|
/** Migration name */
|
||||||
|
name: string;
|
||||||
|
|
||||||
|
/** Migration description */
|
||||||
|
description?: string;
|
||||||
|
|
||||||
|
/** Migration type */
|
||||||
|
type: 'create' | 'update' | 'delete' | 'reindex' | 'alias';
|
||||||
|
|
||||||
|
/** Target index */
|
||||||
|
index: string;
|
||||||
|
|
||||||
|
/** Changes to apply */
|
||||||
|
changes: {
|
||||||
|
/** Settings changes */
|
||||||
|
settings?: Partial<IndexSettings>;
|
||||||
|
|
||||||
|
/** Mapping changes (new or modified fields) */
|
||||||
|
mappings?: {
|
||||||
|
properties?: Record<string, FieldDefinition>;
|
||||||
|
dynamic?: boolean | 'strict' | 'runtime';
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Alias changes */
|
||||||
|
aliases?: {
|
||||||
|
add?: Record<string, unknown>;
|
||||||
|
remove?: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Reindex configuration */
|
||||||
|
reindex?: {
|
||||||
|
source: string;
|
||||||
|
dest: string;
|
||||||
|
script?: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Rollback changes */
|
||||||
|
rollback?: {
|
||||||
|
settings?: Partial<IndexSettings>;
|
||||||
|
mappings?: {
|
||||||
|
properties?: Record<string, FieldDefinition>;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Migration metadata */
|
||||||
|
metadata?: {
|
||||||
|
author?: string;
|
||||||
|
created?: Date;
|
||||||
|
ticket?: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migration status
|
||||||
|
*/
|
||||||
|
export type MigrationStatus = 'pending' | 'running' | 'completed' | 'failed' | 'rolled_back';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migration history entry
|
||||||
|
*/
|
||||||
|
export interface MigrationHistoryEntry {
|
||||||
|
/** Migration version */
|
||||||
|
version: number;
|
||||||
|
|
||||||
|
/** Migration name */
|
||||||
|
name: string;
|
||||||
|
|
||||||
|
/** Migration status */
|
||||||
|
status: MigrationStatus;
|
||||||
|
|
||||||
|
/** Started at */
|
||||||
|
startedAt: Date;
|
||||||
|
|
||||||
|
/** Completed at */
|
||||||
|
completedAt?: Date;
|
||||||
|
|
||||||
|
/** Duration in milliseconds */
|
||||||
|
duration?: number;
|
||||||
|
|
||||||
|
/** Error if failed */
|
||||||
|
error?: string;
|
||||||
|
|
||||||
|
/** Checksum of migration */
|
||||||
|
checksum?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema manager configuration
|
||||||
|
*/
|
||||||
|
export interface SchemaManagerConfig {
|
||||||
|
/** Index for storing migration history */
|
||||||
|
historyIndex?: string;
|
||||||
|
|
||||||
|
/** Enable dry run mode */
|
||||||
|
dryRun?: boolean;
|
||||||
|
|
||||||
|
/** Enable strict mode (fail on warnings) */
|
||||||
|
strict?: boolean;
|
||||||
|
|
||||||
|
/** Timeout for operations */
|
||||||
|
timeout?: number;
|
||||||
|
|
||||||
|
/** Enable logging */
|
||||||
|
enableLogging?: boolean;
|
||||||
|
|
||||||
|
/** Enable metrics */
|
||||||
|
enableMetrics?: boolean;
|
||||||
|
|
||||||
|
/** Validate schemas before applying */
|
||||||
|
validateBeforeApply?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema validation result
|
||||||
|
*/
|
||||||
|
export interface SchemaValidationResult {
|
||||||
|
/** Whether schema is valid */
|
||||||
|
valid: boolean;
|
||||||
|
|
||||||
|
/** Validation errors */
|
||||||
|
errors: Array<{
|
||||||
|
field: string;
|
||||||
|
message: string;
|
||||||
|
severity: 'error' | 'warning';
|
||||||
|
}>;
|
||||||
|
|
||||||
|
/** Validation warnings */
|
||||||
|
warnings: Array<{
|
||||||
|
field: string;
|
||||||
|
message: string;
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema diff result
|
||||||
|
*/
|
||||||
|
export interface SchemaDiff {
|
||||||
|
/** Whether schemas are identical */
|
||||||
|
identical: boolean;
|
||||||
|
|
||||||
|
/** Added fields */
|
||||||
|
added: string[];
|
||||||
|
|
||||||
|
/** Removed fields */
|
||||||
|
removed: string[];
|
||||||
|
|
||||||
|
/** Modified fields */
|
||||||
|
modified: Array<{
|
||||||
|
field: string;
|
||||||
|
from: FieldDefinition;
|
||||||
|
to: FieldDefinition;
|
||||||
|
}>;
|
||||||
|
|
||||||
|
/** Settings changes */
|
||||||
|
settingsChanged: boolean;
|
||||||
|
|
||||||
|
/** Aliases changes */
|
||||||
|
aliasesChanged: boolean;
|
||||||
|
|
||||||
|
/** Breaking changes */
|
||||||
|
breakingChanges: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Template definition
|
||||||
|
*/
|
||||||
|
export interface IndexTemplate {
|
||||||
|
/** Template name */
|
||||||
|
name: string;
|
||||||
|
|
||||||
|
/** Index patterns */
|
||||||
|
index_patterns: string[];
|
||||||
|
|
||||||
|
/** Template priority */
|
||||||
|
priority?: number;
|
||||||
|
|
||||||
|
/** Template version */
|
||||||
|
version?: number;
|
||||||
|
|
||||||
|
/** Composed of component templates */
|
||||||
|
composed_of?: string[];
|
||||||
|
|
||||||
|
/** Template settings */
|
||||||
|
template?: {
|
||||||
|
settings?: IndexSettings;
|
||||||
|
mappings?: IndexMapping;
|
||||||
|
aliases?: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Data stream settings */
|
||||||
|
data_stream?: {
|
||||||
|
hidden?: boolean;
|
||||||
|
allow_custom_routing?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Template metadata */
|
||||||
|
_meta?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Component template
|
||||||
|
*/
|
||||||
|
export interface ComponentTemplate {
|
||||||
|
/** Template name */
|
||||||
|
name: string;
|
||||||
|
|
||||||
|
/** Template content */
|
||||||
|
template: {
|
||||||
|
settings?: IndexSettings;
|
||||||
|
mappings?: IndexMapping;
|
||||||
|
aliases?: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Template version */
|
||||||
|
version?: number;
|
||||||
|
|
||||||
|
/** Template metadata */
|
||||||
|
_meta?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema manager statistics
|
||||||
|
*/
|
||||||
|
export interface SchemaManagerStats {
|
||||||
|
/** Total migrations applied */
|
||||||
|
totalMigrations: number;
|
||||||
|
|
||||||
|
/** Successful migrations */
|
||||||
|
successfulMigrations: number;
|
||||||
|
|
||||||
|
/** Failed migrations */
|
||||||
|
failedMigrations: number;
|
||||||
|
|
||||||
|
/** Rolled back migrations */
|
||||||
|
rolledBackMigrations: number;
|
||||||
|
|
||||||
|
/** Total indices managed */
|
||||||
|
totalIndices: number;
|
||||||
|
|
||||||
|
/** Total templates managed */
|
||||||
|
totalTemplates: number;
|
||||||
|
|
||||||
|
/** Last migration time */
|
||||||
|
lastMigrationTime?: Date;
|
||||||
|
|
||||||
|
/** Average migration duration */
|
||||||
|
avgMigrationDuration: number;
|
||||||
|
}
|
||||||
379
ts/examples/schema/schema-example.ts
Normal file
379
ts/examples/schema/schema-example.ts
Normal file
@@ -0,0 +1,379 @@
|
|||||||
|
/**
|
||||||
|
* Comprehensive Schema Management Example
|
||||||
|
*
|
||||||
|
* Demonstrates index mapping management, templates, and migrations
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
createConfig,
|
||||||
|
ElasticsearchConnectionManager,
|
||||||
|
LogLevel,
|
||||||
|
createSchemaManager,
|
||||||
|
type IndexSchema,
|
||||||
|
type SchemaMigration,
|
||||||
|
type IndexTemplate,
|
||||||
|
} from '../../index.js';
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
console.log('=== Schema Management Example ===\n');
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 1: Configuration
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 1: Configuring Elasticsearch connection...');
|
||||||
|
const config = createConfig()
|
||||||
|
.fromEnv()
|
||||||
|
.nodes(process.env.ELASTICSEARCH_URL || 'http://localhost:9200')
|
||||||
|
.basicAuth(
|
||||||
|
process.env.ELASTICSEARCH_USERNAME || 'elastic',
|
||||||
|
process.env.ELASTICSEARCH_PASSWORD || 'changeme'
|
||||||
|
)
|
||||||
|
.timeout(30000)
|
||||||
|
.retries(3)
|
||||||
|
.logLevel(LogLevel.INFO)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 2: Initialize
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 2: Initializing connection and schema manager...');
|
||||||
|
const connectionManager = ElasticsearchConnectionManager.getInstance(config);
|
||||||
|
await connectionManager.initialize();
|
||||||
|
|
||||||
|
const schemaManager = createSchemaManager({
|
||||||
|
historyIndex: '.test_migrations',
|
||||||
|
dryRun: false,
|
||||||
|
strict: true,
|
||||||
|
enableLogging: true,
|
||||||
|
enableMetrics: true,
|
||||||
|
validateBeforeApply: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await schemaManager.initialize();
|
||||||
|
console.log('✓ Connection and schema manager initialized\n');
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 3: Create Index with Schema
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 3: Creating index with schema...');
|
||||||
|
|
||||||
|
const productSchema: IndexSchema = {
|
||||||
|
name: 'products-v1',
|
||||||
|
version: 1,
|
||||||
|
settings: {
|
||||||
|
number_of_shards: 1,
|
||||||
|
number_of_replicas: 0,
|
||||||
|
refresh_interval: '1s',
|
||||||
|
analysis: {
|
||||||
|
analyzer: {
|
||||||
|
product_analyzer: {
|
||||||
|
type: 'custom',
|
||||||
|
tokenizer: 'standard',
|
||||||
|
filter: ['lowercase', 'snowball'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
mappings: {
|
||||||
|
dynamic: 'strict',
|
||||||
|
properties: {
|
||||||
|
id: { type: 'keyword' },
|
||||||
|
name: {
|
||||||
|
type: 'text',
|
||||||
|
analyzer: 'product_analyzer',
|
||||||
|
properties: {
|
||||||
|
keyword: { type: 'keyword', ignore_above: 256 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
description: { type: 'text' },
|
||||||
|
price: { type: 'scaled_float', scaling_factor: 100 },
|
||||||
|
category: { type: 'keyword' },
|
||||||
|
tags: { type: 'keyword' },
|
||||||
|
inStock: { type: 'boolean' },
|
||||||
|
createdAt: { type: 'date' },
|
||||||
|
updatedAt: { type: 'date' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
aliases: {
|
||||||
|
products: { is_write_index: true },
|
||||||
|
},
|
||||||
|
metadata: {
|
||||||
|
description: 'Product catalog index',
|
||||||
|
owner: 'catalog-team',
|
||||||
|
tags: ['catalog', 'products'],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// Validate schema first
|
||||||
|
const validation = schemaManager.validateSchema(productSchema);
|
||||||
|
console.log(` Schema validation: ${validation.valid ? 'PASSED' : 'FAILED'}`);
|
||||||
|
|
||||||
|
if (validation.warnings.length > 0) {
|
||||||
|
console.log(' Warnings:');
|
||||||
|
for (const warning of validation.warnings) {
|
||||||
|
console.log(` - ${warning.field}: ${warning.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the index
|
||||||
|
await schemaManager.createIndex(productSchema);
|
||||||
|
console.log(' ✓ Index created with mappings and aliases\n');
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 4: Schema Migrations
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 4: Running schema migrations...');
|
||||||
|
|
||||||
|
const migrations: SchemaMigration[] = [
|
||||||
|
{
|
||||||
|
version: 1,
|
||||||
|
name: 'add_rating_field',
|
||||||
|
description: 'Add product rating field',
|
||||||
|
type: 'update',
|
||||||
|
index: 'products-v1',
|
||||||
|
changes: {
|
||||||
|
mappings: {
|
||||||
|
properties: {
|
||||||
|
rating: { type: 'float' },
|
||||||
|
reviewCount: { type: 'integer' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
rollback: {
|
||||||
|
// Note: Can't remove fields in ES, but document for reference
|
||||||
|
},
|
||||||
|
metadata: {
|
||||||
|
author: 'dev-team',
|
||||||
|
ticket: 'PROD-123',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
version: 2,
|
||||||
|
name: 'add_brand_field',
|
||||||
|
description: 'Add brand field with keyword type',
|
||||||
|
type: 'update',
|
||||||
|
index: 'products-v1',
|
||||||
|
changes: {
|
||||||
|
mappings: {
|
||||||
|
properties: {
|
||||||
|
brand: { type: 'keyword' },
|
||||||
|
sku: { type: 'keyword' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
metadata: {
|
||||||
|
author: 'dev-team',
|
||||||
|
ticket: 'PROD-456',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
version: 3,
|
||||||
|
name: 'add_inventory_object',
|
||||||
|
description: 'Add nested inventory object',
|
||||||
|
type: 'update',
|
||||||
|
index: 'products-v1',
|
||||||
|
changes: {
|
||||||
|
mappings: {
|
||||||
|
properties: {
|
||||||
|
inventory: {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
quantity: { type: 'integer' },
|
||||||
|
warehouse: { type: 'keyword' },
|
||||||
|
lastRestocked: { type: 'date' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
metadata: {
|
||||||
|
author: 'inventory-team',
|
||||||
|
ticket: 'INV-789',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const migrationResults = await schemaManager.migrate(migrations);
|
||||||
|
|
||||||
|
console.log(' Migration results:');
|
||||||
|
for (const result of migrationResults) {
|
||||||
|
console.log(` v${result.version} (${result.name}): ${result.status} (${result.duration}ms)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 5: Get Applied Migrations
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 5: Viewing migration history...');
|
||||||
|
|
||||||
|
const appliedMigrations = await schemaManager.getAppliedMigrations();
|
||||||
|
|
||||||
|
console.log(' Applied migrations:');
|
||||||
|
for (const migration of appliedMigrations) {
|
||||||
|
console.log(` v${migration.version}: ${migration.name} (${migration.status})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 6: Compare Schemas
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 6: Comparing schemas...');
|
||||||
|
|
||||||
|
const oldSchema = productSchema;
|
||||||
|
const newSchema: IndexSchema = {
|
||||||
|
...productSchema,
|
||||||
|
version: 2,
|
||||||
|
mappings: {
|
||||||
|
...productSchema.mappings,
|
||||||
|
properties: {
|
||||||
|
...productSchema.mappings.properties,
|
||||||
|
rating: { type: 'float' },
|
||||||
|
discount: { type: 'float' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const diff = schemaManager.diffSchemas(oldSchema, newSchema);
|
||||||
|
|
||||||
|
console.log(' Schema diff:');
|
||||||
|
console.log(` Identical: ${diff.identical}`);
|
||||||
|
console.log(` Added fields: ${diff.added.join(', ') || 'none'}`);
|
||||||
|
console.log(` Removed fields: ${diff.removed.join(', ') || 'none'}`);
|
||||||
|
console.log(` Modified fields: ${diff.modified.length}`);
|
||||||
|
console.log(` Breaking changes: ${diff.breakingChanges.length}`);
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 7: Index Templates
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 7: Managing index templates...');
|
||||||
|
|
||||||
|
const template: IndexTemplate = {
|
||||||
|
name: 'products-template',
|
||||||
|
index_patterns: ['products-*'],
|
||||||
|
priority: 100,
|
||||||
|
version: 1,
|
||||||
|
template: {
|
||||||
|
settings: {
|
||||||
|
number_of_shards: 1,
|
||||||
|
number_of_replicas: 0,
|
||||||
|
},
|
||||||
|
mappings: {
|
||||||
|
dynamic: 'strict',
|
||||||
|
properties: {
|
||||||
|
id: { type: 'keyword' },
|
||||||
|
name: { type: 'text' },
|
||||||
|
createdAt: { type: 'date' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
aliases: {
|
||||||
|
'all-products': {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
_meta: {
|
||||||
|
description: 'Template for product indices',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await schemaManager.putTemplate(template);
|
||||||
|
console.log(' ✓ Index template created');
|
||||||
|
|
||||||
|
// Get template
|
||||||
|
const retrievedTemplate = await schemaManager.getTemplate('products-template');
|
||||||
|
console.log(` Template patterns: ${retrievedTemplate?.index_patterns.join(', ')}`);
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 8: Alias Management
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 8: Managing aliases...');
|
||||||
|
|
||||||
|
// Add alias
|
||||||
|
await schemaManager.addAlias('products-v1', 'products-read', {
|
||||||
|
filter: { term: { inStock: true } },
|
||||||
|
});
|
||||||
|
console.log(' ✓ Added filtered read alias');
|
||||||
|
|
||||||
|
// Get schema to see aliases
|
||||||
|
const currentSchema = await schemaManager.getSchema('products-v1');
|
||||||
|
console.log(` Current aliases: ${Object.keys(currentSchema?.aliases || {}).join(', ')}`);
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 9: Update Settings
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 9: Updating index settings...');
|
||||||
|
|
||||||
|
await schemaManager.updateSettings('products-v1', {
|
||||||
|
refresh_interval: '5s',
|
||||||
|
max_result_window: 20000,
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(' ✓ Settings updated');
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 10: Statistics
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 10: Schema manager statistics...\n');
|
||||||
|
|
||||||
|
const stats = schemaManager.getStats();
|
||||||
|
|
||||||
|
console.log('Schema Manager Statistics:');
|
||||||
|
console.log(` Total migrations: ${stats.totalMigrations}`);
|
||||||
|
console.log(` Successful: ${stats.successfulMigrations}`);
|
||||||
|
console.log(` Failed: ${stats.failedMigrations}`);
|
||||||
|
console.log(` Rolled back: ${stats.rolledBackMigrations}`);
|
||||||
|
console.log(` Total indices: ${stats.totalIndices}`);
|
||||||
|
console.log(` Total templates: ${stats.totalTemplates}`);
|
||||||
|
console.log(` Avg migration duration: ${stats.avgMigrationDuration.toFixed(2)}ms`);
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Step 11: Cleanup
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
console.log('Step 11: Cleanup...');
|
||||||
|
|
||||||
|
await schemaManager.deleteTemplate('products-template');
|
||||||
|
await schemaManager.deleteIndex('products-v1');
|
||||||
|
await connectionManager.destroy();
|
||||||
|
|
||||||
|
console.log('✓ Cleanup complete\n');
|
||||||
|
|
||||||
|
console.log('=== Schema Management Example Complete ===');
|
||||||
|
console.log('\nKey Features Demonstrated:');
|
||||||
|
console.log(' ✓ Index creation with mappings and settings');
|
||||||
|
console.log(' ✓ Schema validation before apply');
|
||||||
|
console.log(' ✓ Versioned migrations with history tracking');
|
||||||
|
console.log(' ✓ Schema diff and comparison');
|
||||||
|
console.log(' ✓ Index templates');
|
||||||
|
console.log(' ✓ Alias management with filters');
|
||||||
|
console.log(' ✓ Settings updates');
|
||||||
|
console.log(' ✓ Migration rollback support');
|
||||||
|
console.log(' ✓ Dry run mode');
|
||||||
|
console.log(' ✓ Comprehensive statistics');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the example
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error('Example failed:', error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
21
ts/index.ts
21
ts/index.ts
@@ -14,6 +14,7 @@ export * from './domain/logging/index.js';
|
|||||||
export * from './domain/bulk/index.js';
|
export * from './domain/bulk/index.js';
|
||||||
export * from './domain/kv/index.js';
|
export * from './domain/kv/index.js';
|
||||||
export * from './domain/transactions/index.js';
|
export * from './domain/transactions/index.js';
|
||||||
|
export * from './domain/schema/index.js';
|
||||||
|
|
||||||
// Re-export commonly used items for convenience
|
// Re-export commonly used items for convenience
|
||||||
export {
|
export {
|
||||||
@@ -172,3 +173,23 @@ export {
|
|||||||
type Savepoint,
|
type Savepoint,
|
||||||
type TransactionCallbacks,
|
type TransactionCallbacks,
|
||||||
} from './domain/transactions/index.js';
|
} from './domain/transactions/index.js';
|
||||||
|
|
||||||
|
export {
|
||||||
|
// Schema
|
||||||
|
SchemaManager,
|
||||||
|
createSchemaManager,
|
||||||
|
type FieldType,
|
||||||
|
type FieldDefinition,
|
||||||
|
type IndexSettings,
|
||||||
|
type IndexMapping,
|
||||||
|
type IndexSchema,
|
||||||
|
type SchemaMigration,
|
||||||
|
type MigrationStatus,
|
||||||
|
type MigrationHistoryEntry,
|
||||||
|
type SchemaManagerConfig,
|
||||||
|
type SchemaValidationResult,
|
||||||
|
type SchemaDiff,
|
||||||
|
type IndexTemplate,
|
||||||
|
type ComponentTemplate,
|
||||||
|
type SchemaManagerStats,
|
||||||
|
} from './domain/schema/index.js';
|
||||||
|
|||||||
Reference in New Issue
Block a user