BREAKING CHANGE(core): Refactor to v3: introduce modular core/domain architecture, plugin system, observability and strict TypeScript configuration; remove legacy classes
This commit is contained in:
571
ts/domain/documents/document-manager.ts
Normal file
571
ts/domain/documents/document-manager.ts
Normal file
@@ -0,0 +1,571 @@
|
||||
import type { Client as ElasticClient } from '@elastic/elasticsearch';
|
||||
import { ElasticsearchConnectionManager } from '../../core/connection/connection-manager.js';
|
||||
import { Logger, defaultLogger } from '../../core/observability/logger.js';
|
||||
import { MetricsCollector, defaultMetricsCollector } from '../../core/observability/metrics.js';
|
||||
import { TracingProvider, defaultTracingProvider } from '../../core/observability/tracing.js';
|
||||
import { DocumentSession } from './document-session.js';
|
||||
import {
|
||||
DocumentWithMeta,
|
||||
SessionConfig,
|
||||
SnapshotProcessor,
|
||||
SnapshotMeta,
|
||||
IteratorOptions,
|
||||
} from './types.js';
|
||||
import { IndexNotFoundError } from '../../core/errors/elasticsearch-error.js';
|
||||
|
||||
/**
|
||||
* Document manager configuration
|
||||
*/
|
||||
export interface DocumentManagerConfig {
|
||||
/** Index name */
|
||||
index: string;
|
||||
|
||||
/** Connection manager (optional, will use singleton if not provided) */
|
||||
connectionManager?: ElasticsearchConnectionManager;
|
||||
|
||||
/** Logger (optional, will use default if not provided) */
|
||||
logger?: Logger;
|
||||
|
||||
/** Metrics collector (optional) */
|
||||
metrics?: MetricsCollector;
|
||||
|
||||
/** Tracing provider (optional) */
|
||||
tracing?: TracingProvider;
|
||||
|
||||
/** Auto-create index if it doesn't exist */
|
||||
autoCreateIndex?: boolean;
|
||||
|
||||
/** Default batch size for operations */
|
||||
defaultBatchSize?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fluent document manager for Elasticsearch
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const docs = new DocumentManager<Product>('products');
|
||||
* await docs.initialize();
|
||||
*
|
||||
* // Session-based operations
|
||||
* await docs
|
||||
* .session()
|
||||
* .start()
|
||||
* .upsert('prod-1', { name: 'Widget', price: 99.99 })
|
||||
* .upsert('prod-2', { name: 'Gadget', price: 149.99 })
|
||||
* .commit();
|
||||
*
|
||||
* // Get a document
|
||||
* const product = await docs.get('prod-1');
|
||||
*
|
||||
* // Create snapshot
|
||||
* const snapshot = await docs.snapshot(async (iterator) => {
|
||||
* const products = [];
|
||||
* for await (const doc of iterator) {
|
||||
* products.push(doc._source);
|
||||
* }
|
||||
* return { totalCount: products.length, products };
|
||||
* });
|
||||
* ```
|
||||
*/
|
||||
export class DocumentManager<T = unknown> {
|
||||
private client: ElasticClient;
|
||||
private connectionManager: ElasticsearchConnectionManager;
|
||||
private logger: Logger;
|
||||
private metrics: MetricsCollector;
|
||||
private tracing: TracingProvider;
|
||||
private index: string;
|
||||
private config: DocumentManagerConfig;
|
||||
private isInitialized = false;
|
||||
|
||||
constructor(config: DocumentManagerConfig) {
|
||||
this.config = config;
|
||||
this.index = config.index;
|
||||
|
||||
// Get or create connection manager
|
||||
this.connectionManager =
|
||||
config.connectionManager || ElasticsearchConnectionManager.getInstance();
|
||||
|
||||
// Set up observability
|
||||
this.logger = config.logger || defaultLogger.child(`documents:${this.index}`);
|
||||
this.metrics = config.metrics || defaultMetricsCollector;
|
||||
this.tracing = config.tracing || defaultTracingProvider;
|
||||
|
||||
// Get client (will throw if connection manager not initialized)
|
||||
this.client = this.connectionManager.getClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* Static factory method for fluent creation
|
||||
*/
|
||||
static create<T = unknown>(index: string, config: Omit<DocumentManagerConfig, 'index'> = {}): DocumentManager<T> {
|
||||
return new DocumentManager<T>({ ...config, index });
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the document manager
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
return this.tracing.withSpan('DocumentManager.initialize', async (span) => {
|
||||
span.setAttribute('index', this.index);
|
||||
|
||||
try {
|
||||
// Check if index exists
|
||||
const exists = await this.client.indices.exists({ index: this.index });
|
||||
|
||||
if (!exists && this.config.autoCreateIndex) {
|
||||
this.logger.info('Creating index', { index: this.index });
|
||||
await this.client.indices.create({ index: this.index });
|
||||
this.logger.info('Index created', { index: this.index });
|
||||
} else if (!exists) {
|
||||
throw new IndexNotFoundError(this.index);
|
||||
}
|
||||
|
||||
this.isInitialized = true;
|
||||
this.logger.info('Document manager initialized', { index: this.index });
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to initialize document manager', error as Error, {
|
||||
index: this.index,
|
||||
});
|
||||
span.recordException(error as Error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new session for batch operations
|
||||
*/
|
||||
session(config?: SessionConfig): DocumentSession<T> {
|
||||
this.ensureInitialized();
|
||||
return new DocumentSession<T>(this.client, this.index, this.logger, config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single document by ID
|
||||
*/
|
||||
async get(documentId: string): Promise<DocumentWithMeta<T> | null> {
|
||||
this.ensureInitialized();
|
||||
|
||||
return this.tracing.withSpan('DocumentManager.get', async (span) => {
|
||||
span.setAttributes({
|
||||
'document.id': documentId,
|
||||
'document.index': this.index,
|
||||
});
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const result = await this.client.get({
|
||||
index: this.index,
|
||||
id: documentId,
|
||||
});
|
||||
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
this.metrics.requestDuration.observe(duration, {
|
||||
operation: 'get',
|
||||
index: this.index,
|
||||
});
|
||||
|
||||
return {
|
||||
_id: result._id,
|
||||
_source: result._source as T,
|
||||
_version: result._version,
|
||||
_seq_no: result._seq_no,
|
||||
_primary_term: result._primary_term,
|
||||
_index: result._index,
|
||||
};
|
||||
} catch (error: any) {
|
||||
if (error.statusCode === 404) {
|
||||
this.logger.debug('Document not found', { documentId, index: this.index });
|
||||
return null;
|
||||
}
|
||||
|
||||
this.logger.error('Failed to get document', error, { documentId, index: this.index });
|
||||
span.recordException(error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a document
|
||||
*/
|
||||
async create(documentId: string, document: T): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
|
||||
return this.tracing.withSpan('DocumentManager.create', async (span) => {
|
||||
span.setAttributes({
|
||||
'document.id': documentId,
|
||||
'document.index': this.index,
|
||||
});
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
await this.client.create({
|
||||
index: this.index,
|
||||
id: documentId,
|
||||
body: document,
|
||||
refresh: true,
|
||||
});
|
||||
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
this.metrics.requestDuration.observe(duration, {
|
||||
operation: 'create',
|
||||
index: this.index,
|
||||
});
|
||||
|
||||
this.logger.debug('Document created', { documentId, index: this.index });
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to create document', error as Error, {
|
||||
documentId,
|
||||
index: this.index,
|
||||
});
|
||||
span.recordException(error as Error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a document
|
||||
*/
|
||||
async update(
|
||||
documentId: string,
|
||||
document: Partial<T>,
|
||||
options?: { seqNo?: number; primaryTerm?: number }
|
||||
): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
|
||||
return this.tracing.withSpan('DocumentManager.update', async (span) => {
|
||||
span.setAttributes({
|
||||
'document.id': documentId,
|
||||
'document.index': this.index,
|
||||
});
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
await this.client.update({
|
||||
index: this.index,
|
||||
id: documentId,
|
||||
body: { doc: document },
|
||||
refresh: true,
|
||||
...(options?.seqNo !== undefined && { if_seq_no: options.seqNo }),
|
||||
...(options?.primaryTerm !== undefined && { if_primary_term: options.primaryTerm }),
|
||||
});
|
||||
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
this.metrics.requestDuration.observe(duration, {
|
||||
operation: 'update',
|
||||
index: this.index,
|
||||
});
|
||||
|
||||
this.logger.debug('Document updated', { documentId, index: this.index });
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to update document', error as Error, {
|
||||
documentId,
|
||||
index: this.index,
|
||||
});
|
||||
span.recordException(error as Error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert a document (create or update)
|
||||
*/
|
||||
async upsert(documentId: string, document: T): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
|
||||
return this.tracing.withSpan('DocumentManager.upsert', async (span) => {
|
||||
span.setAttributes({
|
||||
'document.id': documentId,
|
||||
'document.index': this.index,
|
||||
});
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
await this.client.index({
|
||||
index: this.index,
|
||||
id: documentId,
|
||||
body: document,
|
||||
refresh: true,
|
||||
});
|
||||
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
this.metrics.requestDuration.observe(duration, {
|
||||
operation: 'upsert',
|
||||
index: this.index,
|
||||
});
|
||||
|
||||
this.logger.debug('Document upserted', { documentId, index: this.index });
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to upsert document', error as Error, {
|
||||
documentId,
|
||||
index: this.index,
|
||||
});
|
||||
span.recordException(error as Error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a document
|
||||
*/
|
||||
async delete(documentId: string): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
|
||||
return this.tracing.withSpan('DocumentManager.delete', async (span) => {
|
||||
span.setAttributes({
|
||||
'document.id': documentId,
|
||||
'document.index': this.index,
|
||||
});
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
await this.client.delete({
|
||||
index: this.index,
|
||||
id: documentId,
|
||||
refresh: true,
|
||||
});
|
||||
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
this.metrics.requestDuration.observe(duration, {
|
||||
operation: 'delete',
|
||||
index: this.index,
|
||||
});
|
||||
|
||||
this.logger.debug('Document deleted', { documentId, index: this.index });
|
||||
} catch (error: any) {
|
||||
if (error.statusCode === 404) {
|
||||
this.logger.debug('Document not found for deletion', { documentId, index: this.index });
|
||||
return; // Idempotent delete
|
||||
}
|
||||
|
||||
this.logger.error('Failed to delete document', error, { documentId, index: this.index });
|
||||
span.recordException(error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if index exists
|
||||
*/
|
||||
async exists(): Promise<boolean> {
|
||||
try {
|
||||
return await this.client.indices.exists({ index: this.index });
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to check if index exists', error as Error, {
|
||||
index: this.index,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete the index
|
||||
*/
|
||||
async deleteIndex(): Promise<void> {
|
||||
return this.tracing.withSpan('DocumentManager.deleteIndex', async (span) => {
|
||||
span.setAttribute('index', this.index);
|
||||
|
||||
try {
|
||||
await this.client.indices.delete({ index: this.index });
|
||||
this.isInitialized = false;
|
||||
this.logger.info('Index deleted', { index: this.index });
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to delete index', error as Error, { index: this.index });
|
||||
span.recordException(error as Error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get document count
|
||||
*/
|
||||
async count(query?: unknown): Promise<number> {
|
||||
this.ensureInitialized();
|
||||
|
||||
try {
|
||||
const result = await this.client.count({
|
||||
index: this.index,
|
||||
...(query && { body: { query } }),
|
||||
});
|
||||
|
||||
return result.count;
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to count documents', error as Error, { index: this.index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a snapshot with custom processor
|
||||
*/
|
||||
async snapshot<R>(processor: SnapshotProcessor<T, R>): Promise<SnapshotMeta<R>> {
|
||||
this.ensureInitialized();
|
||||
|
||||
return this.tracing.withSpan('DocumentManager.snapshot', async (span) => {
|
||||
span.setAttribute('index', this.index);
|
||||
|
||||
const startTime = Date.now();
|
||||
const snapshotIndex = `${this.index}-snapshots`;
|
||||
|
||||
try {
|
||||
// Get previous snapshot
|
||||
const previousSnapshot = await this.getLatestSnapshot<R>(snapshotIndex);
|
||||
|
||||
// Create iterator for all documents
|
||||
const iterator = this.iterate();
|
||||
|
||||
// Process snapshot
|
||||
const snapshotData = await processor(iterator, previousSnapshot);
|
||||
|
||||
// Count documents
|
||||
const documentCount = await this.count();
|
||||
|
||||
// Store snapshot
|
||||
const snapshot: SnapshotMeta<R> = {
|
||||
date: new Date(),
|
||||
data: snapshotData,
|
||||
documentCount,
|
||||
processingTime: Date.now() - startTime,
|
||||
};
|
||||
|
||||
await this.storeSnapshot(snapshotIndex, snapshot);
|
||||
|
||||
this.logger.info('Snapshot created', {
|
||||
index: this.index,
|
||||
documentCount,
|
||||
processingTime: snapshot.processingTime,
|
||||
});
|
||||
|
||||
return snapshot;
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to create snapshot', error as Error, { index: this.index });
|
||||
span.recordException(error as Error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate over all documents
|
||||
*/
|
||||
async *iterate(options: IteratorOptions = {}): AsyncIterableIterator<DocumentWithMeta<T>> {
|
||||
this.ensureInitialized();
|
||||
|
||||
const batchSize = options.batchSize || this.config.defaultBatchSize || 1000;
|
||||
|
||||
// TODO: Use Point-in-Time API for better performance
|
||||
// For now, use basic search with search_after
|
||||
|
||||
let searchAfter: any[] | undefined;
|
||||
let hasMore = true;
|
||||
|
||||
while (hasMore) {
|
||||
const result = await this.client.search({
|
||||
index: this.index,
|
||||
body: {
|
||||
size: batchSize,
|
||||
...(searchAfter && { search_after: searchAfter }),
|
||||
sort: options.sort || [{ _id: 'asc' }],
|
||||
...(options.query && { query: options.query }),
|
||||
},
|
||||
});
|
||||
|
||||
const hits = result.hits.hits;
|
||||
|
||||
if (hits.length === 0) {
|
||||
hasMore = false;
|
||||
break;
|
||||
}
|
||||
|
||||
for (const hit of hits) {
|
||||
yield {
|
||||
_id: hit._id,
|
||||
_source: hit._source as T,
|
||||
_version: hit._version,
|
||||
_seq_no: hit._seq_no,
|
||||
_primary_term: hit._primary_term,
|
||||
_index: hit._index,
|
||||
_score: hit._score,
|
||||
};
|
||||
}
|
||||
|
||||
// Get last sort value for pagination
|
||||
const lastHit = hits[hits.length - 1];
|
||||
searchAfter = lastHit.sort;
|
||||
|
||||
if (hits.length < batchSize) {
|
||||
hasMore = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get latest snapshot
|
||||
*/
|
||||
private async getLatestSnapshot<R>(snapshotIndex: string): Promise<R | null> {
|
||||
try {
|
||||
const result = await this.client.search({
|
||||
index: snapshotIndex,
|
||||
body: {
|
||||
size: 1,
|
||||
sort: [{ 'date': 'desc' }],
|
||||
},
|
||||
});
|
||||
|
||||
if (result.hits.hits.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const snapshot = result.hits.hits[0]._source as SnapshotMeta<R>;
|
||||
return snapshot.data;
|
||||
} catch (error: any) {
|
||||
if (error.statusCode === 404) {
|
||||
return null; // Index doesn't exist yet
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store snapshot
|
||||
*/
|
||||
private async storeSnapshot<R>(snapshotIndex: string, snapshot: SnapshotMeta<R>): Promise<void> {
|
||||
await this.client.index({
|
||||
index: snapshotIndex,
|
||||
body: snapshot,
|
||||
refresh: true,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure manager is initialized
|
||||
*/
|
||||
private ensureInitialized(): void {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('DocumentManager not initialized. Call initialize() first.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get index name
|
||||
*/
|
||||
getIndex(): string {
|
||||
return this.index;
|
||||
}
|
||||
}
|
||||
356
ts/domain/documents/document-session.ts
Normal file
356
ts/domain/documents/document-session.ts
Normal file
@@ -0,0 +1,356 @@
|
||||
import type { Client as ElasticClient } from '@elastic/elasticsearch';
|
||||
import {
|
||||
BatchOperation,
|
||||
BatchResult,
|
||||
DocumentOperation,
|
||||
SessionConfig,
|
||||
} from './types.js';
|
||||
import { Logger } from '../../core/observability/logger.js';
|
||||
import { BulkOperationError } from '../../core/errors/elasticsearch-error.js';
|
||||
|
||||
/**
|
||||
* Document session for managing document lifecycle
|
||||
*
|
||||
* Tracks documents during a session and can clean up stale ones at the end.
|
||||
*/
|
||||
export class DocumentSession<T = unknown> {
|
||||
private operations: BatchOperation<T>[] = [];
|
||||
private seenDocuments = new Set<string>();
|
||||
private config: Required<SessionConfig>;
|
||||
private startTimestamp: Date;
|
||||
private isActive = false;
|
||||
|
||||
constructor(
|
||||
private client: ElasticClient,
|
||||
private index: string,
|
||||
private logger: Logger,
|
||||
config: SessionConfig = {}
|
||||
) {
|
||||
this.config = {
|
||||
onlyNew: config.onlyNew || false,
|
||||
fromTimestamp: config.fromTimestamp || new Date(),
|
||||
cleanupStale: config.cleanupStale !== false,
|
||||
batchSize: config.batchSize || 1000,
|
||||
};
|
||||
this.startTimestamp = new Date();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the session
|
||||
*/
|
||||
start(): this {
|
||||
if (this.isActive) {
|
||||
throw new Error('Session already active');
|
||||
}
|
||||
|
||||
this.isActive = true;
|
||||
this.operations = [];
|
||||
this.seenDocuments.clear();
|
||||
this.startTimestamp = new Date();
|
||||
|
||||
this.logger.debug('Document session started', {
|
||||
index: this.index,
|
||||
config: this.config,
|
||||
});
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a document (upsert - create or update)
|
||||
*/
|
||||
upsert(documentId: string, document: T): this {
|
||||
this.ensureActive();
|
||||
|
||||
this.operations.push({
|
||||
operation: DocumentOperation.UPSERT,
|
||||
documentId,
|
||||
document,
|
||||
});
|
||||
|
||||
this.seenDocuments.add(documentId);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a document (fails if exists)
|
||||
*/
|
||||
create(documentId: string, document: T): this {
|
||||
this.ensureActive();
|
||||
|
||||
this.operations.push({
|
||||
operation: DocumentOperation.CREATE,
|
||||
documentId,
|
||||
document,
|
||||
});
|
||||
|
||||
this.seenDocuments.add(documentId);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a document (fails if doesn't exist)
|
||||
*/
|
||||
update(documentId: string, document: T, version?: { seqNo: number; primaryTerm: number }): this {
|
||||
this.ensureActive();
|
||||
|
||||
this.operations.push({
|
||||
operation: DocumentOperation.UPDATE,
|
||||
documentId,
|
||||
document,
|
||||
...(version && {
|
||||
seqNo: version.seqNo,
|
||||
primaryTerm: version.primaryTerm,
|
||||
}),
|
||||
});
|
||||
|
||||
this.seenDocuments.add(documentId);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a document
|
||||
*/
|
||||
delete(documentId: string): this {
|
||||
this.ensureActive();
|
||||
|
||||
this.operations.push({
|
||||
operation: DocumentOperation.DELETE,
|
||||
documentId,
|
||||
});
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit the session and execute all operations
|
||||
*/
|
||||
async commit(): Promise<BatchResult> {
|
||||
this.ensureActive();
|
||||
|
||||
try {
|
||||
// Execute batched operations
|
||||
const result = await this.executeBatch();
|
||||
|
||||
// Clean up stale documents if configured
|
||||
if (this.config.cleanupStale) {
|
||||
await this.cleanupStaleDocuments();
|
||||
}
|
||||
|
||||
this.isActive = false;
|
||||
|
||||
this.logger.info('Session committed', {
|
||||
index: this.index,
|
||||
successful: result.successful,
|
||||
failed: result.failed,
|
||||
duration: Date.now() - this.startTimestamp.getTime(),
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.logger.error('Session commit failed', error as Error, {
|
||||
index: this.index,
|
||||
operationCount: this.operations.length,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rollback the session (discard all operations)
|
||||
*/
|
||||
rollback(): void {
|
||||
this.operations = [];
|
||||
this.seenDocuments.clear();
|
||||
this.isActive = false;
|
||||
|
||||
this.logger.debug('Session rolled back', { index: this.index });
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute batch operations
|
||||
*/
|
||||
private async executeBatch(): Promise<BatchResult> {
|
||||
if (this.operations.length === 0) {
|
||||
return {
|
||||
successful: 0,
|
||||
failed: 0,
|
||||
errors: [],
|
||||
took: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
const bulkBody: any[] = [];
|
||||
|
||||
// Build bulk request body
|
||||
for (const op of this.operations) {
|
||||
switch (op.operation) {
|
||||
case DocumentOperation.CREATE:
|
||||
bulkBody.push({ create: { _index: this.index, _id: op.documentId } });
|
||||
bulkBody.push(op.document);
|
||||
break;
|
||||
|
||||
case DocumentOperation.UPDATE:
|
||||
bulkBody.push({
|
||||
update: {
|
||||
_index: this.index,
|
||||
_id: op.documentId,
|
||||
...(op.seqNo !== undefined && { if_seq_no: op.seqNo }),
|
||||
...(op.primaryTerm !== undefined && { if_primary_term: op.primaryTerm }),
|
||||
},
|
||||
});
|
||||
bulkBody.push({ doc: op.document });
|
||||
break;
|
||||
|
||||
case DocumentOperation.UPSERT:
|
||||
bulkBody.push({ index: { _index: this.index, _id: op.documentId } });
|
||||
bulkBody.push(op.document);
|
||||
break;
|
||||
|
||||
case DocumentOperation.DELETE:
|
||||
bulkBody.push({ delete: { _index: this.index, _id: op.documentId } });
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Execute bulk request
|
||||
const response = await this.client.bulk({
|
||||
body: bulkBody,
|
||||
refresh: true, // Make changes immediately visible
|
||||
});
|
||||
|
||||
const took = Date.now() - startTime;
|
||||
|
||||
// Process results
|
||||
let successful = 0;
|
||||
let failed = 0;
|
||||
const errors: Array<{
|
||||
documentId: string;
|
||||
operation: DocumentOperation;
|
||||
error: string;
|
||||
statusCode: number;
|
||||
}> = [];
|
||||
|
||||
if (response.errors) {
|
||||
for (let i = 0; i < response.items.length; i++) {
|
||||
const item = response.items[i];
|
||||
const operation = this.operations[i];
|
||||
|
||||
const action = Object.keys(item)[0];
|
||||
const result = item[action as keyof typeof item] as any;
|
||||
|
||||
if (result.error) {
|
||||
failed++;
|
||||
errors.push({
|
||||
documentId: operation.documentId,
|
||||
operation: operation.operation,
|
||||
error: result.error.reason || result.error,
|
||||
statusCode: result.status,
|
||||
});
|
||||
} else {
|
||||
successful++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
successful = response.items.length;
|
||||
}
|
||||
|
||||
const result: BatchResult = {
|
||||
successful,
|
||||
failed,
|
||||
errors,
|
||||
took,
|
||||
};
|
||||
|
||||
if (failed > 0) {
|
||||
this.logger.warn('Batch operation had failures', {
|
||||
successful,
|
||||
failed,
|
||||
errors: errors.slice(0, 5), // Log first 5 errors
|
||||
});
|
||||
|
||||
if (failed === this.operations.length) {
|
||||
// Complete failure
|
||||
throw new BulkOperationError(
|
||||
'All bulk operations failed',
|
||||
successful,
|
||||
failed,
|
||||
errors
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up documents not seen in this session
|
||||
*/
|
||||
private async cleanupStaleDocuments(): Promise<void> {
|
||||
if (this.seenDocuments.size === 0) {
|
||||
return; // No documents to keep, skip cleanup
|
||||
}
|
||||
|
||||
this.logger.debug('Cleaning up stale documents', {
|
||||
index: this.index,
|
||||
seenCount: this.seenDocuments.size,
|
||||
});
|
||||
|
||||
try {
|
||||
// Use deleteByQuery to remove documents not in seen set
|
||||
// This is more efficient than the old scroll-and-compare approach
|
||||
const seenIds = Array.from(this.seenDocuments);
|
||||
|
||||
await this.client.deleteByQuery({
|
||||
index: this.index,
|
||||
body: {
|
||||
query: {
|
||||
bool: {
|
||||
must_not: {
|
||||
ids: {
|
||||
values: seenIds,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
refresh: true,
|
||||
});
|
||||
|
||||
this.logger.debug('Stale documents cleaned up', { index: this.index });
|
||||
} catch (error) {
|
||||
this.logger.warn('Failed to cleanup stale documents', undefined, {
|
||||
index: this.index,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
// Don't throw - cleanup is best-effort
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure session is active
|
||||
*/
|
||||
private ensureActive(): void {
|
||||
if (!this.isActive) {
|
||||
throw new Error('Session not active. Call start() first.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session statistics
|
||||
*/
|
||||
getStats(): {
|
||||
isActive: boolean;
|
||||
operationCount: number;
|
||||
seenDocumentCount: number;
|
||||
startTime: Date;
|
||||
} {
|
||||
return {
|
||||
isActive: this.isActive,
|
||||
operationCount: this.operations.length,
|
||||
seenDocumentCount: this.seenDocuments.size,
|
||||
startTime: this.startTimestamp,
|
||||
};
|
||||
}
|
||||
}
|
||||
16
ts/domain/documents/index.ts
Normal file
16
ts/domain/documents/index.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* Document management API
|
||||
*
|
||||
* This module provides:
|
||||
* - Fluent document manager with full CRUD operations
|
||||
* - Session-based batch operations with automatic cleanup
|
||||
* - Snapshot functionality for point-in-time analytics
|
||||
* - Async iteration over documents
|
||||
* - Optimistic locking support
|
||||
*
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
export * from './types.js';
|
||||
export * from './document-session.js';
|
||||
export * from './document-manager.js';
|
||||
122
ts/domain/documents/types.ts
Normal file
122
ts/domain/documents/types.ts
Normal file
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Document operation types
|
||||
*/
|
||||
export enum DocumentOperation {
|
||||
CREATE = 'create',
|
||||
UPDATE = 'update',
|
||||
UPSERT = 'upsert',
|
||||
DELETE = 'delete',
|
||||
}
|
||||
|
||||
/**
|
||||
* Document with metadata
|
||||
*/
|
||||
export interface DocumentWithMeta<T = unknown> {
|
||||
/** Document ID */
|
||||
_id: string;
|
||||
|
||||
/** Document source */
|
||||
_source: T;
|
||||
|
||||
/** Document version (for optimistic locking) */
|
||||
_version?: number;
|
||||
|
||||
/** Sequence number (for optimistic locking) */
|
||||
_seq_no?: number;
|
||||
|
||||
/** Primary term (for optimistic locking) */
|
||||
_primary_term?: number;
|
||||
|
||||
/** Document index */
|
||||
_index?: string;
|
||||
|
||||
/** Document score (from search) */
|
||||
_score?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch operation for bulk requests
|
||||
*/
|
||||
export interface BatchOperation<T = unknown> {
|
||||
operation: DocumentOperation;
|
||||
documentId: string;
|
||||
document?: T;
|
||||
version?: number;
|
||||
seqNo?: number;
|
||||
primaryTerm?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch result
|
||||
*/
|
||||
export interface BatchResult {
|
||||
successful: number;
|
||||
failed: number;
|
||||
errors: Array<{
|
||||
documentId: string;
|
||||
operation: DocumentOperation;
|
||||
error: string;
|
||||
statusCode: number;
|
||||
}>;
|
||||
took: number; // Time in milliseconds
|
||||
}
|
||||
|
||||
/**
|
||||
* Session configuration
|
||||
*/
|
||||
export interface SessionConfig {
|
||||
/** Only process documents newer than a timestamp */
|
||||
onlyNew?: boolean;
|
||||
|
||||
/** Start from a specific point in time */
|
||||
fromTimestamp?: Date;
|
||||
|
||||
/** Delete documents not seen in session */
|
||||
cleanupStale?: boolean;
|
||||
|
||||
/** Batch size for operations */
|
||||
batchSize?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Snapshot processor function
|
||||
*/
|
||||
export type SnapshotProcessor<T, R> = (
|
||||
iterator: AsyncIterableIterator<DocumentWithMeta<T>>,
|
||||
previousSnapshot: R | null
|
||||
) => Promise<R>;
|
||||
|
||||
/**
|
||||
* Snapshot metadata
|
||||
*/
|
||||
export interface SnapshotMeta<T = unknown> {
|
||||
date: Date;
|
||||
data: T;
|
||||
documentCount: number;
|
||||
processingTime: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Document iterator options
|
||||
*/
|
||||
export interface IteratorOptions {
|
||||
/** Batch size for scrolling */
|
||||
batchSize?: number;
|
||||
|
||||
/** Filter by timestamp */
|
||||
fromTimestamp?: Date;
|
||||
|
||||
/** Sort order */
|
||||
sort?: Array<{ [key: string]: 'asc' | 'desc' }>;
|
||||
|
||||
/** Query filter */
|
||||
query?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Point-in-time ID for pagination
|
||||
*/
|
||||
export interface PitId {
|
||||
id: string;
|
||||
keepAlive: string;
|
||||
}
|
||||
Reference in New Issue
Block a user