/** * Comprehensive Logging API Example * * Demonstrates enterprise logging with structured log ingestion */ import { createConfig, ElasticsearchConnectionManager, LogLevel, LogDestination, addHostInfo, addEnvironment, addServiceInfo, addTimestamp, sanitizeSensitiveData, addDynamicTags, chainEnrichers, } from '../../index.js'; async function main() { console.log('=== Logging API Example ===\n'); // ============================================================================ // Step 1: Configuration // ============================================================================ console.log('Step 1: Configuring Elasticsearch connection...'); const config = createConfig() .fromEnv() .nodes(process.env.ELASTICSEARCH_URL || 'http://localhost:9200') .basicAuth( process.env.ELASTICSEARCH_USERNAME || 'elastic', process.env.ELASTICSEARCH_PASSWORD || 'changeme' ) .timeout(30000) .retries(3) .logLevel(LogLevel.INFO) .enableMetrics(true) .enableTracing(true, { serviceName: 'logging-example', serviceVersion: '1.0.0' }) .build(); // ============================================================================ // Step 2: Initialize Connection // ============================================================================ console.log('Step 2: Initializing connection manager...'); const connectionManager = ElasticsearchConnectionManager.getInstance(config); await connectionManager.initialize(); console.log('✓ Connection manager initialized\n'); // ============================================================================ // Step 3: Basic Log Destination // ============================================================================ console.log('Step 3: Creating basic log destination...'); const basicLogDest = new LogDestination({ indexPattern: 'logs-example-basic-{now/d}', batchSize: 50, flushIntervalMs: 3000, autoCreateTemplate: true, }); await basicLogDest.initialize(); console.log('✓ Basic log destination initialized'); // Send basic logs await basicLogDest.send({ timestamp: new Date().toISOString(), level: 'INFO', message: 'Application started', service: 'example-app', version: '1.0.0', }); await basicLogDest.send({ timestamp: new Date().toISOString(), level: 'WARN', message: 'High memory usage detected', metadata: { memoryUsage: '85%', threshold: '80%', }, }); await basicLogDest.send({ timestamp: new Date().toISOString(), level: 'ERROR', message: 'Failed to connect to database', error: { name: 'ConnectionError', message: 'ECONNREFUSED', stack: 'Error: ECONNREFUSED\\n at ...', code: 'ECONNREFUSED', }, }); await basicLogDest.flush(); console.log('✓ Basic logs sent (3 logs)'); console.log(' Stats:', basicLogDest.getStats()); console.log(); // ============================================================================ // Step 4: Log Destination with Enrichers // ============================================================================ console.log('Step 4: Creating log destination with enrichers...'); const enrichedLogDest = new LogDestination({ indexPattern: 'logs-example-enriched-{now/d}', batchSize: 100, flushIntervalMs: 5000, enrichers: [ addTimestamp, addHostInfo, addEnvironment, addServiceInfo, sanitizeSensitiveData([ { path: 'metadata.password' }, { path: 'metadata.apiKey' }, { path: 'metadata.token' }, ]), ], }); await enrichedLogDest.initialize(); console.log('✓ Enriched log destination initialized'); // Send logs that will be enriched await enrichedLogDest.send({ timestamp: new Date().toISOString(), level: 'INFO', message: 'User authenticated', metadata: { userId: 'user-123', username: 'john.doe', password: 'supersecret', // Will be sanitized ipAddress: '192.168.1.100', }, }); await enrichedLogDest.send({ timestamp: new Date().toISOString(), level: 'INFO', message: 'API request processed', metadata: { method: 'POST', path: '/api/users', statusCode: 201, apiKey: 'sk-1234567890', // Will be sanitized duration: 45, }, }); await enrichedLogDest.flush(); console.log('✓ Enriched logs sent (2 logs)'); console.log(' Logs enriched with: timestamp, host, environment, service info'); console.log(' Sensitive data sanitized: password, apiKey'); console.log(); // ============================================================================ // Step 5: Sampling Strategies // ============================================================================ console.log('Step 5: Demonstrating sampling strategies...'); // 5.1: Errors-only sampling console.log('5.1: Errors-only sampling'); const errorsOnlyDest = new LogDestination({ indexPattern: 'logs-example-errors-{now/d}', batchSize: 50, flushIntervalMs: 3000, sampling: { strategy: 'errors-only', }, }); await errorsOnlyDest.initialize(); // Send mixed logs await errorsOnlyDest.send({ timestamp: new Date().toISOString(), level: 'INFO', message: 'This will be sampled out', }); await errorsOnlyDest.send({ timestamp: new Date().toISOString(), level: 'ERROR', message: 'This error will be kept', error: { name: 'Error', message: 'Something went wrong' }, }); await errorsOnlyDest.flush(); console.log(' Sent 2 logs (1 INFO, 1 ERROR)'); const errorsStats = errorsOnlyDest.getStats(); console.log(` Indexed: ${errorsStats.totalSuccessful}, Sampled out: ${errorsStats.totalSampled}`); console.log(); // 5.2: Percentage sampling console.log('5.2: Percentage sampling (10%)'); const percentageDest = new LogDestination({ indexPattern: 'logs-example-percentage-{now/d}', batchSize: 50, flushIntervalMs: 3000, sampling: { strategy: 'percentage', percentage: 10, alwaysSampleErrors: true, }, }); await percentageDest.initialize(); // Send many logs for (let i = 0; i < 100; i++) { await percentageDest.send({ timestamp: new Date().toISOString(), level: 'INFO', message: `Log ${i}`, }); } // Send an error (should always be kept) await percentageDest.send({ timestamp: new Date().toISOString(), level: 'ERROR', message: 'Error log (always kept)', }); await percentageDest.flush(); const percentageStats = percentageDest.getStats(); console.log(` Sent 101 logs (100 INFO, 1 ERROR)`); console.log(` Indexed: ${percentageStats.totalSuccessful} (~10% of INFO + 1 ERROR)`); console.log(` Sampled out: ${percentageStats.totalSampled}`); console.log(); // ============================================================================ // Step 6: Dynamic Tags // ============================================================================ console.log('Step 6: Adding dynamic tags based on content...'); const taggedDest = new LogDestination({ indexPattern: 'logs-example-tagged-{now/d}', batchSize: 50, flushIntervalMs: 3000, enrichers: [ addDynamicTags([ { condition: (entry) => entry.level === 'ERROR', tag: 'alert', }, { condition: (entry) => entry.message.toLowerCase().includes('payment') || entry.message.toLowerCase().includes('transaction'), tag: 'financial', }, { condition: (entry) => entry.metadata?.statusCode !== undefined && (entry.metadata.statusCode as number) >= 500, tag: 'server-error', }, ]), ], }); await taggedDest.initialize(); await taggedDest.send({ timestamp: new Date().toISOString(), level: 'INFO', message: 'Payment processed successfully', metadata: { amount: 99.99, currency: 'USD', statusCode: 200 }, }); await taggedDest.send({ timestamp: new Date().toISOString(), level: 'ERROR', message: 'Transaction failed', metadata: { statusCode: 500 }, error: { name: 'PaymentError', message: 'Card declined' }, }); await taggedDest.flush(); console.log('✓ Tagged logs sent'); console.log(' First log tagged: [financial]'); console.log(' Second log tagged: [alert, financial, server-error]'); console.log(); // ============================================================================ // Step 7: ILM (Index Lifecycle Management) // ============================================================================ console.log('Step 7: Creating log destination with ILM policy...'); const ilmDest = new LogDestination({ indexPattern: 'logs-example-ilm-{now/d}', batchSize: 50, flushIntervalMs: 3000, ilm: { name: 'logs-example-policy', hotDuration: '7d', warmDuration: '30d', deleteDuration: '90d', rollover: { maxSize: '50gb', maxAge: '1d', }, }, }); await ilmDest.initialize(); console.log('✓ ILM policy created'); console.log(' Hot: 7 days → Warm: 30 days → Delete: 90 days'); console.log(' Rollover: 50GB or 1 day'); console.log(); // ============================================================================ // Step 8: Metric Extraction // ============================================================================ console.log('Step 8: Extracting metrics from logs...'); const metricsDest = new LogDestination({ indexPattern: 'logs-example-metrics-{now/d}', batchSize: 50, flushIntervalMs: 3000, metrics: [ { name: 'api_response_time', field: 'metrics.duration', type: 'histogram', labels: ['metadata.method', 'metadata.statusCode'], }, { name: 'errors_total', field: 'level', type: 'counter', labels: ['error.name'], }, ], }); await metricsDest.initialize(); // Send logs with metrics await metricsDest.send({ timestamp: new Date().toISOString(), level: 'INFO', message: 'API request completed', metadata: { method: 'GET', path: '/api/users', statusCode: 200, }, metrics: { duration: 42, }, }); await metricsDest.send({ timestamp: new Date().toISOString(), level: 'ERROR', message: 'API request failed', metadata: { method: 'POST', statusCode: 500, }, error: { name: 'ValidationError', message: 'Invalid input', }, metrics: { duration: 15, }, }); await metricsDest.flush(); console.log('✓ Logs with metrics sent'); console.log(' Metrics extracted: api_response_time, errors_total'); console.log(); // ============================================================================ // Step 9: Chained Enrichers // ============================================================================ console.log('Step 9: Using chained enrichers...'); const chainedDest = new LogDestination({ indexPattern: 'logs-example-chained-{now/d}', batchSize: 50, flushIntervalMs: 3000, enrichers: [ chainEnrichers( addTimestamp, addHostInfo, addEnvironment, addServiceInfo, sanitizeSensitiveData([{ path: 'metadata.secret' }]), addDynamicTags([ { condition: (entry) => entry.level === 'ERROR', tag: 'needs-attention', }, ]) ), ], }); await chainedDest.initialize(); await chainedDest.send({ timestamp: new Date().toISOString(), level: 'ERROR', message: 'Critical error occurred', metadata: { secret: 'should-be-redacted', component: 'auth-service', }, }); await chainedDest.flush(); console.log('✓ Log sent through enrichment chain'); console.log(' Applied: timestamp → host → env → service → sanitize → tags'); console.log(); // ============================================================================ // Step 10: Statistics and Monitoring // ============================================================================ console.log('Step 10: Reviewing statistics...\n'); const stats = enrichedLogDest.getStats(); console.log('Enriched Log Destination Stats:'); console.log(` Total logs: ${stats.totalLogs}`); console.log(` Successfully indexed: ${stats.totalSuccessful}`); console.log(` Failed: ${stats.totalFailed}`); console.log(` Sampled out: ${stats.totalSampled}`); console.log(` Dropped (queue overflow): ${stats.totalDropped}`); console.log(` Current queue size: ${stats.queueSize}`); console.log(` Avg batch duration: ${stats.avgBatchDurationMs.toFixed(2)}ms`); if (stats.lastFlushAt) { console.log(` Last flush: ${stats.lastFlushAt.toISOString()}`); } console.log(); // ============================================================================ // Step 11: Cleanup // ============================================================================ console.log('Step 11: Cleanup...'); await basicLogDest.destroy(); await enrichedLogDest.destroy(); await errorsOnlyDest.destroy(); await percentageDest.destroy(); await taggedDest.destroy(); await ilmDest.destroy(); await metricsDest.destroy(); await chainedDest.destroy(); console.log('✓ All log destinations destroyed (flushed and closed)'); await connectionManager.destroy(); console.log('✓ Connection closed\n'); console.log('=== Logging API Example Complete ==='); console.log('\nKey Features Demonstrated:'); console.log(' ✓ Basic structured logging'); console.log(' ✓ Log enrichment (host, environment, service info)'); console.log(' ✓ Sensitive data sanitization'); console.log(' ✓ Sampling strategies (errors-only, percentage)'); console.log(' ✓ Dynamic tagging based on content'); console.log(' ✓ ILM (Index Lifecycle Management)'); console.log(' ✓ Metric extraction from logs'); console.log(' ✓ Chained enrichers'); console.log(' ✓ Batching and auto-flushing'); console.log(' ✓ Statistics and monitoring'); } // Run the example main().catch((error) => { console.error('Example failed:', error); process.exit(1); });