feat(metrics): add comprehensive metrics collection system
Implement real-time stats tracking including connection counts, request metrics, bandwidth usage, and route-specific monitoring. Adds MetricsCollector with observable streams for reactive monitoring integration.
This commit is contained in:
Juergen Kunz
250
test/test.keepalive-support.node.ts
Normal file
250
test/test.keepalive-support.node.ts
Normal file
@ -0,0 +1,250 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as net from 'net';
|
||||
import { SmartProxy } from '../ts/index.js';
|
||||
import * as plugins from '../ts/plugins.js';
|
||||
|
||||
tap.test('keepalive support - verify keepalive connections are properly handled', async (tools) => {
|
||||
console.log('\n=== KeepAlive Support Test ===');
|
||||
console.log('Purpose: Verify that keepalive connections are not prematurely cleaned up');
|
||||
|
||||
// Create a simple echo backend
|
||||
const echoBackend = net.createServer((socket) => {
|
||||
socket.on('data', (data) => {
|
||||
// Echo back received data
|
||||
try {
|
||||
socket.write(data);
|
||||
} catch (err) {
|
||||
// Ignore write errors during shutdown
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('error', (err) => {
|
||||
// Ignore errors from backend sockets
|
||||
console.log(`Backend socket error (expected during cleanup): ${err.code}`);
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
echoBackend.listen(9998, () => {
|
||||
console.log('✓ Echo backend started on port 9998');
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// Test 1: Standard keepalive treatment
|
||||
console.log('\n--- Test 1: Standard KeepAlive Treatment ---');
|
||||
|
||||
const proxy1 = new SmartProxy({
|
||||
routes: [{
|
||||
name: 'keepalive-route',
|
||||
match: { ports: 8590 },
|
||||
action: {
|
||||
type: 'forward',
|
||||
target: { host: 'localhost', port: 9998 }
|
||||
}
|
||||
}],
|
||||
keepAlive: true,
|
||||
keepAliveTreatment: 'standard',
|
||||
inactivityTimeout: 5000, // 5 seconds for faster testing
|
||||
enableDetailedLogging: false,
|
||||
});
|
||||
|
||||
await proxy1.start();
|
||||
console.log('✓ Proxy with standard keepalive started on port 8590');
|
||||
|
||||
// Create a keepalive connection
|
||||
const client1 = net.connect(8590, 'localhost');
|
||||
|
||||
// Add error handler to prevent unhandled errors
|
||||
client1.on('error', (err) => {
|
||||
console.log(`Client1 error (expected during cleanup): ${err.code}`);
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
client1.on('connect', () => {
|
||||
console.log('Client connected');
|
||||
client1.setKeepAlive(true, 1000);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// Send initial data
|
||||
client1.write('Hello keepalive\n');
|
||||
|
||||
// Wait for echo
|
||||
await new Promise<void>((resolve) => {
|
||||
client1.once('data', (data) => {
|
||||
console.log(`Received echo: ${data.toString().trim()}`);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// Check connection is marked as keepalive
|
||||
const cm1 = (proxy1 as any).connectionManager;
|
||||
const connections1 = cm1.getConnections();
|
||||
let keepAliveCount = 0;
|
||||
|
||||
for (const [id, record] of connections1) {
|
||||
if (record.hasKeepAlive) {
|
||||
keepAliveCount++;
|
||||
console.log(`KeepAlive connection ${id}: hasKeepAlive=${record.hasKeepAlive}`);
|
||||
}
|
||||
}
|
||||
|
||||
expect(keepAliveCount).toEqual(1);
|
||||
|
||||
// Wait to ensure it's not cleaned up prematurely
|
||||
await plugins.smartdelay.delayFor(6000);
|
||||
|
||||
const afterWaitCount1 = cm1.getConnectionCount();
|
||||
console.log(`Connections after 6s wait: ${afterWaitCount1}`);
|
||||
expect(afterWaitCount1).toEqual(1); // Should still be connected
|
||||
|
||||
// Send more data to keep it alive
|
||||
client1.write('Still alive\n');
|
||||
|
||||
// Clean up test 1
|
||||
client1.destroy();
|
||||
await proxy1.stop();
|
||||
await plugins.smartdelay.delayFor(500); // Wait for port to be released
|
||||
|
||||
// Test 2: Extended keepalive treatment
|
||||
console.log('\n--- Test 2: Extended KeepAlive Treatment ---');
|
||||
|
||||
const proxy2 = new SmartProxy({
|
||||
routes: [{
|
||||
name: 'keepalive-extended',
|
||||
match: { ports: 8591 },
|
||||
action: {
|
||||
type: 'forward',
|
||||
target: { host: 'localhost', port: 9998 }
|
||||
}
|
||||
}],
|
||||
keepAlive: true,
|
||||
keepAliveTreatment: 'extended',
|
||||
keepAliveInactivityMultiplier: 6,
|
||||
inactivityTimeout: 2000, // 2 seconds base, 12 seconds with multiplier
|
||||
enableDetailedLogging: false,
|
||||
});
|
||||
|
||||
await proxy2.start();
|
||||
console.log('✓ Proxy with extended keepalive started on port 8591');
|
||||
|
||||
const client2 = net.connect(8591, 'localhost');
|
||||
|
||||
// Add error handler to prevent unhandled errors
|
||||
client2.on('error', (err) => {
|
||||
console.log(`Client2 error (expected during cleanup): ${err.code}`);
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
client2.on('connect', () => {
|
||||
console.log('Client connected with extended timeout');
|
||||
client2.setKeepAlive(true, 1000);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// Send initial data
|
||||
client2.write('Extended keepalive\n');
|
||||
|
||||
// Check connection
|
||||
const cm2 = (proxy2 as any).connectionManager;
|
||||
await plugins.smartdelay.delayFor(1000);
|
||||
|
||||
const connections2 = cm2.getConnections();
|
||||
for (const [id, record] of connections2) {
|
||||
console.log(`Extended connection ${id}: hasKeepAlive=${record.hasKeepAlive}, treatment=extended`);
|
||||
}
|
||||
|
||||
// Wait 3 seconds (would timeout with standard treatment)
|
||||
await plugins.smartdelay.delayFor(3000);
|
||||
|
||||
const midWaitCount = cm2.getConnectionCount();
|
||||
console.log(`Connections after 3s (base timeout exceeded): ${midWaitCount}`);
|
||||
expect(midWaitCount).toEqual(1); // Should still be connected due to extended treatment
|
||||
|
||||
// Clean up test 2
|
||||
client2.destroy();
|
||||
await proxy2.stop();
|
||||
await plugins.smartdelay.delayFor(500); // Wait for port to be released
|
||||
|
||||
// Test 3: Immortal keepalive treatment
|
||||
console.log('\n--- Test 3: Immortal KeepAlive Treatment ---');
|
||||
|
||||
const proxy3 = new SmartProxy({
|
||||
routes: [{
|
||||
name: 'keepalive-immortal',
|
||||
match: { ports: 8592 },
|
||||
action: {
|
||||
type: 'forward',
|
||||
target: { host: 'localhost', port: 9998 }
|
||||
}
|
||||
}],
|
||||
keepAlive: true,
|
||||
keepAliveTreatment: 'immortal',
|
||||
inactivityTimeout: 1000, // 1 second - should be ignored for immortal
|
||||
enableDetailedLogging: false,
|
||||
});
|
||||
|
||||
await proxy3.start();
|
||||
console.log('✓ Proxy with immortal keepalive started on port 8592');
|
||||
|
||||
const client3 = net.connect(8592, 'localhost');
|
||||
|
||||
// Add error handler to prevent unhandled errors
|
||||
client3.on('error', (err) => {
|
||||
console.log(`Client3 error (expected during cleanup): ${err.code}`);
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
client3.on('connect', () => {
|
||||
console.log('Client connected with immortal treatment');
|
||||
client3.setKeepAlive(true, 1000);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// Send initial data
|
||||
client3.write('Immortal connection\n');
|
||||
|
||||
// Wait well beyond normal timeout
|
||||
await plugins.smartdelay.delayFor(5000);
|
||||
|
||||
const cm3 = (proxy3 as any).connectionManager;
|
||||
const immortalCount = cm3.getConnectionCount();
|
||||
console.log(`Immortal connections after 5s inactivity: ${immortalCount}`);
|
||||
expect(immortalCount).toEqual(1); // Should never timeout
|
||||
|
||||
// Verify zombie detection doesn't affect immortal connections
|
||||
console.log('\n--- Verifying zombie detection respects keepalive ---');
|
||||
|
||||
// Manually trigger inactivity check
|
||||
cm3.performOptimizedInactivityCheck();
|
||||
|
||||
await plugins.smartdelay.delayFor(1000);
|
||||
|
||||
const afterCheckCount = cm3.getConnectionCount();
|
||||
console.log(`Connections after manual inactivity check: ${afterCheckCount}`);
|
||||
expect(afterCheckCount).toEqual(1); // Should still be alive
|
||||
|
||||
// Clean up
|
||||
client3.destroy();
|
||||
await proxy3.stop();
|
||||
|
||||
// Close backend and wait for it to fully close
|
||||
await new Promise<void>((resolve) => {
|
||||
echoBackend.close(() => {
|
||||
console.log('Echo backend closed');
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
console.log('\n✓ All keepalive tests passed:');
|
||||
console.log(' - Standard treatment works correctly');
|
||||
console.log(' - Extended treatment applies multiplier');
|
||||
console.log(' - Immortal treatment never times out');
|
||||
console.log(' - Zombie detection respects keepalive settings');
|
||||
});
|
||||
|
||||
tap.start();
|
||||
280
test/test.metrics-collector.ts
Normal file
280
test/test.metrics-collector.ts
Normal file
@ -0,0 +1,280 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { SmartProxy } from '../ts/index.js';
|
||||
import * as net from 'net';
|
||||
import * as plugins from '../ts/plugins.js';
|
||||
|
||||
tap.test('MetricsCollector provides accurate metrics', async (tools) => {
|
||||
console.log('\n=== MetricsCollector Test ===');
|
||||
|
||||
// Create a simple echo server for testing
|
||||
const echoServer = net.createServer((socket) => {
|
||||
socket.on('data', (data) => {
|
||||
socket.write(data);
|
||||
});
|
||||
socket.on('error', () => {}); // Ignore errors
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
echoServer.listen(9995, () => {
|
||||
console.log('✓ Echo server started on port 9995');
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// Create SmartProxy with test routes
|
||||
const proxy = new SmartProxy({
|
||||
routes: [
|
||||
{
|
||||
name: 'test-route-1',
|
||||
match: { ports: 8700 },
|
||||
action: {
|
||||
type: 'forward',
|
||||
target: { host: 'localhost', port: 9995 }
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'test-route-2',
|
||||
match: { ports: 8701 },
|
||||
action: {
|
||||
type: 'forward',
|
||||
target: { host: 'localhost', port: 9995 }
|
||||
}
|
||||
}
|
||||
],
|
||||
enableDetailedLogging: true,
|
||||
});
|
||||
|
||||
await proxy.start();
|
||||
console.log('✓ Proxy started on ports 8700 and 8701');
|
||||
|
||||
// Get stats interface
|
||||
const stats = proxy.getStats();
|
||||
|
||||
// Test 1: Initial state
|
||||
console.log('\n--- Test 1: Initial State ---');
|
||||
expect(stats.getActiveConnections()).toEqual(0);
|
||||
expect(stats.getTotalConnections()).toEqual(0);
|
||||
expect(stats.getRequestsPerSecond()).toEqual(0);
|
||||
expect(stats.getConnectionsByRoute().size).toEqual(0);
|
||||
expect(stats.getConnectionsByIP().size).toEqual(0);
|
||||
|
||||
const throughput = stats.getThroughput();
|
||||
expect(throughput.bytesIn).toEqual(0);
|
||||
expect(throughput.bytesOut).toEqual(0);
|
||||
console.log('✓ Initial metrics are all zero');
|
||||
|
||||
// Test 2: Create connections and verify metrics
|
||||
console.log('\n--- Test 2: Active Connections ---');
|
||||
const clients: net.Socket[] = [];
|
||||
|
||||
// Create 3 connections to route 1
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const client = net.connect(8700, 'localhost');
|
||||
clients.push(client);
|
||||
await new Promise<void>((resolve) => {
|
||||
client.on('connect', resolve);
|
||||
client.on('error', () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
// Create 2 connections to route 2
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const client = net.connect(8701, 'localhost');
|
||||
clients.push(client);
|
||||
await new Promise<void>((resolve) => {
|
||||
client.on('connect', resolve);
|
||||
client.on('error', () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
// Wait for connections to be fully established and routed
|
||||
await plugins.smartdelay.delayFor(300);
|
||||
|
||||
// Verify connection counts
|
||||
expect(stats.getActiveConnections()).toEqual(5);
|
||||
expect(stats.getTotalConnections()).toEqual(5);
|
||||
console.log(`✓ Active connections: ${stats.getActiveConnections()}`);
|
||||
console.log(`✓ Total connections: ${stats.getTotalConnections()}`);
|
||||
|
||||
// Test 3: Connections by route
|
||||
console.log('\n--- Test 3: Connections by Route ---');
|
||||
const routeConnections = stats.getConnectionsByRoute();
|
||||
console.log('Route connections:', Array.from(routeConnections.entries()));
|
||||
|
||||
// Check if we have the expected counts
|
||||
let route1Count = 0;
|
||||
let route2Count = 0;
|
||||
for (const [routeName, count] of routeConnections) {
|
||||
if (routeName === 'test-route-1') route1Count = count;
|
||||
if (routeName === 'test-route-2') route2Count = count;
|
||||
}
|
||||
|
||||
expect(route1Count).toEqual(3);
|
||||
expect(route2Count).toEqual(2);
|
||||
console.log('✓ Route test-route-1 has 3 connections');
|
||||
console.log('✓ Route test-route-2 has 2 connections');
|
||||
|
||||
// Test 4: Connections by IP
|
||||
console.log('\n--- Test 4: Connections by IP ---');
|
||||
const ipConnections = stats.getConnectionsByIP();
|
||||
// All connections are from localhost (127.0.0.1 or ::1)
|
||||
let totalIPConnections = 0;
|
||||
for (const [ip, count] of ipConnections) {
|
||||
console.log(` IP ${ip}: ${count} connections`);
|
||||
totalIPConnections += count;
|
||||
}
|
||||
expect(totalIPConnections).toEqual(5);
|
||||
console.log('✓ Total connections by IP matches active connections');
|
||||
|
||||
// Test 5: RPS calculation
|
||||
console.log('\n--- Test 5: Requests Per Second ---');
|
||||
const rps = stats.getRequestsPerSecond();
|
||||
console.log(` Current RPS: ${rps.toFixed(2)}`);
|
||||
// We created 5 connections, so RPS should be > 0
|
||||
expect(rps).toBeGreaterThan(0);
|
||||
console.log('✓ RPS is greater than 0');
|
||||
|
||||
// Test 6: Throughput
|
||||
console.log('\n--- Test 6: Throughput ---');
|
||||
// Send some data through connections
|
||||
for (const client of clients) {
|
||||
if (!client.destroyed) {
|
||||
client.write('Hello metrics!\n');
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for data to be transmitted
|
||||
await plugins.smartdelay.delayFor(100);
|
||||
|
||||
const throughputAfter = stats.getThroughput();
|
||||
console.log(` Bytes in: ${throughputAfter.bytesIn}`);
|
||||
console.log(` Bytes out: ${throughputAfter.bytesOut}`);
|
||||
expect(throughputAfter.bytesIn).toBeGreaterThan(0);
|
||||
expect(throughputAfter.bytesOut).toBeGreaterThan(0);
|
||||
console.log('✓ Throughput shows bytes transferred');
|
||||
|
||||
// Test 7: Close some connections
|
||||
console.log('\n--- Test 7: Connection Cleanup ---');
|
||||
// Close first 2 clients
|
||||
clients[0].destroy();
|
||||
clients[1].destroy();
|
||||
|
||||
await plugins.smartdelay.delayFor(100);
|
||||
|
||||
expect(stats.getActiveConnections()).toEqual(3);
|
||||
expect(stats.getTotalConnections()).toEqual(5); // Total should remain the same
|
||||
console.log(`✓ Active connections reduced to ${stats.getActiveConnections()}`);
|
||||
console.log(`✓ Total connections still ${stats.getTotalConnections()}`);
|
||||
|
||||
// Test 8: Helper methods
|
||||
console.log('\n--- Test 8: Helper Methods ---');
|
||||
|
||||
// Test getTopIPs
|
||||
const topIPs = (stats as any).getTopIPs(5);
|
||||
expect(topIPs.length).toBeGreaterThan(0);
|
||||
console.log('✓ getTopIPs returns IP list');
|
||||
|
||||
// Test isIPBlocked
|
||||
const isBlocked = (stats as any).isIPBlocked('127.0.0.1', 10);
|
||||
expect(isBlocked).toEqual(false); // Should not be blocked with limit of 10
|
||||
console.log('✓ isIPBlocked works correctly');
|
||||
|
||||
// Test throughput rate
|
||||
const throughputRate = (stats as any).getThroughputRate();
|
||||
console.log(` Throughput rate: ${throughputRate.bytesInPerSec} bytes/sec in, ${throughputRate.bytesOutPerSec} bytes/sec out`);
|
||||
console.log('✓ getThroughputRate calculates rates');
|
||||
|
||||
// Cleanup
|
||||
console.log('\n--- Cleanup ---');
|
||||
for (const client of clients) {
|
||||
if (!client.destroyed) {
|
||||
client.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
await proxy.stop();
|
||||
echoServer.close();
|
||||
|
||||
console.log('\n✓ All MetricsCollector tests passed');
|
||||
});
|
||||
|
||||
// Test with mock data for unit testing
|
||||
tap.test('MetricsCollector unit test with mock data', async () => {
|
||||
console.log('\n=== MetricsCollector Unit Test ===');
|
||||
|
||||
// Create a mock SmartProxy with mock ConnectionManager
|
||||
const mockConnections = new Map([
|
||||
['conn1', {
|
||||
remoteIP: '192.168.1.1',
|
||||
routeName: 'api',
|
||||
bytesReceived: 1000,
|
||||
bytesSent: 500,
|
||||
incomingStartTime: Date.now() - 5000
|
||||
}],
|
||||
['conn2', {
|
||||
remoteIP: '192.168.1.1',
|
||||
routeName: 'web',
|
||||
bytesReceived: 2000,
|
||||
bytesSent: 1500,
|
||||
incomingStartTime: Date.now() - 10000
|
||||
}],
|
||||
['conn3', {
|
||||
remoteIP: '192.168.1.2',
|
||||
routeName: 'api',
|
||||
bytesReceived: 500,
|
||||
bytesSent: 250,
|
||||
incomingStartTime: Date.now() - 3000
|
||||
}]
|
||||
]);
|
||||
|
||||
const mockSmartProxy = {
|
||||
connectionManager: {
|
||||
getConnectionCount: () => mockConnections.size,
|
||||
getConnections: () => mockConnections,
|
||||
getTerminationStats: () => ({
|
||||
incoming: { normal: 10, timeout: 2, error: 1 }
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
// Import MetricsCollector directly
|
||||
const { MetricsCollector } = await import('../ts/proxies/smart-proxy/metrics-collector.js');
|
||||
const metrics = new MetricsCollector(mockSmartProxy as any);
|
||||
|
||||
// Test metrics calculation
|
||||
console.log('\n--- Testing with Mock Data ---');
|
||||
|
||||
expect(metrics.getActiveConnections()).toEqual(3);
|
||||
console.log(`✓ Active connections: ${metrics.getActiveConnections()}`);
|
||||
|
||||
expect(metrics.getTotalConnections()).toEqual(16); // 3 active + 13 terminated
|
||||
console.log(`✓ Total connections: ${metrics.getTotalConnections()}`);
|
||||
|
||||
const routeConns = metrics.getConnectionsByRoute();
|
||||
expect(routeConns.get('api')).toEqual(2);
|
||||
expect(routeConns.get('web')).toEqual(1);
|
||||
console.log('✓ Connections by route calculated correctly');
|
||||
|
||||
const ipConns = metrics.getConnectionsByIP();
|
||||
expect(ipConns.get('192.168.1.1')).toEqual(2);
|
||||
expect(ipConns.get('192.168.1.2')).toEqual(1);
|
||||
console.log('✓ Connections by IP calculated correctly');
|
||||
|
||||
const throughput = metrics.getThroughput();
|
||||
expect(throughput.bytesIn).toEqual(3500);
|
||||
expect(throughput.bytesOut).toEqual(2250);
|
||||
console.log(`✓ Throughput: ${throughput.bytesIn} bytes in, ${throughput.bytesOut} bytes out`);
|
||||
|
||||
// Test RPS tracking
|
||||
metrics.recordRequest();
|
||||
metrics.recordRequest();
|
||||
metrics.recordRequest();
|
||||
|
||||
const rps = metrics.getRequestsPerSecond();
|
||||
expect(rps).toBeGreaterThan(0);
|
||||
console.log(`✓ RPS tracking works: ${rps.toFixed(2)} req/sec`);
|
||||
|
||||
console.log('\n✓ All unit tests passed');
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
Reference in New Issue
Block a user