git.zone/tsview
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(streaming): add real-time streaming (MongoDB change streams & S3 bucket watchers) with WebSocket subscriptions and activity stream UI

Browse Source
This commit is contained in:
Jürgen Kunz
2026-01-25 21:41:55 +00:00
parent c60cbf5215
commit 20e08d123f
23 changed files with 2456 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
changelog.md
View File

@@ -1,5 +1,15 @@
# Changelog
## 2026-01-25 - 1.5.0 - feat(streaming)
add real-time streaming (MongoDB change streams & S3 bucket watchers) with WebSocket subscriptions and activity stream UI
- Server: add ChangeStreamManager to manage MongoDB change streams and S3 BucketWatcher subscriptions, handle subscription lifecycle, activity ring buffer and push events via TypedSocket.
- API: introduce streaming TypedRequest interfaces and register new handlers (subscribeMongo/unsubscribeMongo, subscribeS3/unsubscribeS3, subscribeActivity/unsubscribeActivity, getRecentActivity, push events).
- Client: add ChangeStreamService (TypedSocket connection, reconnection, RxJS subjects) and integrate into tsview-app, mongo and S3 browsers to show live status, change counters and refresh content on updates.
- UI: add new Activity Stream component (tsview-activity-stream), Activity tab and navigation, plus visual indicators (Live/Offline, change badges, auto-scroll and filters).
- Dependencies & plugins: add @api.global/typedsocket and @push.rocks/smartrx, bump @push.rocks/smartbucket; expose smartrx/typedsocket in plugins for server and web bundles.
- Docs: update readme.hints.md with Real-Time Streaming architecture, interfaces and dependency notes.
## 2026-01-25 - 1.4.0 - feat(web)
add database overview panel, collection overview and resizable panels; show/hide system databases; use code editor with change-tracking in document view; add getDatabaseStats API and typings; enable overwrite for S3 uploads

4
package.json
View File

@@ -19,11 +19,13 @@
"tsview": "cli.js"
},
"devDependencies": {
"@api.global/typedsocket": "^4.1.0",
"@git.zone/tsbuild": "^4.1.2",
"@git.zone/tsbundle": "^2.8.3",
"@git.zone/tsrun": "^2.0.1",
"@git.zone/tstest": "^3.1.7",
"@git.zone/tswatch": "3.0.1",
"@push.rocks/smartrx": "^3.0.10",
"@types/node": "^25.0.10"
},
"dependencies": {
@@ -35,7 +37,7 @@
"@design.estate/dees-element": "^2.1.5",
"@push.rocks/early": "^4.0.4",
"@push.rocks/npmextra": "^5.3.3",
"@push.rocks/smartbucket": "^4.3.1",
"@push.rocks/smartbucket": "^4.4.1",
"@push.rocks/smartcli": "^4.0.20",
"@push.rocks/smartdata": "^7.0.15",
"@push.rocks/smartfile": "^13.1.2",

18
pnpm-lock.yaml generated
View File

@@ -33,8 +33,8 @@ importers:
specifier: ^5.3.3
version: 5.3.3
'@push.rocks/smartbucket':
specifier: ^4.3.1
version: 4.3.1
specifier: ^4.4.1
version: 4.4.1
'@push.rocks/smartcli':
specifier: ^4.0.20
version: 4.0.20
@@ -66,6 +66,9 @@ importers:
specifier: ^7.0.0
version: 7.0.0(socks@2.8.7)
devDependencies:
'@api.global/typedsocket':
specifier: ^4.1.0
version: 4.1.0(@push.rocks/smartserve@2.0.1)
'@git.zone/tsbuild':
specifier: ^4.1.2
version: 4.1.2
@@ -81,6 +84,9 @@ importers:
'@git.zone/tswatch':
specifier: 3.0.1
version: 3.0.1(@tiptap/pm@2.27.2)
'@push.rocks/smartrx':
specifier: ^3.0.10
version: 3.0.10
'@types/node':
specifier: ^25.0.10
version: 25.0.10
@@ -826,8 +832,8 @@ packages:
'@push.rocks/smartbucket@3.3.10':
resolution: {integrity: sha512-0H2MioALspC8Aj0Q1FPCs2w4k2u9oJg7Q5yM8+1TZo7aRfrdxgM5HQ7z3apUaqC3ZEDewW6vSlttjHFHhMEC3A==}
'@push.rocks/smartbucket@4.3.1':
resolution: {integrity: sha512-fMA8w98/E+usaaLkLm6wDj1XSpR0shTtG8AxTdwWIlH1YemQj/aCf4wReezDxUFVoUpC3HMzzV2RTFtQvHndeQ==}
'@push.rocks/smartbucket@4.4.1':
resolution: {integrity: sha512-68GFLgJKW+LXvuN+yuV8O/FozGMecraoT+PkI5whdRPFe7N3u2iYIHWAUjvQvVU4ygpdJv0kih2JDf5k3PYycw==}
'@push.rocks/smartbuffer@3.0.5':
resolution: {integrity: sha512-pWYF08Mn8s/KF/9nHRk7pZPzuMjmYVQay2c5gGexdayxn1W4eCSYYhWH73vR2JBfGeGq/izbRNuUuEaIEeTIKA==}
@@ -5527,7 +5533,7 @@ snapshots:
transitivePeerDependencies:
- aws-crt
'@push.rocks/smartbucket@4.3.1':
'@push.rocks/smartbucket@4.4.1':
dependencies:
'@aws-sdk/client-s3': 3.975.0
'@push.rocks/smartmime': 2.0.4
@@ -5983,7 +5989,7 @@ snapshots:
'@push.rocks/smarts3@3.0.3':
dependencies:
'@push.rocks/smartbucket': 4.3.1
'@push.rocks/smartbucket': 4.4.1
'@push.rocks/smartfs': 1.3.1
'@push.rocks/smartpath': 6.0.0
'@push.rocks/smartxml': 2.0.0

44
readme.hints.md
View File

@@ -53,3 +53,47 @@ typedrouter.addTypedHandler(
)
);
```
## Real-Time Streaming (v1.5.0+)
### Architecture
- `ts/streaming/` - Backend streaming infrastructure
- `classes.changestream-manager.ts` - Manages MongoDB and S3 watchers
- `interfaces.streaming.ts` - TypedRequest interfaces for subscriptions
- `ts_web/services/changestream.service.ts` - Frontend WebSocket client
### MongoDB Change Streams
- Uses native MongoDB Change Streams via `SmartdataDb.mongoDbClient`
- Subscription per collection: `db/collection`
- Events: insert, update, delete, replace, drop
### S3 Change Detection
- Uses `@push.rocks/smartbucket` BucketWatcher (polling-based)
- Polling interval: 5 seconds
- Events: add, modify, delete
### Frontend Components
- `tsview-activity-stream.ts` - Combined activity view with filtering
- MongoDB/S3 browsers auto-subscribe to current resource
- Visual indicators for "Live" status and recent change count
### Key Streaming Interfaces
```typescript
// Subscribe to collection changes
IReq_SubscribeMongo: { database, collection } -> { success, subscriptionId }
// Subscribe to bucket changes
IReq_SubscribeS3: { bucket, prefix? } -> { success, subscriptionId }
// Subscribe to activity stream (all changes)
IReq_SubscribeActivity: {} -> { success, subscriptionId }
// Server pushes changes to subscribed clients
IReq_PushMongoChange: { event: IMongoChangeEvent } -> { received }
IReq_PushS3Change: { event: IS3ChangeEvent } -> { received }
IReq_PushActivityEvent: { event: IActivityEvent } -> { received }
```
### Dependencies Added
- `@api.global/typedsocket` - WebSocket client/server
- `@push.rocks/smartrx` - RxJS utilities

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@git.zone/tsview',
version: '1.4.0',
version: '1.5.0',
description: 'A CLI tool for viewing S3 and MongoDB data with a web UI'
}

2
ts/bundled_ui.ts
View File

File diff suppressed because one or more lines are too long

5
ts/interfaces/index.ts
View File

@@ -535,3 +535,8 @@ export interface IReq_GetDatabaseStats extends plugins.typedrequestInterfaces.im
stats: IDatabaseStats | null;
};
}
// ===========================================
// Re-export streaming interfaces
// ===========================================
export * from '../streaming/interfaces.streaming.js';

2
ts/plugins.ts
View File

@@ -17,6 +17,7 @@ import * as smartnetwork from '@push.rocks/smartnetwork';
import * as smartopen from '@push.rocks/smartopen';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrx from '@push.rocks/smartrx';
export {
early,
@@ -31,6 +32,7 @@ export {
smartopen,
smartpath,
smartpromise,
smartrx,
};
// AWS S3 SDK for direct S3 operations

168
ts/server/classes.viewserver.ts
View File

@@ -1,7 +1,9 @@
import * as plugins from '../plugins.js';
import type { TsView } from '../tsview.classes.tsview.js';
import type * as interfaces from '../interfaces/index.js';
import { registerS3Handlers } from '../api/handlers.s3.js';
import { registerMongoHandlers } from '../api/handlers.mongodb.js';
import { ChangeStreamManager } from '../streaming/index.js';
import { files as bundledUiFiles } from '../bundled_ui.js';
/**
@@ -11,6 +13,7 @@ export class ViewServer {
private tsview: TsView;
private port: number;
private typedServer: plugins.typedserver.TypedServer | null = null;
private changeStreamManager: ChangeStreamManager | null = null;
public typedrouter: plugins.typedrequest.TypedRouter;
constructor(tsview: TsView, port: number) {
@@ -41,14 +44,179 @@ export class ViewServer {
await registerMongoHandlers(this.typedServer.typedrouter, this.tsview);
}
// Initialize ChangeStreamManager for real-time updates
this.changeStreamManager = new ChangeStreamManager(this.tsview);
// Register streaming handlers
await this.registerStreamingHandlers();
// Start server
await this.typedServer.start();
// Set TypedSocket reference after server starts
if (this.typedServer.typedsocket) {
this.changeStreamManager.setTypedSocket(this.typedServer.typedsocket);
}
}
/**
* Register WebSocket streaming handlers
*/
private async registerStreamingHandlers(): Promise<void> {
if (!this.typedServer || !this.changeStreamManager) return;
const typedrouter = this.typedServer.typedrouter;
// Subscribe to MongoDB collection changes
typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.IReq_SubscribeMongo>(
'subscribeMongo',
async (reqData, context) => {
const connectionId = this.getConnectionId(context);
if (!connectionId) {
return { success: false, subscriptionId: '' };
}
const result = await this.changeStreamManager!.subscribeToMongo(
connectionId,
reqData.database,
reqData.collection
);
return result;
}
)
);
// Unsubscribe from MongoDB collection changes
typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.IReq_UnsubscribeMongo>(
'unsubscribeMongo',
async (reqData, context) => {
const connectionId = this.getConnectionId(context);
if (!connectionId) {
return { success: false };
}
const success = await this.changeStreamManager!.unsubscribeFromMongo(
connectionId,
reqData.database,
reqData.collection
);
return { success };
}
)
);
// Subscribe to S3 bucket changes
typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.IReq_SubscribeS3>(
'subscribeS3',
async (reqData, context) => {
const connectionId = this.getConnectionId(context);
if (!connectionId) {
return { success: false, subscriptionId: '' };
}
const result = await this.changeStreamManager!.subscribeToS3(
connectionId,
reqData.bucket,
reqData.prefix
);
return result;
}
)
);
// Unsubscribe from S3 bucket changes
typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.IReq_UnsubscribeS3>(
'unsubscribeS3',
async (reqData, context) => {
const connectionId = this.getConnectionId(context);
if (!connectionId) {
return { success: false };
}
const success = await this.changeStreamManager!.unsubscribeFromS3(
connectionId,
reqData.bucket,
reqData.prefix
);
return { success };
}
)
);
// Subscribe to activity stream
typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.IReq_SubscribeActivity>(
'subscribeActivity',
async (reqData, context) => {
const connectionId = this.getConnectionId(context);
if (!connectionId) {
return { success: false, subscriptionId: '' };
}
const result = await this.changeStreamManager!.subscribeToActivity(connectionId);
return result;
}
)
);
// Unsubscribe from activity stream
typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.IReq_UnsubscribeActivity>(
'unsubscribeActivity',
async (reqData, context) => {
const connectionId = this.getConnectionId(context);
if (!connectionId) {
return { success: false };
}
const success = await this.changeStreamManager!.unsubscribeFromActivity(connectionId);
return { success };
}
)
);
// Get recent activity events
typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.IReq_GetRecentActivity>(
'getRecentActivity',
async (reqData) => {
const events = this.changeStreamManager!.getRecentActivity(reqData.limit || 100);
return { events };
}
)
);
}
/**
* Extract connection ID from request context
*/
private getConnectionId(context: any): string | null {
// Try to get connection ID from WebSocket context
if (context?.socketConnection?.socketId) {
return context.socketConnection.socketId;
}
if (context?.socketConnection?.alias) {
return context.socketConnection.alias;
}
// Fallback: generate a unique ID for HTTP requests
// Note: Real-time streaming requires WebSocket connection
return null;
}
/**
* Stop the server
*/
public async stop(): Promise<void> {
// Stop change stream manager first
if (this.changeStreamManager) {
await this.changeStreamManager.stop();
this.changeStreamManager = null;
}
if (this.typedServer) {
await this.typedServer.stop();
this.typedServer = null;

591
ts/streaming/classes.changestream-manager.ts Normal file
View File

@@ -0,0 +1,591 @@
import * as plugins from '../plugins.js';
import type { TsView } from '../tsview.classes.tsview.js';
import type * as interfaces from './interfaces.streaming.js';
import type { IS3ChangeEvent } from '@push.rocks/smartbucket';
/**
* Subscription entry tracking a client's subscription to a resource
*/
interface ISubscriptionEntry {
connectionId: string;
subscriptionId: string;
createdAt: Date;
}
/**
* MongoDB watcher entry
*/
interface IMongoWatcherEntry {
watcher: plugins.mongodb.ChangeStream;
subscriptions: Map<string, ISubscriptionEntry>; // connectionId -> subscription
}
/**
* S3 watcher entry
*/
interface IS3WatcherEntry {
watcher: plugins.smartbucket.BucketWatcher;
subscriptions: Map<string, ISubscriptionEntry>; // connectionId -> subscription
}
/**
* ChangeStreamManager manages real-time change streaming for both MongoDB and S3.
*
* Features:
* - MongoDB Change Streams for real-time database updates
* - S3 BucketWatcher for polling-based S3 change detection
* - Subscription management per WebSocket client
* - Activity stream with ring buffer for recent events
* - Automatic cleanup on client disconnect
*/
export class ChangeStreamManager {
private tsview: TsView;
private typedSocket: plugins.typedserver.TypedServer['typedsocket'] | null = null;
// MongoDB watchers: "db/collection" -> watcher entry
private mongoWatchers: Map<string, IMongoWatcherEntry> = new Map();
// S3 watchers: "bucket/prefix" -> watcher entry
private s3Watchers: Map<string, IS3WatcherEntry> = new Map();
// Activity subscribers: connectionId -> subscription entry
private activitySubscribers: Map<string, ISubscriptionEntry> = new Map();
// Activity ring buffer (max 1000 events)
private activityBuffer: interfaces.IActivityEvent[] = [];
private readonly ACTIVITY_BUFFER_SIZE = 1000;
// Counter for generating unique subscription IDs
private subscriptionCounter = 0;
constructor(tsview: TsView) {
this.tsview = tsview;
}
/**
* Initialize the manager with a TypedSocket instance
*/
public setTypedSocket(typedSocket: plugins.typedserver.TypedServer['typedsocket']): void {
this.typedSocket = typedSocket;
}
/**
* Generate a unique subscription ID
*/
private generateSubscriptionId(): string {
return `sub_${Date.now()}_${++this.subscriptionCounter}`;
}
/**
* Get the MongoDB key for a database/collection pair
*/
private getMongoKey(database: string, collection: string): string {
return `${database}/${collection}`;
}
/**
* Get the S3 key for a bucket/prefix pair
*/
private getS3Key(bucket: string, prefix?: string): string {
return prefix ? `${bucket}/${prefix}` : bucket;
}
// ===========================================
// MongoDB Change Streams
// ===========================================
/**
* Subscribe a client to MongoDB collection changes
*/
public async subscribeToMongo(
connectionId: string,
database: string,
collection: string
): Promise<{ success: boolean; subscriptionId: string }> {
const key = this.getMongoKey(database, collection);
let entry = this.mongoWatchers.get(key);
// Create watcher if it doesn't exist
if (!entry) {
const watcher = await this.createMongoWatcher(database, collection);
if (!watcher) {
return { success: false, subscriptionId: '' };
}
entry = {
watcher,
subscriptions: new Map(),
};
this.mongoWatchers.set(key, entry);
}
// Add subscription
const subscriptionId = this.generateSubscriptionId();
entry.subscriptions.set(connectionId, {
connectionId,
subscriptionId,
createdAt: new Date(),
});
console.log(`[ChangeStream] MongoDB subscription added: ${key} for connection ${connectionId}`);
return { success: true, subscriptionId };
}
/**
* Unsubscribe a client from MongoDB collection changes
*/
public async unsubscribeFromMongo(
connectionId: string,
database: string,
collection: string
): Promise<boolean> {
const key = this.getMongoKey(database, collection);
const entry = this.mongoWatchers.get(key);
if (!entry) {
return false;
}
entry.subscriptions.delete(connectionId);
console.log(`[ChangeStream] MongoDB subscription removed: ${key} for connection ${connectionId}`);
// Close watcher if no more subscribers
if (entry.subscriptions.size === 0) {
await this.closeMongoWatcher(key);
}
return true;
}
/**
* Create a MongoDB change stream for a collection
*/
private async createMongoWatcher(
database: string,
collection: string
): Promise<plugins.mongodb.ChangeStream | null> {
try {
const db = await this.tsview.getMongoDb();
if (!db) {
console.error('[ChangeStream] MongoDB not configured');
return null;
}
const client = (db as any).mongoDbClient;
const mongoDb = client.db(database);
const mongoCollection = mongoDb.collection(collection);
// Create change stream
const changeStream = mongoCollection.watch([], {
fullDocument: 'updateLookup',
});
// Handle change events
changeStream.on('change', (change: any) => {
this.handleMongoChange(database, collection, change);
});
changeStream.on('error', (error: Error) => {
console.error(`[ChangeStream] MongoDB error for ${database}/${collection}:`, error);
});
console.log(`[ChangeStream] MongoDB watcher created for ${database}/${collection}`);
return changeStream;
} catch (error) {
console.error(`[ChangeStream] Failed to create MongoDB watcher for ${database}/${collection}:`, error);
return null;
}
}
/**
* Handle a MongoDB change event
*/
private handleMongoChange(database: string, collection: string, change: any): void {
const key = this.getMongoKey(database, collection);
const entry = this.mongoWatchers.get(key);
if (!entry) return;
// Convert MongoDB change event to our interface
const event: interfaces.IMongoChangeEvent = {
type: change.operationType as interfaces.IMongoChangeEvent['type'],
database,
collection,
documentId: change.documentKey?._id?.toString(),
document: change.fullDocument,
updateDescription: change.updateDescription,
timestamp: new Date().toISOString(),
};
// Add to activity buffer
this.addToActivityBuffer('mongodb', event);
// Push to all subscribed clients
this.pushMongoChangeToClients(key, event);
}
/**
* Push MongoDB change to subscribed clients
*/
private async pushMongoChangeToClients(
key: string,
event: interfaces.IMongoChangeEvent
): Promise<void> {
const entry = this.mongoWatchers.get(key);
if (!entry || !this.typedSocket) return;
for (const [connectionId, _sub] of entry.subscriptions) {
try {
// Find the connection and push the event
const connection = await this.typedSocket.findTargetConnection(async (conn: any) => {
return conn.alias === connectionId || conn.socketId === connectionId;
});
if (connection) {
const request = this.typedSocket.createTypedRequest<interfaces.IReq_PushMongoChange>(
'pushMongoChange',
connection
);
await request.fire({ event });
}
} catch (error) {
console.error(`[ChangeStream] Failed to push MongoDB change to ${connectionId}:`, error);
}
}
}
/**
* Close a MongoDB change stream
*/
private async closeMongoWatcher(key: string): Promise<void> {
const entry = this.mongoWatchers.get(key);
if (!entry) return;
try {
await entry.watcher.close();
this.mongoWatchers.delete(key);
console.log(`[ChangeStream] MongoDB watcher closed for ${key}`);
} catch (error) {
console.error(`[ChangeStream] Error closing MongoDB watcher for ${key}:`, error);
}
}
// ===========================================
// S3 Change Watching
// ===========================================
/**
* Subscribe a client to S3 bucket/prefix changes
*/
public async subscribeToS3(
connectionId: string,
bucket: string,
prefix?: string
): Promise<{ success: boolean; subscriptionId: string }> {
const key = this.getS3Key(bucket, prefix);
let entry = this.s3Watchers.get(key);
// Create watcher if it doesn't exist
if (!entry) {
const watcher = await this.createS3Watcher(bucket, prefix);
if (!watcher) {
return { success: false, subscriptionId: '' };
}
entry = {
watcher,
subscriptions: new Map(),
};
this.s3Watchers.set(key, entry);
}
// Add subscription
const subscriptionId = this.generateSubscriptionId();
entry.subscriptions.set(connectionId, {
connectionId,
subscriptionId,
createdAt: new Date(),
});
console.log(`[ChangeStream] S3 subscription added: ${key} for connection ${connectionId}`);
return { success: true, subscriptionId };
}
/**
* Unsubscribe a client from S3 bucket/prefix changes
*/
public async unsubscribeFromS3(
connectionId: string,
bucket: string,
prefix?: string
): Promise<boolean> {
const key = this.getS3Key(bucket, prefix);
const entry = this.s3Watchers.get(key);
if (!entry) {
return false;
}
entry.subscriptions.delete(connectionId);
console.log(`[ChangeStream] S3 subscription removed: ${key} for connection ${connectionId}`);
// Close watcher if no more subscribers
if (entry.subscriptions.size === 0) {
await this.closeS3Watcher(key);
}
return true;
}
/**
* Create an S3 bucket watcher
*/
private async createS3Watcher(
bucket: string,
prefix?: string
): Promise<plugins.smartbucket.BucketWatcher | null> {
try {
const smartbucket = await this.tsview.getSmartBucket();
if (!smartbucket) {
console.error('[ChangeStream] S3 not configured');
return null;
}
const bucketInstance = await smartbucket.getBucketByName(bucket);
// Create watcher using smartbucket's BucketWatcher
const watcher = bucketInstance.createWatcher({
prefix: prefix || '',
pollIntervalMs: 5000,
bufferTimeMs: 500,
});
// Subscribe to change events
watcher.changeSubject.subscribe((eventOrEvents: IS3ChangeEvent | IS3ChangeEvent[]) => {
const events = Array.isArray(eventOrEvents) ? eventOrEvents : [eventOrEvents];
for (const event of events) {
this.handleS3Change(bucket, prefix, event);
}
});
// Start the watcher
await watcher.start();
await watcher.readyDeferred.promise;
console.log(`[ChangeStream] S3 watcher created for ${bucket}${prefix ? '/' + prefix : ''}`);
return watcher;
} catch (error) {
console.error(`[ChangeStream] Failed to create S3 watcher for ${bucket}:`, error);
return null;
}
}
/**
* Handle an S3 change event
*/
private handleS3Change(bucket: string, prefix: string | undefined, event: IS3ChangeEvent): void {
const key = this.getS3Key(bucket, prefix);
const entry = this.s3Watchers.get(key);
if (!entry) return;
// Add to activity buffer
this.addToActivityBuffer('s3', event);
// Push to all subscribed clients
this.pushS3ChangeToClients(key, event);
}
/**
* Push S3 change to subscribed clients
*/
private async pushS3ChangeToClients(
key: string,
event: IS3ChangeEvent
): Promise<void> {
const entry = this.s3Watchers.get(key);
if (!entry || !this.typedSocket) return;
for (const [connectionId, _sub] of entry.subscriptions) {
try {
const connection = await this.typedSocket.findTargetConnection(async (conn: any) => {
return conn.alias === connectionId || conn.socketId === connectionId;
});
if (connection) {
const request = this.typedSocket.createTypedRequest<interfaces.IReq_PushS3Change>(
'pushS3Change',
connection
);
await request.fire({ event });
}
} catch (error) {
console.error(`[ChangeStream] Failed to push S3 change to ${connectionId}:`, error);
}
}
}
/**
* Close an S3 bucket watcher
*/
private async closeS3Watcher(key: string): Promise<void> {
const entry = this.s3Watchers.get(key);
if (!entry) return;
try {
await entry.watcher.stop();
this.s3Watchers.delete(key);
console.log(`[ChangeStream] S3 watcher closed for ${key}`);
} catch (error) {
console.error(`[ChangeStream] Error closing S3 watcher for ${key}:`, error);
}
}
// ===========================================
// Activity Stream
// ===========================================
/**
* Subscribe a client to the activity stream
*/
public async subscribeToActivity(connectionId: string): Promise<{ success: boolean; subscriptionId: string }> {
const subscriptionId = this.generateSubscriptionId();
this.activitySubscribers.set(connectionId, {
connectionId,
subscriptionId,
createdAt: new Date(),
});
console.log(`[ChangeStream] Activity subscription added for connection ${connectionId}`);
return { success: true, subscriptionId };
}
/**
* Unsubscribe a client from the activity stream
*/
public async unsubscribeFromActivity(connectionId: string): Promise<boolean> {
const result = this.activitySubscribers.delete(connectionId);
if (result) {
console.log(`[ChangeStream] Activity subscription removed for connection ${connectionId}`);
}
return result;
}
/**
* Get recent activity events
*/
public getRecentActivity(limit: number = 100): interfaces.IActivityEvent[] {
const count = Math.min(limit, this.activityBuffer.length);
return this.activityBuffer.slice(-count);
}
/**
* Add an event to the activity buffer
*/
private addToActivityBuffer(
source: 'mongodb' | 's3',
event: interfaces.IMongoChangeEvent | IS3ChangeEvent
): void {
const activityEvent: interfaces.IActivityEvent = {
id: `evt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
source,
event,
timestamp: new Date().toISOString(),
};
this.activityBuffer.push(activityEvent);
// Trim buffer if it exceeds max size
if (this.activityBuffer.length > this.ACTIVITY_BUFFER_SIZE) {
this.activityBuffer = this.activityBuffer.slice(-this.ACTIVITY_BUFFER_SIZE);
}
// Push to activity subscribers
this.pushActivityToClients(activityEvent);
}
/**
* Push activity event to subscribed clients
*/
private async pushActivityToClients(event: interfaces.IActivityEvent): Promise<void> {
if (!this.typedSocket || this.activitySubscribers.size === 0) return;
for (const [connectionId, _sub] of this.activitySubscribers) {
try {
const connection = await this.typedSocket.findTargetConnection(async (conn: any) => {
return conn.alias === connectionId || conn.socketId === connectionId;
});
if (connection) {
const request = this.typedSocket.createTypedRequest<interfaces.IReq_PushActivityEvent>(
'pushActivityEvent',
connection
);
await request.fire({ event });
}
} catch (error) {
console.error(`[ChangeStream] Failed to push activity to ${connectionId}:`, error);
}
}
}
// ===========================================
// Connection Management
// ===========================================
/**
* Handle client disconnect - clean up all subscriptions
*/
public async handleDisconnect(connectionId: string): Promise<void> {
console.log(`[ChangeStream] Cleaning up subscriptions for disconnected connection ${connectionId}`);
// Clean up MongoDB subscriptions
for (const [key, entry] of this.mongoWatchers) {
if (entry.subscriptions.has(connectionId)) {
entry.subscriptions.delete(connectionId);
if (entry.subscriptions.size === 0) {
await this.closeMongoWatcher(key);
}
}
}
// Clean up S3 subscriptions
for (const [key, entry] of this.s3Watchers) {
if (entry.subscriptions.has(connectionId)) {
entry.subscriptions.delete(connectionId);
if (entry.subscriptions.size === 0) {
await this.closeS3Watcher(key);
}
}
}
// Clean up activity subscription
this.activitySubscribers.delete(connectionId);
}
/**
* Stop all watchers and clean up
*/
public async stop(): Promise<void> {
console.log('[ChangeStream] Stopping all watchers...');
// Close all MongoDB watchers
for (const key of this.mongoWatchers.keys()) {
await this.closeMongoWatcher(key);
}
// Close all S3 watchers
for (const key of this.s3Watchers.keys()) {
await this.closeS3Watcher(key);
}
// Clear activity buffer and subscribers
this.activityBuffer = [];
this.activitySubscribers.clear();
console.log('[ChangeStream] All watchers stopped');
}
}

2
ts/streaming/index.ts Normal file
View File

@@ -0,0 +1,2 @@
export * from './interfaces.streaming.js';
export * from './classes.changestream-manager.js';

212
ts/streaming/interfaces.streaming.ts Normal file
View File

@@ -0,0 +1,212 @@
import type * as plugins from '../plugins.js';
// Re-export S3 change event from smartbucket
export type { IS3ChangeEvent } from '@push.rocks/smartbucket';
/**
* MongoDB change event - wraps smartdata watcher output
*/
export interface IMongoChangeEvent {
type: 'insert' | 'update' | 'delete' | 'replace' | 'drop' | 'invalidate';
database: string;
collection: string;
documentId?: string;
document?: Record<string, unknown>;
updateDescription?: {
updatedFields?: Record<string, unknown>;
removedFields?: string[];
};
timestamp: string;
}
/**
* Combined activity event for the activity stream
*/
export interface IActivityEvent {
id: string;
source: 'mongodb' | 's3';
event: IMongoChangeEvent | import('@push.rocks/smartbucket').IS3ChangeEvent;
timestamp: string;
}
// ===========================================
// TypedRequest interfaces for streaming subscriptions
// ===========================================
/**
* Subscribe to MongoDB collection changes
*/
export interface IReq_SubscribeMongo extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_SubscribeMongo
> {
method: 'subscribeMongo';
request: {
database: string;
collection: string;
};
response: {
success: boolean;
subscriptionId: string;
};
}
/**
* Unsubscribe from MongoDB collection changes
*/
export interface IReq_UnsubscribeMongo extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_UnsubscribeMongo
> {
method: 'unsubscribeMongo';
request: {
database: string;
collection: string;
};
response: {
success: boolean;
};
}
/**
* Subscribe to S3 bucket/prefix changes
*/
export interface IReq_SubscribeS3 extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_SubscribeS3
> {
method: 'subscribeS3';
request: {
bucket: string;
prefix?: string;
};
response: {
success: boolean;
subscriptionId: string;
};
}
/**
* Unsubscribe from S3 bucket/prefix changes
*/
export interface IReq_UnsubscribeS3 extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_UnsubscribeS3
> {
method: 'unsubscribeS3';
request: {
bucket: string;
prefix?: string;
};
response: {
success: boolean;
};
}
/**
* Subscribe to activity stream (all changes from MongoDB and S3)
*/
export interface IReq_SubscribeActivity extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_SubscribeActivity
> {
method: 'subscribeActivity';
request: {};
response: {
success: boolean;
subscriptionId: string;
};
}
/**
* Unsubscribe from activity stream
*/
export interface IReq_UnsubscribeActivity extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_UnsubscribeActivity
> {
method: 'unsubscribeActivity';
request: {};
response: {
success: boolean;
};
}
/**
* Get recent activity events (for initial load or reconnection)
*/
export interface IReq_GetRecentActivity extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_GetRecentActivity
> {
method: 'getRecentActivity';
request: {
limit?: number; // Default: 100
};
response: {
events: IActivityEvent[];
};
}
// ===========================================
// Server-to-client push event interfaces
// ===========================================
/**
* Server pushes MongoDB change to client
*/
export interface IReq_PushMongoChange extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_PushMongoChange
> {
method: 'pushMongoChange';
request: {
event: IMongoChangeEvent;
};
response: {
received: boolean;
};
}
/**
* Server pushes S3 change to client
*/
export interface IReq_PushS3Change extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_PushS3Change
> {
method: 'pushS3Change';
request: {
event: import('@push.rocks/smartbucket').IS3ChangeEvent;
};
response: {
received: boolean;
};
}
/**
* Server pushes activity event to client
*/
export interface IReq_PushActivityEvent extends plugins.typedrequestInterfaces.implementsTR<
plugins.typedrequestInterfaces.ITypedRequest,
IReq_PushActivityEvent
> {
method: 'pushActivityEvent';
request: {
event: IActivityEvent;
};
response: {
received: boolean;
};
}
/**
* Connection tag for tracking subscriptions
*/
export interface ISubscriptionTag extends plugins.typedrequestInterfaces.ITag {
name: 'subscription';
payload: {
type: 'mongo' | 's3' | 'activity';
key: string; // e.g., "db/collection" or "bucket/prefix" or "activity"
};
}

2
ts_web/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@git.zone/tsview',
version: '1.4.0',
version: '1.5.0',
description: 'A CLI tool for viewing S3 and MongoDB data with a web UI'
}

3
ts_web/elements/index.ts
View File

@@ -14,3 +14,6 @@ export * from './tsview-mongo-documents.js';
export * from './tsview-mongo-document.js';
export * from './tsview-mongo-indexes.js';
export * from './tsview-mongo-db-overview.js';
// Activity stream component
export * from './tsview-activity-stream.js';

561
ts_web/elements/tsview-activity-stream.ts Normal file
View File

@@ -0,0 +1,561 @@
import * as plugins from '../plugins.js';
import { changeStreamService, type IActivityEvent, type IMongoChangeEvent, type IS3ChangeEvent } from '../services/index.js';
import { themeStyles } from '../styles/index.js';
const { html, css, cssManager, customElement, property, state, DeesElement } = plugins;
type TFilterMode = 'all' | 'mongodb' | 's3';
@customElement('tsview-activity-stream')
export class TsviewActivityStream extends DeesElement {
@state()
private accessor events: IActivityEvent[] = [];
@state()
private accessor filterMode: TFilterMode = 'all';
@state()
private accessor isConnected: boolean = false;
@state()
private accessor isLoading: boolean = true;
@state()
private accessor autoScroll: boolean = true;
private subscription: plugins.smartrx.rxjs.Subscription | null = null;
private connectionSubscription: plugins.smartrx.rxjs.Subscription | null = null;
public static styles = [
cssManager.defaultStyles,
themeStyles,
css`
:host {
display: block;
height: 100%;
overflow: hidden;
}
.activity-container {
display: flex;
flex-direction: column;
height: 100%;
}
.header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 16px;
border-bottom: 1px solid #333;
}
.header-title {
font-size: 18px;
font-weight: 600;
color: #fff;
display: flex;
align-items: center;
gap: 12px;
}
.connection-status {
display: flex;
align-items: center;
gap: 6px;
font-size: 12px;
font-weight: 400;
}
.status-dot {
width: 8px;
height: 8px;
border-radius: 50%;
}
.status-dot.connected {
background: #22c55e;
box-shadow: 0 0 6px rgba(34, 197, 94, 0.5);
}
.status-dot.disconnected {
background: #ef4444;
}
.status-dot.connecting {
background: #f59e0b;
animation: pulse 1s infinite;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.5; }
}
.header-controls {
display: flex;
align-items: center;
gap: 12px;
}
.filter-tabs {
display: flex;
gap: 4px;
}
.filter-tab {
padding: 6px 12px;
background: transparent;
border: 1px solid #444;
color: #888;
border-radius: 4px;
cursor: pointer;
font-size: 12px;
transition: all 0.2s;
}
.filter-tab:hover {
border-color: #666;
color: #aaa;
}
.filter-tab.active {
background: rgba(255, 255, 255, 0.1);
border-color: #666;
color: #e0e0e0;
}
.auto-scroll-toggle {
display: flex;
align-items: center;
gap: 6px;
font-size: 12px;
color: #888;
cursor: pointer;
}
.auto-scroll-toggle input {
cursor: pointer;
}
.events-list {
flex: 1;
overflow-y: auto;
padding: 12px;
}
.event-item {
display: flex;
gap: 12px;
padding: 12px;
margin-bottom: 8px;
background: rgba(0, 0, 0, 0.2);
border-radius: 8px;
cursor: pointer;
transition: background 0.1s;
}
.event-item:hover {
background: rgba(255, 255, 255, 0.05);
}
.event-icon {
width: 36px;
height: 36px;
border-radius: 8px;
display: flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
}
.event-icon.mongodb {
background: rgba(16, 185, 129, 0.2);
color: #10b981;
}
.event-icon.s3 {
background: rgba(245, 158, 11, 0.2);
color: #f59e0b;
}
.event-icon svg {
width: 18px;
height: 18px;
}
.event-content {
flex: 1;
min-width: 0;
}
.event-header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 4px;
}
.event-title {
font-size: 13px;
font-weight: 500;
color: #e0e0e0;
}
.event-time {
font-size: 11px;
color: #666;
}
.event-details {
font-size: 12px;
color: #888;
font-family: monospace;
}
.event-type {
display: inline-block;
padding: 2px 6px;
border-radius: 4px;
font-size: 10px;
font-weight: 500;
text-transform: uppercase;
margin-right: 8px;
}
.event-type.insert, .event-type.add {
background: rgba(34, 197, 94, 0.2);
color: #4ade80;
}
.event-type.update, .event-type.modify {
background: rgba(59, 130, 246, 0.2);
color: #60a5fa;
}
.event-type.delete {
background: rgba(239, 68, 68, 0.2);
color: #f87171;
}
.event-type.replace {
background: rgba(168, 85, 247, 0.2);
color: #c084fc;
}
.event-type.drop, .event-type.invalidate {
background: rgba(239, 68, 68, 0.3);
color: #f87171;
}
.empty-state {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
height: 100%;
color: #666;
gap: 16px;
}
.empty-state svg {
width: 64px;
height: 64px;
opacity: 0.5;
}
.empty-state p {
font-size: 14px;
}
.loading-state {
display: flex;
align-items: center;
justify-content: center;
height: 100%;
color: #888;
}
.event-path {
color: #aaa;
word-break: break-all;
}
`,
];
async connectedCallback() {
super.connectedCallback();
await this.initializeStreaming();
}
disconnectedCallback() {
super.disconnectedCallback();
this.cleanup();
}
private async initializeStreaming() {
this.isLoading = true;
try {
// Connect to WebSocket if not connected
await changeStreamService.connect();
// Subscribe to connection status
this.connectionSubscription = changeStreamService.connectionStatus$.subscribe((status) => {
this.isConnected = status === 'connected';
});
// Subscribe to activity stream
await changeStreamService.subscribeToActivity();
// Load recent events
const recentEvents = await changeStreamService.getRecentActivity(100);
this.events = recentEvents;
// Subscribe to new events
this.subscription = changeStreamService.getActivityStream().subscribe((event) => {
this.events = [...this.events, event].slice(-500); // Keep last 500 events
// Auto-scroll if enabled
if (this.autoScroll) {
this.scrollToBottom();
}
});
this.isConnected = true;
} catch (error) {
console.error('Failed to initialize activity stream:', error);
this.isConnected = false;
}
this.isLoading = false;
}
private cleanup() {
if (this.subscription) {
this.subscription.unsubscribe();
this.subscription = null;
}
if (this.connectionSubscription) {
this.connectionSubscription.unsubscribe();
this.connectionSubscription = null;
}
changeStreamService.unsubscribeFromActivity();
}
private scrollToBottom() {
requestAnimationFrame(() => {
const list = this.shadowRoot?.querySelector('.events-list');
if (list) {
list.scrollTop = list.scrollHeight;
}
});
}
private setFilterMode(mode: TFilterMode) {
this.filterMode = mode;
}
private toggleAutoScroll() {
this.autoScroll = !this.autoScroll;
}
private get filteredEvents(): IActivityEvent[] {
if (this.filterMode === 'all') {
return this.events;
}
return this.events.filter((e) => e.source === this.filterMode);
}
private formatTime(timestamp: string): string {
const date = new Date(timestamp);
return date.toLocaleTimeString('en-US', {
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
});
}
private formatRelativeTime(timestamp: string): string {
const date = new Date(timestamp);
const now = new Date();
const diff = now.getTime() - date.getTime();
if (diff < 60000) {
return 'just now';
} else if (diff < 3600000) {
const mins = Math.floor(diff / 60000);
return `${mins}m ago`;
} else if (diff < 86400000) {
const hours = Math.floor(diff / 3600000);
return `${hours}h ago`;
} else {
return date.toLocaleDateString();
}
}
private getEventTitle(event: IActivityEvent): string {
if (event.source === 'mongodb') {
const mongoEvent = event.event as IMongoChangeEvent;
return `${mongoEvent.database}.${mongoEvent.collection}`;
} else {
const s3Event = event.event as IS3ChangeEvent;
return s3Event.bucket;
}
}
private getEventDetails(event: IActivityEvent): string {
if (event.source === 'mongodb') {
const mongoEvent = event.event as IMongoChangeEvent;
if (mongoEvent.documentId) {
return `Document: ${mongoEvent.documentId}`;
}
return '';
} else {
const s3Event = event.event as IS3ChangeEvent;
return s3Event.key;
}
}
private getEventType(event: IActivityEvent): string {
return event.event.type;
}
private handleEventClick(event: IActivityEvent) {
// Dispatch navigation event
if (event.source === 'mongodb') {
const mongoEvent = event.event as IMongoChangeEvent;
this.dispatchEvent(
new CustomEvent('navigate-to-mongo', {
detail: {
database: mongoEvent.database,
collection: mongoEvent.collection,
documentId: mongoEvent.documentId,
},
bubbles: true,
composed: true,
})
);
} else {
const s3Event = event.event as IS3ChangeEvent;
this.dispatchEvent(
new CustomEvent('navigate-to-s3', {
detail: {
bucket: s3Event.bucket,
key: s3Event.key,
},
bubbles: true,
composed: true,
})
);
}
}
private renderMongoIcon() {
return html`
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<ellipse cx="12" cy="5" rx="9" ry="3"></ellipse>
<path d="M21 12c0 1.66-4 3-9 3s-9-1.34-9-3"></path>
<path d="M3 5v14c0 1.66 4 3 9 3s9-1.34 9-3V5"></path>
</svg>
`;
}
private renderS3Icon() {
return html`
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M22 19a2 2 0 0 1-2 2H4a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h5l2 3h9a2 2 0 0 1 2 2z"></path>
</svg>
`;
}
private getConnectionStatusText(): string {
if (this.isConnected) {
return 'Live';
}
return 'Disconnected';
}
render() {
return html`
<div class="activity-container">
<div class="header">
<div class="header-title">
Activity Stream
<div class="connection-status">
<span class="status-dot ${this.isConnected ? 'connected' : 'disconnected'}"></span>
${this.getConnectionStatusText()}
</div>
</div>
<div class="header-controls">
<div class="filter-tabs">
<button
class="filter-tab ${this.filterMode === 'all' ? 'active' : ''}"
@click=${() => this.setFilterMode('all')}
>
All
</button>
<button
class="filter-tab ${this.filterMode === 'mongodb' ? 'active' : ''}"
@click=${() => this.setFilterMode('mongodb')}
>
MongoDB
</button>
<button
class="filter-tab ${this.filterMode === 's3' ? 'active' : ''}"
@click=${() => this.setFilterMode('s3')}
>
S3
</button>
</div>
<label class="auto-scroll-toggle">
<input
type="checkbox"
.checked=${this.autoScroll}
@change=${this.toggleAutoScroll}
/>
Auto-scroll
</label>
</div>
</div>
<div class="events-list">
${this.isLoading
? html`<div class="loading-state">Connecting to activity stream...</div>`
: this.filteredEvents.length === 0
? html`
<div class="empty-state">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M22 12h-4l-3 9L9 3l-3 9H2"></path>
</svg>
<p>No activity yet. Changes will appear here in real-time.</p>
</div>
`
: this.filteredEvents.map(
(event) => html`
<div class="event-item" @click=${() => this.handleEventClick(event)}>
<div class="event-icon ${event.source}">
${event.source === 'mongodb' ? this.renderMongoIcon() : this.renderS3Icon()}
</div>
<div class="event-content">
<div class="event-header">
<div class="event-title">
<span class="event-type ${this.getEventType(event)}">${this.getEventType(event)}</span>
${this.getEventTitle(event)}
</div>
<div class="event-time" title=${this.formatTime(event.timestamp)}>
${this.formatRelativeTime(event.timestamp)}
</div>
</div>
<div class="event-details">
<span class="event-path">${this.getEventDetails(event)}</span>
</div>
</div>
</div>
`
)}
</div>
</div>
`;
}
}

62
ts_web/elements/tsview-app.ts
View File

@@ -1,11 +1,11 @@
import * as plugins from '../plugins.js';
import { apiService } from '../services/index.js';
import { apiService, changeStreamService } from '../services/index.js';
import { themeStyles } from '../styles/index.js';
const { html, css, cssManager, customElement, state, DeesElement } = plugins;
const { DeesContextmenu } = plugins.deesCatalog;
type TViewMode = 's3' | 'mongo' | 'settings';
type TViewMode = 's3' | 'mongo' | 'activity' | 'settings';
@customElement('tsview-app')
export class TsviewApp extends DeesElement {
@@ -422,6 +422,17 @@ export class TsviewApp extends DeesElement {
async connectedCallback() {
super.connectedCallback();
await this.loadData();
// Initialize WebSocket connection for real-time updates
this.initializeChangeStream();
}
private async initializeChangeStream() {
try {
await changeStreamService.connect();
console.log('[TsviewApp] ChangeStream connected');
} catch (error) {
console.warn('[TsviewApp] Failed to connect to ChangeStream:', error);
}
}
private async loadData() {
@@ -729,6 +740,12 @@ export class TsviewApp extends DeesElement {
>
MongoDB
</button>
<button
class="nav-tab ${this.viewMode === 'activity' ? 'active' : ''}"
@click=${() => this.setViewMode('activity')}
>
Activity
</button>
<button
class="nav-tab ${this.viewMode === 'settings' ? 'active' : ''}"
@click=${() => this.setViewMode('settings')}
@@ -940,6 +957,19 @@ export class TsviewApp extends DeesElement {
`;
}
if (this.viewMode === 'activity') {
return html`
<aside class="sidebar">
<div class="sidebar-header">Activity Stream</div>
<div class="sidebar-list">
<div class="sidebar-item" style="color: #888; font-size: 12px; cursor: default;">
Real-time changes from MongoDB and S3 appear here automatically.
</div>
</div>
</aside>
`;
}
return html`
<aside class="sidebar">
<div class="sidebar-header">Settings</div>
@@ -1048,6 +1078,17 @@ export class TsviewApp extends DeesElement {
`;
}
if (this.viewMode === 'activity') {
return html`
<div class="content-area" style="padding: 0;">
<tsview-activity-stream
@navigate-to-mongo=${this.handleNavigateToMongo}
@navigate-to-s3=${this.handleNavigateToS3}
></tsview-activity-stream>
</div>
`;
}
return html`
<div class="content-area">
<h2>Settings</h2>
@@ -1055,4 +1096,21 @@ export class TsviewApp extends DeesElement {
</div>
`;
}
private handleNavigateToMongo(e: CustomEvent) {
const { database, collection, documentId } = e.detail;
this.viewMode = 'mongo';
this.selectedDatabase = database;
this.selectedCollection = collection;
// If documentId is provided, we could potentially scroll to or highlight that document
// For now, just navigate to the collection
}
private handleNavigateToS3(e: CustomEvent) {
const { bucket, key } = e.detail;
this.viewMode = 's3';
this.selectedBucket = bucket;
// The S3 browser will need to be updated to navigate to the specific key
// For now, just navigate to the bucket
}
}

117
ts_web/elements/tsview-mongo-browser.ts
View File

@@ -1,5 +1,5 @@
import * as plugins from '../plugins.js';
import { apiService, type ICollectionStats } from '../services/index.js';
import { apiService, changeStreamService, type ICollectionStats, type IMongoChangeEvent } from '../services/index.js';
import { formatSize, formatCount } from '../utilities/index.js';
import { themeStyles } from '../styles/index.js';
@@ -30,6 +30,14 @@ export class TsviewMongoBrowser extends DeesElement {
@state()
private accessor isResizingEditor: boolean = false;
@state()
private accessor recentChangeCount: number = 0;
@state()
private accessor isStreamConnected: boolean = false;
private changeSubscription: plugins.smartrx.rxjs.Subscription | null = null;
public static styles = [
cssManager.defaultStyles,
themeStyles,
@@ -148,18 +156,114 @@ export class TsviewMongoBrowser extends DeesElement {
display: none;
}
}
.change-indicator {
display: flex;
align-items: center;
gap: 6px;
padding: 4px 8px;
background: rgba(34, 197, 94, 0.2);
border-radius: 4px;
font-size: 11px;
color: #4ade80;
}
.change-indicator.pulse {
animation: pulse-green 1s ease-in-out;
}
@keyframes pulse-green {
0% { background: rgba(34, 197, 94, 0.4); }
100% { background: rgba(34, 197, 94, 0.2); }
}
.stream-status {
display: flex;
align-items: center;
gap: 4px;
font-size: 11px;
color: #888;
}
.stream-dot {
width: 6px;
height: 6px;
border-radius: 50%;
background: #888;
}
.stream-dot.connected {
background: #22c55e;
}
`,
];
async connectedCallback() {
super.connectedCallback();
await this.loadStats();
this.subscribeToChanges();
}
disconnectedCallback() {
super.disconnectedCallback();
this.unsubscribeFromChanges();
}
updated(changedProperties: Map<string, unknown>) {
if (changedProperties.has('databaseName') || changedProperties.has('collectionName')) {
this.loadStats();
this.selectedDocumentId = '';
this.recentChangeCount = 0;
// Re-subscribe to the new collection
this.unsubscribeFromChanges();
this.subscribeToChanges();
}
}
private async subscribeToChanges() {
if (!this.databaseName || !this.collectionName) return;
try {
// Subscribe to collection changes
const success = await changeStreamService.subscribeToCollection(this.databaseName, this.collectionName);
this.isStreamConnected = success;
if (success) {
// Listen for changes
this.changeSubscription = changeStreamService
.getCollectionChanges(this.databaseName, this.collectionName)
.subscribe((event) => {
this.handleChange(event);
});
}
} catch (error) {
console.warn('[MongoBrowser] Failed to subscribe to changes:', error);
this.isStreamConnected = false;
}
}
private unsubscribeFromChanges() {
if (this.changeSubscription) {
this.changeSubscription.unsubscribe();
this.changeSubscription = null;
}
if (this.databaseName && this.collectionName) {
changeStreamService.unsubscribeFromCollection(this.databaseName, this.collectionName);
}
this.isStreamConnected = false;
}
private handleChange(event: IMongoChangeEvent) {
console.log('[MongoBrowser] Received change:', event);
this.recentChangeCount++;
// Refresh stats to reflect changes
this.loadStats();
// Notify the documents component to refresh
const documentsEl = this.shadowRoot?.querySelector('tsview-mongo-documents') as any;
if (documentsEl?.refresh) {
documentsEl.refresh();
}
}
@@ -219,6 +323,17 @@ export class TsviewMongoBrowser extends DeesElement {
</div>
`
: ''}
<div class="stream-status">
<span class="stream-dot ${this.isStreamConnected ? 'connected' : ''}"></span>
${this.isStreamConnected ? 'Live' : 'Offline'}
</div>
${this.recentChangeCount > 0
? html`
<div class="change-indicator pulse">
${this.recentChangeCount} change${this.recentChangeCount > 1 ? 's' : ''}
</div>
`
: ''}
</div>
<div class="tabs">

7
ts_web/elements/tsview-mongo-documents.ts
View File

@@ -259,6 +259,13 @@ export class TsviewMongoDocuments extends DeesElement {
this.loading = false;
}
/**
* Public method to refresh documents (called by parent on change events)
*/
public async refresh() {
await this.loadDocuments();
}
private handleFilterInput(e: Event) {
this.filterText = (e.target as HTMLInputElement).value;
}

118
ts_web/elements/tsview-s3-browser.ts
View File

@@ -1,5 +1,5 @@
import * as plugins from '../plugins.js';
import { apiService } from '../services/index.js';
import { apiService, changeStreamService, type IS3ChangeEvent } from '../services/index.js';
import { themeStyles } from '../styles/index.js';
const { html, css, cssManager, customElement, property, state, DeesElement } = plugins;
@@ -29,6 +29,14 @@ export class TsviewS3Browser extends DeesElement {
@state()
private accessor isResizingPreview: boolean = false;
@state()
private accessor recentChangeCount: number = 0;
@state()
private accessor isStreamConnected: boolean = false;
private changeSubscription: plugins.smartrx.rxjs.Subscription | null = null;
public static styles = [
cssManager.defaultStyles,
themeStyles,
@@ -154,9 +162,61 @@ export class TsviewS3Browser extends DeesElement {
display: none;
}
}
.stream-status {
display: flex;
align-items: center;
gap: 4px;
font-size: 11px;
color: #888;
margin-left: auto;
margin-right: 12px;
}
.stream-dot {
width: 6px;
height: 6px;
border-radius: 50%;
background: #888;
}
.stream-dot.connected {
background: #22c55e;
}
.change-indicator {
display: flex;
align-items: center;
gap: 6px;
padding: 4px 8px;
background: rgba(245, 158, 11, 0.2);
border-radius: 4px;
font-size: 11px;
color: #f59e0b;
margin-right: 12px;
}
.change-indicator.pulse {
animation: pulse-orange 1s ease-in-out;
}
@keyframes pulse-orange {
0% { background: rgba(245, 158, 11, 0.4); }
100% { background: rgba(245, 158, 11, 0.2); }
}
`,
];
async connectedCallback() {
super.connectedCallback();
this.subscribeToChanges();
}
disconnectedCallback() {
super.disconnectedCallback();
this.unsubscribeFromChanges();
}
private setViewType(type: TViewType) {
this.viewType = type;
}
@@ -185,9 +245,54 @@ export class TsviewS3Browser extends DeesElement {
// Clear selection when bucket changes
this.selectedKey = '';
this.currentPrefix = '';
this.recentChangeCount = 0;
// Re-subscribe to the new bucket
this.unsubscribeFromChanges();
this.subscribeToChanges();
}
}
private async subscribeToChanges() {
if (!this.bucketName) return;
try {
// Subscribe to bucket changes (with optional prefix)
const success = await changeStreamService.subscribeToBucket(this.bucketName, this.currentPrefix || undefined);
this.isStreamConnected = success;
if (success) {
// Listen for changes
this.changeSubscription = changeStreamService
.getBucketChanges(this.bucketName, this.currentPrefix || undefined)
.subscribe((event) => {
this.handleChange(event);
});
}
} catch (error) {
console.warn('[S3Browser] Failed to subscribe to changes:', error);
this.isStreamConnected = false;
}
}
private unsubscribeFromChanges() {
if (this.changeSubscription) {
this.changeSubscription.unsubscribe();
this.changeSubscription = null;
}
if (this.bucketName) {
changeStreamService.unsubscribeFromBucket(this.bucketName, this.currentPrefix || undefined);
}
this.isStreamConnected = false;
}
private handleChange(event: IS3ChangeEvent) {
console.log('[S3Browser] Received change:', event);
this.recentChangeCount++;
// Trigger refresh of child components
this.refreshKey++;
}
private startPreviewResize = (e: MouseEvent) => {
e.preventDefault();
this.isResizingPreview = true;
@@ -239,6 +344,17 @@ export class TsviewS3Browser extends DeesElement {
})}
</div>
<div class="stream-status">
<span class="stream-dot ${this.isStreamConnected ? 'connected' : ''}"></span>
${this.isStreamConnected ? 'Live' : 'Offline'}
</div>
${this.recentChangeCount > 0
? html`
<div class="change-indicator pulse">
${this.recentChangeCount} change${this.recentChangeCount > 1 ? 's' : ''}
</div>
`
: ''}
<div class="view-toggle">
<button
class="view-btn ${this.viewType === 'columns' ? 'active' : ''}"

13
ts_web/elements/tsview-s3-preview.ts
View File

@@ -256,10 +256,15 @@ export class TsviewS3Preview extends DeesElement {
try {
const result = await apiService.getObject(this.bucketName, this.objectKey);
this.content = result.content;
this.contentType = result.contentType;
this.size = result.size;
this.lastModified = result.lastModified;
if (!result) {
this.error = 'Object not found';
this.loading = false;
return;
}
this.content = result.content || '';
this.contentType = result.contentType || '';
this.size = result.size || 0;
this.lastModified = result.lastModified || '';
// For text files, decode and store original content
if (this.isText()) {

7
ts_web/plugins.ts
View File

@@ -18,4 +18,9 @@ export const DeesElement = deesElement.DeesElement;
// @api.global scope
import * as typedrequest from '@api.global/typedrequest';
export { typedrequest };
import * as typedsocket from '@api.global/typedsocket';
export { typedrequest, typedsocket };
// @push.rocks scope
import * as smartrx from '@push.rocks/smartrx';
export { smartrx };

521
ts_web/services/changestream.service.ts Normal file
View File

@@ -0,0 +1,521 @@
import * as plugins from '../plugins.js';
/**
* MongoDB change event
*/
export interface IMongoChangeEvent {
type: 'insert' | 'update' | 'delete' | 'replace' | 'drop' | 'invalidate';
database: string;
collection: string;
documentId?: string;
document?: Record<string, unknown>;
updateDescription?: {
updatedFields?: Record<string, unknown>;
removedFields?: string[];
};
timestamp: string;
}
/**
* S3 change event
*/
export interface IS3ChangeEvent {
type: 'add' | 'modify' | 'delete';
key: string;
size?: number;
etag?: string;
lastModified?: Date;
bucket: string;
}
/**
* Combined activity event
*/
export interface IActivityEvent {
id: string;
source: 'mongodb' | 's3';
event: IMongoChangeEvent | IS3ChangeEvent;
timestamp: string;
}
/**
* Subscription info tracked by the service
*/
interface ISubscription {
type: 'mongo' | 's3' | 'activity';
key: string; // "db/collection" or "bucket/prefix" or "activity"
subscriptionId: string;
}
/**
* ChangeStreamService manages real-time change subscriptions from the browser.
*
* Features:
* - WebSocket connection via TypedSocket
* - Automatic reconnection with subscription restoration
* - RxJS Subjects for reactive UI updates
* - Subscription lifecycle management
*/
export class ChangeStreamService {
private typedSocket: plugins.typedsocket.TypedSocket | null = null;
private isConnected = false;
private isConnecting = false;
private subscriptions: Map<string, ISubscription> = new Map();
// RxJS Subjects for UI components
public readonly mongoChanges$ = new plugins.smartrx.rxjs.Subject<IMongoChangeEvent>();
public readonly s3Changes$ = new plugins.smartrx.rxjs.Subject<IS3ChangeEvent>();
public readonly activityEvents$ = new plugins.smartrx.rxjs.Subject<IActivityEvent>();
public readonly connectionStatus$ = new plugins.smartrx.rxjs.ReplaySubject<'connected' | 'disconnected' | 'connecting'>(1);
constructor() {
// Emit initial disconnected status
this.connectionStatus$.next('disconnected');
}
/**
* Connect to the WebSocket server
*/
public async connect(): Promise<void> {
if (this.isConnected || this.isConnecting) {
return;
}
this.isConnecting = true;
this.connectionStatus$.next('connecting');
try {
// Create client router to handle server-initiated pushes
const clientRouter = new plugins.typedrequest.TypedRouter();
// Register handlers for server push events
this.registerPushHandlers(clientRouter);
// Connect to WebSocket server using current origin
this.typedSocket = await plugins.typedsocket.TypedSocket.createClient(
clientRouter,
plugins.typedsocket.TypedSocket.useWindowLocationOriginUrl()
);
this.isConnected = true;
this.isConnecting = false;
this.connectionStatus$.next('connected');
console.log('[ChangeStream] WebSocket connected');
// Handle reconnection events via statusSubject
this.typedSocket.statusSubject.subscribe((status) => {
if (status === 'disconnected') {
this.handleDisconnect();
} else if (status === 'connected') {
this.handleReconnect();
}
});
} catch (error) {
this.isConnecting = false;
this.connectionStatus$.next('disconnected');
console.error('[ChangeStream] Failed to connect:', error);
throw error;
}
}
/**
* Disconnect from the WebSocket server
*/
public async disconnect(): Promise<void> {
if (!this.typedSocket) {
return;
}
try {
await this.typedSocket.stop();
} catch (error) {
console.error('[ChangeStream] Error during disconnect:', error);
}
this.typedSocket = null;
this.isConnected = false;
this.subscriptions.clear();
this.connectionStatus$.next('disconnected');
console.log('[ChangeStream] WebSocket disconnected');
}
/**
* Register handlers for server push events
*/
private registerPushHandlers(router: plugins.typedrequest.TypedRouter): void {
// Handle MongoDB change push
router.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>(
'pushMongoChange',
async (data: { event: IMongoChangeEvent }) => {
this.mongoChanges$.next(data.event);
return { received: true };
}
)
);
// Handle S3 change push
router.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>(
'pushS3Change',
async (data: { event: IS3ChangeEvent }) => {
this.s3Changes$.next(data.event);
return { received: true };
}
)
);
// Handle activity event push
router.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>(
'pushActivityEvent',
async (data: { event: IActivityEvent }) => {
this.activityEvents$.next(data.event);
return { received: true };
}
)
);
}
/**
* Handle WebSocket disconnection
*/
private handleDisconnect(): void {
this.isConnected = false;
this.connectionStatus$.next('disconnected');
console.log('[ChangeStream] WebSocket disconnected, waiting for reconnect...');
}
/**
* Handle WebSocket reconnection - restore all subscriptions
*/
private async handleReconnect(): Promise<void> {
this.isConnected = true;
this.connectionStatus$.next('connected');
console.log('[ChangeStream] WebSocket reconnected, restoring subscriptions...');
// Restore all subscriptions
const subscriptionsToRestore = Array.from(this.subscriptions.values());
this.subscriptions.clear();
for (const sub of subscriptionsToRestore) {
try {
if (sub.type === 'mongo') {
const [database, collection] = sub.key.split('/');
await this.subscribeToCollection(database, collection);
} else if (sub.type === 's3') {
const parts = sub.key.split('/');
const bucket = parts[0];
const prefix = parts.slice(1).join('/') || undefined;
await this.subscribeToBucket(bucket, prefix);
} else if (sub.type === 'activity') {
await this.subscribeToActivity();
}
} catch (error) {
console.error(`[ChangeStream] Failed to restore subscription ${sub.key}:`, error);
}
}
}
// ===========================================
// MongoDB Subscriptions
// ===========================================
/**
* Subscribe to changes in a MongoDB collection
*/
public async subscribeToCollection(database: string, collection: string): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
}
const key = `${database}/${collection}`;
// Check if already subscribed
if (this.subscriptions.has(`mongo:${key}`)) {
return true;
}
try {
const request = this.typedSocket.createTypedRequest<any>('subscribeMongo');
const response = await request.fire({ database, collection });
if (response.success) {
this.subscriptions.set(`mongo:${key}`, {
type: 'mongo',
key,
subscriptionId: response.subscriptionId,
});
console.log(`[ChangeStream] Subscribed to MongoDB ${key}`);
return true;
}
return false;
} catch (error) {
console.error(`[ChangeStream] Failed to subscribe to MongoDB ${key}:`, error);
return false;
}
}
/**
* Unsubscribe from changes in a MongoDB collection
*/
public async unsubscribeFromCollection(database: string, collection: string): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
return false;
}
const key = `${database}/${collection}`;
const subKey = `mongo:${key}`;
if (!this.subscriptions.has(subKey)) {
return true;
}
try {
const request = this.typedSocket.createTypedRequest<any>('unsubscribeMongo');
const response = await request.fire({ database, collection });
if (response.success) {
this.subscriptions.delete(subKey);
console.log(`[ChangeStream] Unsubscribed from MongoDB ${key}`);
}
return response.success;
} catch (error) {
console.error(`[ChangeStream] Failed to unsubscribe from MongoDB ${key}:`, error);
return false;
}
}
/**
* Check if subscribed to a MongoDB collection
*/
public isSubscribedToCollection(database: string, collection: string): boolean {
return this.subscriptions.has(`mongo:${database}/${collection}`);
}
// ===========================================
// S3 Subscriptions
// ===========================================
/**
* Subscribe to changes in an S3 bucket/prefix
*/
public async subscribeToBucket(bucket: string, prefix?: string): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
}
const key = prefix ? `${bucket}/${prefix}` : bucket;
// Check if already subscribed
if (this.subscriptions.has(`s3:${key}`)) {
return true;
}
try {
const request = this.typedSocket.createTypedRequest<any>('subscribeS3');
const response = await request.fire({ bucket, prefix });
if (response.success) {
this.subscriptions.set(`s3:${key}`, {
type: 's3',
key,
subscriptionId: response.subscriptionId,
});
console.log(`[ChangeStream] Subscribed to S3 ${key}`);
return true;
}
return false;
} catch (error) {
console.error(`[ChangeStream] Failed to subscribe to S3 ${key}:`, error);
return false;
}
}
/**
* Unsubscribe from changes in an S3 bucket/prefix
*/
public async unsubscribeFromBucket(bucket: string, prefix?: string): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
return false;
}
const key = prefix ? `${bucket}/${prefix}` : bucket;
const subKey = `s3:${key}`;
if (!this.subscriptions.has(subKey)) {
return true;
}
try {
const request = this.typedSocket.createTypedRequest<any>('unsubscribeS3');
const response = await request.fire({ bucket, prefix });
if (response.success) {
this.subscriptions.delete(subKey);
console.log(`[ChangeStream] Unsubscribed from S3 ${key}`);
}
return response.success;
} catch (error) {
console.error(`[ChangeStream] Failed to unsubscribe from S3 ${key}:`, error);
return false;
}
}
/**
* Check if subscribed to an S3 bucket/prefix
*/
public isSubscribedToBucket(bucket: string, prefix?: string): boolean {
const key = prefix ? `${bucket}/${prefix}` : bucket;
return this.subscriptions.has(`s3:${key}`);
}
// ===========================================
// Activity Stream Subscriptions
// ===========================================
/**
* Subscribe to the activity stream
*/
public async subscribeToActivity(): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
}
// Check if already subscribed
if (this.subscriptions.has('activity:activity')) {
return true;
}
try {
const request = this.typedSocket.createTypedRequest<any>('subscribeActivity');
const response = await request.fire({});
if (response.success) {
this.subscriptions.set('activity:activity', {
type: 'activity',
key: 'activity',
subscriptionId: response.subscriptionId,
});
console.log('[ChangeStream] Subscribed to activity stream');
return true;
}
return false;
} catch (error) {
console.error('[ChangeStream] Failed to subscribe to activity stream:', error);
return false;
}
}
/**
* Unsubscribe from the activity stream
*/
public async unsubscribeFromActivity(): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
return false;
}
if (!this.subscriptions.has('activity:activity')) {
return true;
}
try {
const request = this.typedSocket.createTypedRequest<any>('unsubscribeActivity');
const response = await request.fire({});
if (response.success) {
this.subscriptions.delete('activity:activity');
console.log('[ChangeStream] Unsubscribed from activity stream');
}
return response.success;
} catch (error) {
console.error('[ChangeStream] Failed to unsubscribe from activity stream:', error);
return false;
}
}
/**
* Get recent activity events
*/
public async getRecentActivity(limit: number = 100): Promise<IActivityEvent[]> {
if (!this.typedSocket || !this.isConnected) {
return [];
}
try {
const request = this.typedSocket.createTypedRequest<any>('getRecentActivity');
const response = await request.fire({ limit });
return response.events || [];
} catch (error) {
console.error('[ChangeStream] Failed to get recent activity:', error);
return [];
}
}
/**
* Check if subscribed to activity stream
*/
public isSubscribedToActivity(): boolean {
return this.subscriptions.has('activity:activity');
}
// ===========================================
// Observables for UI Components
// ===========================================
/**
* Get MongoDB changes as an Observable
*/
public getMongoChanges(): plugins.smartrx.rxjs.Observable<IMongoChangeEvent> {
return this.mongoChanges$.asObservable();
}
/**
* Get S3 changes as an Observable
*/
public getS3Changes(): plugins.smartrx.rxjs.Observable<IS3ChangeEvent> {
return this.s3Changes$.asObservable();
}
/**
* Get activity events as an Observable
*/
public getActivityStream(): plugins.smartrx.rxjs.Observable<IActivityEvent> {
return this.activityEvents$.asObservable();
}
/**
* Get filtered MongoDB changes for a specific collection
*/
public getCollectionChanges(database: string, collection: string): plugins.smartrx.rxjs.Observable<IMongoChangeEvent> {
return this.mongoChanges$.pipe(
plugins.smartrx.rxjs.ops.filter(
(event) => event.database === database && event.collection === collection
)
);
}
/**
* Get filtered S3 changes for a specific bucket/prefix
*/
public getBucketChanges(bucket: string, prefix?: string): plugins.smartrx.rxjs.Observable<IS3ChangeEvent> {
return this.s3Changes$.pipe(
plugins.smartrx.rxjs.ops.filter((event) => {
if (event.bucket !== bucket) return false;
if (prefix && !event.key.startsWith(prefix)) return false;
return true;
})
);
}
}

4
ts_web/services/index.ts
View File

@@ -1,4 +1,8 @@
export * from './api.service.js';
export * from './changestream.service.js';
import { ApiService } from './api.service.js';
import { ChangeStreamService } from './changestream.service.js';
export const apiService = new ApiService();
export const changeStreamService = new ChangeStreamService();