git.zone/tsview
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(streaming): add global activity watchers, client-side buffering, and improved real-time streaming UX

Browse Source
This commit is contained in:
Jürgen Kunz
2026-01-28 14:02:48 +00:00
parent ad8529cb0f
commit 8cc9a1850a
14 changed files with 630 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
changelog.md
View File

@@ -1,5 +1,18 @@
# Changelog
## 2026-01-28 - 1.8.0 - feat(streaming)
add global activity watchers, client-side buffering, and improved real-time streaming UX
- Introduce global MongoDB and S3 watchers in ChangeStreamManager to feed a deployment-level activity stream (start/stop automatically based on subscribers)
- ChangeStreamManager: activity buffering, globalWatchersActive flag, start/stop global watchers, and emitMongoActivityEvent API for handlers
- Mongo API handlers accept an optional ChangeStreamManager and emit activity events for DB/collection/document operations
- Server now initializes ChangeStreamManager earlier and passes it to registerMongoHandlers; request context uses localData.peer.id for WebSocket peer lookup
- Client: ChangeStreamService adds connection promise/ensureConnected, activity buffering across tabs, and more robust connect/subscribe flows (waits for connect in-flight)
- UI updates: activity stream shows relative times with live clock, buffers events from app-level subscription, auto-scroll behavior adjusted, and connection-based re-subscription
- Mongo/S3 browser components ensure RxJS listeners are attached before server-side subscribe and surface connection status
- S3 browser: add drag-and-drop folder upload support (webkitdirectory), recursive folder entry traversal, upload with preserved relative paths, and column refresh/refreshAll logic
- Minor API/behavioral changes use connection peer id via conn.peer?.id when resolving target connections
## 2026-01-27 - 1.7.0 - feat(s3)
add drag-and-drop and context-menu file uploads, inline text editing in preview, and increase preview width

22
readme.hints.md
View File

@@ -106,6 +106,28 @@ IReq_PushS3Change: { event: IS3ChangeEvent } -> { received }
IReq_PushActivityEvent: { event: IActivityEvent } -> { received }
```
### WebSocket Context Pattern
When a TypedHandler receives a WebSocket request, the transport context is available via the second argument (`TypedTools` instance). SmartServe attaches the WebSocket peer to `localData.peer`:
```typescript
// In a TypedHandler callback:
async (reqData, context) => {
// context is a TypedTools instance
const peerId = context.localData?.peer?.id; // unique WebSocket connection ID
}
```
To push events back to a specific client, use `findTargetConnection`:
```typescript
const conn = await typedSocket.findTargetConnection(async (c: any) => {
return c.peer?.id === connectionId;
});
const request = typedSocket.createTypedRequest<IReq_Push>('pushEvent', conn);
await request.fire({ event });
```
### Dependencies Added
- `@api.global/typedsocket` - WebSocket client/server

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@git.zone/tsview',
version: '1.7.0',
version: '1.8.0',
description: 'A CLI tool for viewing S3 and MongoDB data with a web UI'
}

66
ts/api/handlers.mongodb.ts
View File

@@ -1,13 +1,15 @@
import * as plugins from '../plugins.js';
import type * as interfaces from '../interfaces/index.js';
import type { TsView } from '../tsview.classes.tsview.js';
import type { ChangeStreamManager } from '../streaming/index.js';
/**
* Register MongoDB API handlers
*/
export async function registerMongoHandlers(
typedrouter: plugins.typedrequest.TypedRouter,
tsview: TsView
tsview: TsView,
changeStreamManager?: ChangeStreamManager
): Promise<void> {
// Helper to get the native MongoDB client
const getMongoClient = async () => {
@@ -122,6 +124,14 @@ export async function registerMongoHandlers(
const db = client.db(reqData.databaseName);
// Create a placeholder collection to materialize the database
await db.createCollection('_tsview_init');
if (changeStreamManager) {
changeStreamManager.emitMongoActivityEvent({
type: 'insert',
database: reqData.databaseName,
collection: '_tsview_init',
timestamp: new Date().toISOString(),
});
}
return { success: true };
} catch (err) {
console.error('Error creating database:', err);
@@ -140,6 +150,14 @@ export async function registerMongoHandlers(
const client = await getMongoClient();
const db = client.db(reqData.databaseName);
await db.dropDatabase();
if (changeStreamManager) {
changeStreamManager.emitMongoActivityEvent({
type: 'drop',
database: reqData.databaseName,
collection: '*',
timestamp: new Date().toISOString(),
});
}
return { success: true };
} catch (err) {
console.error('Error dropping database:', err);
@@ -158,6 +176,14 @@ export async function registerMongoHandlers(
const client = await getMongoClient();
const db = client.db(reqData.databaseName);
await db.createCollection(reqData.collectionName);
if (changeStreamManager) {
changeStreamManager.emitMongoActivityEvent({
type: 'insert',
database: reqData.databaseName,
collection: reqData.collectionName,
timestamp: new Date().toISOString(),
});
}
return { success: true };
} catch (err) {
console.error('Error creating collection:', err);
@@ -176,6 +202,14 @@ export async function registerMongoHandlers(
const client = await getMongoClient();
const db = client.db(reqData.databaseName);
await db.dropCollection(reqData.collectionName);
if (changeStreamManager) {
changeStreamManager.emitMongoActivityEvent({
type: 'drop',
database: reqData.databaseName,
collection: reqData.collectionName,
timestamp: new Date().toISOString(),
});
}
return { success: true };
} catch (err) {
console.error('Error dropping collection:', err);
@@ -267,6 +301,16 @@ export async function registerMongoHandlers(
const result = await collection.insertOne(reqData.document);
if (changeStreamManager) {
changeStreamManager.emitMongoActivityEvent({
type: 'insert',
database: reqData.databaseName,
collection: reqData.collectionName,
documentId: result.insertedId.toString(),
timestamp: new Date().toISOString(),
});
}
return { insertedId: result.insertedId.toString() };
} catch (err) {
console.error('Error inserting document:', err);
@@ -294,6 +338,16 @@ export async function registerMongoHandlers(
const result = await collection.updateOne(filter, updateDoc);
if (changeStreamManager && (result.modifiedCount > 0 || result.matchedCount > 0)) {
changeStreamManager.emitMongoActivityEvent({
type: 'update',
database: reqData.databaseName,
collection: reqData.collectionName,
documentId: reqData.documentId,
timestamp: new Date().toISOString(),
});
}
return {
success: result.modifiedCount > 0 || result.matchedCount > 0,
modifiedCount: result.modifiedCount,
@@ -319,6 +373,16 @@ export async function registerMongoHandlers(
const filter = createIdFilter(reqData.documentId);
const result = await collection.deleteOne(filter);
if (changeStreamManager && result.deletedCount > 0) {
changeStreamManager.emitMongoActivityEvent({
type: 'delete',
database: reqData.databaseName,
collection: reqData.collectionName,
documentId: reqData.documentId,
timestamp: new Date().toISOString(),
});
}
return {
success: result.deletedCount > 0,
deletedCount: result.deletedCount,

2
ts/bundled_ui.ts
View File

File diff suppressed because one or more lines are too long

24
ts/server/classes.viewserver.ts
View File

@@ -35,18 +35,18 @@ export class ViewServer {
noCache: true,
});
// Initialize ChangeStreamManager for real-time updates (before handlers so they can emit events)
this.changeStreamManager = new ChangeStreamManager(this.tsview);
// Register API handlers directly to server's router
if (this.tsview.config.hasS3()) {
await registerS3Handlers(this.typedServer.typedrouter, this.tsview);
}
if (this.tsview.config.hasMongo()) {
await registerMongoHandlers(this.typedServer.typedrouter, this.tsview);
await registerMongoHandlers(this.typedServer.typedrouter, this.tsview, this.changeStreamManager);
}
// Initialize ChangeStreamManager for real-time updates
this.changeStreamManager = new ChangeStreamManager(this.tsview);
// Register streaming handlers
await this.registerStreamingHandlers();
@@ -192,18 +192,16 @@ export class ViewServer {
}
/**
* Extract connection ID from request context
* Extract connection ID from request context.
* SmartServe attaches the WebSocket peer to `localData.peer` on each request.
*/
private getConnectionId(context: any): string | null {
// Try to get connection ID from WebSocket context
if (context?.socketConnection?.socketId) {
return context.socketConnection.socketId;
// The TypedTools instance carries localData from the transport layer.
// SmartServe puts the IWebSocketPeer at localData.peer.
if (context?.localData?.peer?.id) {
return context.localData.peer.id;
}
if (context?.socketConnection?.alias) {
return context.socketConnection.alias;
}
// Fallback: generate a unique ID for HTTP requests
// Note: Real-time streaming requires WebSocket connection
// HTTP requests don't have a peer — real-time streaming requires WebSocket.
return null;
}

199
ts/streaming/classes.changestream-manager.ts
View File

@@ -55,6 +55,11 @@ export class ChangeStreamManager {
private activityBuffer: interfaces.IActivityEvent[] = [];
private readonly ACTIVITY_BUFFER_SIZE = 1000;
// Global watchers for the activity stream (started lazily on first subscriber)
private globalMongoWatcher: plugins.mongodb.ChangeStream | null = null;
private globalS3Watchers: Map<string, plugins.smartbucket.BucketWatcher> = new Map();
private globalWatchersActive: boolean = false;
// Counter for generating unique subscription IDs
private subscriptionCounter = 0;
@@ -218,8 +223,11 @@ export class ChangeStreamManager {
timestamp: new Date().toISOString(),
};
// Add to activity buffer
this.addToActivityBuffer('mongodb', event);
// Only add to activity buffer if global watchers are NOT active.
// When active, the global MongoDB watcher already feeds the activity stream.
if (!this.globalWatchersActive) {
this.addToActivityBuffer('mongodb', event);
}
// Push to all subscribed clients
this.pushMongoChangeToClients(key, event);
@@ -239,7 +247,7 @@ export class ChangeStreamManager {
try {
// Find the connection and push the event
const connection = await this.typedSocket.findTargetConnection(async (conn: any) => {
return conn.alias === connectionId || conn.socketId === connectionId;
return conn.peer?.id === connectionId;
});
if (connection) {
@@ -391,8 +399,11 @@ export class ChangeStreamManager {
if (!entry) return;
// Add to activity buffer
this.addToActivityBuffer('s3', event);
// Only add to activity buffer if global watchers are NOT active.
// When active, the global S3 watchers already feed the activity stream.
if (!this.globalWatchersActive) {
this.addToActivityBuffer('s3', event);
}
// Push to all subscribed clients
this.pushS3ChangeToClients(key, event);
@@ -411,7 +422,7 @@ export class ChangeStreamManager {
for (const [connectionId, _sub] of entry.subscriptions) {
try {
const connection = await this.typedSocket.findTargetConnection(async (conn: any) => {
return conn.alias === connectionId || conn.socketId === connectionId;
return conn.peer?.id === connectionId;
});
if (connection) {
@@ -460,6 +471,12 @@ export class ChangeStreamManager {
});
console.log(`[ChangeStream] Activity subscription added for connection ${connectionId}`);
// Start global watchers when the first activity subscriber connects
if (this.activitySubscribers.size === 1) {
await this.startGlobalWatchers();
}
return { success: true, subscriptionId };
}
@@ -470,6 +487,11 @@ export class ChangeStreamManager {
const result = this.activitySubscribers.delete(connectionId);
if (result) {
console.log(`[ChangeStream] Activity subscription removed for connection ${connectionId}`);
// Stop global watchers when no activity subscribers remain
if (this.activitySubscribers.size === 0) {
await this.stopGlobalWatchers();
}
}
return result;
}
@@ -482,6 +504,13 @@ export class ChangeStreamManager {
return this.activityBuffer.slice(-count);
}
/**
* Emit a MongoDB activity event from an API handler (no change stream required).
*/
public emitMongoActivityEvent(event: interfaces.IMongoChangeEvent): void {
this.addToActivityBuffer('mongodb', event);
}
/**
* Add an event to the activity buffer
*/
@@ -516,7 +545,7 @@ export class ChangeStreamManager {
for (const [connectionId, _sub] of this.activitySubscribers) {
try {
const connection = await this.typedSocket.findTargetConnection(async (conn: any) => {
return conn.alias === connectionId || conn.socketId === connectionId;
return conn.peer?.id === connectionId;
});
if (connection) {
@@ -532,6 +561,154 @@ export class ChangeStreamManager {
}
}
// ===========================================
// Global Watchers for Activity Stream
// ===========================================
/**
* Start global watchers when the first activity subscriber connects.
* These watch all MongoDB and S3 activity and feed into the activity buffer.
*/
private async startGlobalWatchers(): Promise<void> {
if (this.globalWatchersActive) return;
this.globalWatchersActive = true;
console.log('[ChangeStream] Starting global watchers for activity stream...');
await Promise.all([
this.startGlobalMongoWatcher(),
this.startGlobalS3Watchers(),
]);
}
/**
* Start a deployment-level MongoDB change stream that watches ALL databases/collections.
*/
private async startGlobalMongoWatcher(): Promise<void> {
try {
const db = await this.tsview.getMongoDb();
if (!db) {
console.log('[ChangeStream] MongoDB not configured, skipping global MongoDB watcher');
return;
}
const client = (db as any).mongoDbClient as plugins.mongodb.MongoClient;
// Deployment-level watch — one stream for everything
const changeStream = client.watch([], {
fullDocument: 'updateLookup',
});
changeStream.on('change', (change: any) => {
const database = change.ns?.db || 'unknown';
const collection = change.ns?.coll || 'unknown';
const event: interfaces.IMongoChangeEvent = {
type: change.operationType as interfaces.IMongoChangeEvent['type'],
database,
collection,
documentId: change.documentKey?._id?.toString(),
document: change.fullDocument,
updateDescription: change.updateDescription,
timestamp: new Date().toISOString(),
};
this.addToActivityBuffer('mongodb', event);
});
changeStream.on('error', (error: Error) => {
console.error('[ChangeStream] Global MongoDB watcher error:', error);
});
this.globalMongoWatcher = changeStream;
console.log('[ChangeStream] Global MongoDB watcher started');
} catch (error) {
console.warn('[ChangeStream] MongoDB change streams unavailable (requires replica set). MongoDB activity events will come from API operations only.');
}
}
/**
* Start S3 bucket watchers — one BucketWatcher per bucket.
*/
private async startGlobalS3Watchers(): Promise<void> {
try {
const smartbucket = await this.tsview.getSmartBucket();
if (!smartbucket) {
console.log('[ChangeStream] S3 not configured, skipping global S3 watchers');
return;
}
// List all buckets
const command = new plugins.s3.ListBucketsCommand({});
const response = await smartbucket.s3Client.send(command) as plugins.s3.ListBucketsCommandOutput;
const bucketNames = response.Buckets?.map(b => b.Name).filter((name): name is string => !!name) || [];
for (const bucketName of bucketNames) {
try {
const bucketInstance = await smartbucket.getBucketByName(bucketName);
const watcher = bucketInstance.createWatcher({
prefix: '',
pollIntervalMs: 5000,
bufferTimeMs: 500,
});
watcher.changeSubject.subscribe((eventOrEvents: IS3ChangeEvent | IS3ChangeEvent[]) => {
const events = Array.isArray(eventOrEvents) ? eventOrEvents : [eventOrEvents];
for (const event of events) {
this.addToActivityBuffer('s3', event);
}
});
await watcher.start();
await watcher.readyDeferred.promise;
this.globalS3Watchers.set(bucketName, watcher);
console.log(`[ChangeStream] Global S3 watcher started for bucket: ${bucketName}`);
} catch (bucketError) {
console.error(`[ChangeStream] Failed to start global S3 watcher for bucket ${bucketName}:`, bucketError);
}
}
console.log(`[ChangeStream] Global S3 watchers started (${this.globalS3Watchers.size}/${bucketNames.length} buckets)`);
} catch (error) {
console.error('[ChangeStream] Failed to start global S3 watchers:', error);
}
}
/**
* Stop all global watchers when no activity subscribers remain.
*/
private async stopGlobalWatchers(): Promise<void> {
if (!this.globalWatchersActive) return;
console.log('[ChangeStream] Stopping global watchers...');
// Close global MongoDB watcher
if (this.globalMongoWatcher) {
try {
await this.globalMongoWatcher.close();
console.log('[ChangeStream] Global MongoDB watcher stopped');
} catch (error) {
console.error('[ChangeStream] Error closing global MongoDB watcher:', error);
}
this.globalMongoWatcher = null;
}
// Close all global S3 watchers
for (const [bucketName, watcher] of this.globalS3Watchers) {
try {
await watcher.stop();
console.log(`[ChangeStream] Global S3 watcher stopped for bucket: ${bucketName}`);
} catch (error) {
console.error(`[ChangeStream] Error closing global S3 watcher for ${bucketName}:`, error);
}
}
this.globalS3Watchers.clear();
this.globalWatchersActive = false;
console.log('[ChangeStream] Global watchers stopped');
}
// ===========================================
// Connection Management
// ===========================================
@@ -564,6 +741,11 @@ export class ChangeStreamManager {
// Clean up activity subscription
this.activitySubscribers.delete(connectionId);
// Stop global watchers if no activity subscribers remain
if (this.activitySubscribers.size === 0) {
await this.stopGlobalWatchers();
}
}
/**
@@ -572,6 +754,9 @@ export class ChangeStreamManager {
public async stop(): Promise<void> {
console.log('[ChangeStream] Stopping all watchers...');
// Stop global watchers first
await this.stopGlobalWatchers();
// Close all MongoDB watchers
for (const key of this.mongoWatchers.keys()) {
await this.closeMongoWatcher(key);

2
ts_web/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@git.zone/tsview',
version: '1.7.0',
version: '1.8.0',
description: 'A CLI tool for viewing S3 and MongoDB data with a web UI'
}

117
ts_web/elements/tsview-activity-stream.ts
View File

@@ -23,8 +23,12 @@ export class TsviewActivityStream extends DeesElement {
@state()
private accessor autoScroll: boolean = true;
@state()
private accessor now: number = Date.now();
private subscription: plugins.smartrx.rxjs.Subscription | null = null;
private connectionSubscription: plugins.smartrx.rxjs.Subscription | null = null;
private nowInterval: ReturnType<typeof setInterval> | null = null;
public static styles = [
cssManager.defaultStyles,
@@ -159,6 +163,15 @@ export class TsviewActivityStream extends DeesElement {
background: rgba(255, 255, 255, 0.05);
}
.event-time-col {
width: 70px;
flex-shrink: 0;
text-align: right;
font-size: 11px;
color: #666;
padding-top: 2px;
}
.event-icon {
width: 36px;
height: 36px;
@@ -192,7 +205,6 @@ export class TsviewActivityStream extends DeesElement {
.event-header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 4px;
}
@@ -202,11 +214,6 @@ export class TsviewActivityStream extends DeesElement {
color: #e0e0e0;
}
.event-time {
font-size: 11px;
color: #666;
}
.event-details {
font-size: 12px;
color: #888;
@@ -285,50 +292,66 @@ export class TsviewActivityStream extends DeesElement {
async connectedCallback() {
super.connectedCallback();
this.nowInterval = setInterval(() => {
this.now = Date.now();
}, 1000);
await this.initializeStreaming();
}
disconnectedCallback() {
super.disconnectedCallback();
if (this.nowInterval) {
clearInterval(this.nowInterval);
this.nowInterval = null;
}
this.cleanup();
}
private async initializeStreaming() {
this.isLoading = true;
// Subscribe to connection status and trigger re-subscription on reconnect
this.connectionSubscription = changeStreamService.connectionStatus$.subscribe(async (status) => {
const wasConnected = this.isConnected;
this.isConnected = status === 'connected';
if (status === 'connected' && !wasConnected) {
await this.setupSubscriptions();
this.isLoading = false;
}
});
try {
// Connect to WebSocket if not connected
await changeStreamService.connect();
// Subscribe to connection status
this.connectionSubscription = changeStreamService.connectionStatus$.subscribe((status) => {
this.isConnected = status === 'connected';
});
// Subscribe to activity stream
await changeStreamService.subscribeToActivity();
// Load recent events
const recentEvents = await changeStreamService.getRecentActivity(100);
this.events = recentEvents;
// Subscribe to new events
this.subscription = changeStreamService.getActivityStream().subscribe((event) => {
this.events = [...this.events, event].slice(-500); // Keep last 500 events
// Auto-scroll if enabled
if (this.autoScroll) {
this.scrollToBottom();
}
});
this.isConnected = true;
// setupSubscriptions() is triggered by connectionStatus$ subscriber above
} catch (error) {
console.error('Failed to initialize activity stream:', error);
this.isConnected = false;
this.isLoading = false;
}
}
private async setupSubscriptions() {
// Read buffered events (captured while on other tabs by app-level subscription)
const buffered = changeStreamService.getBufferedActivity();
if (buffered.length > 0) {
this.events = buffered;
} else {
// Buffer empty (fresh start) — fetch from server
const recentEvents = await changeStreamService.getRecentActivity(100);
if (recentEvents.length > 0) {
this.events = recentEvents;
}
}
this.isLoading = false;
// Set up RxJS listener only once for new live events
if (!this.subscription) {
this.subscription = changeStreamService.getActivityStream().subscribe((event) => {
this.events = [...this.events, event].slice(-500);
if (this.autoScroll) {
this.scrollToTop();
}
});
}
}
private cleanup() {
@@ -340,14 +363,15 @@ export class TsviewActivityStream extends DeesElement {
this.connectionSubscription.unsubscribe();
this.connectionSubscription = null;
}
changeStreamService.unsubscribeFromActivity();
// DO NOT call changeStreamService.unsubscribeFromActivity()
// The app-level subscription keeps the server sending events for buffering
}
private scrollToBottom() {
private scrollToTop() {
requestAnimationFrame(() => {
const list = this.shadowRoot?.querySelector('.events-list');
if (list) {
list.scrollTop = list.scrollHeight;
list.scrollTop = 0;
}
});
}
@@ -361,10 +385,11 @@ export class TsviewActivityStream extends DeesElement {
}
private get filteredEvents(): IActivityEvent[] {
if (this.filterMode === 'all') {
return this.events;
let events = this.events;
if (this.filterMode !== 'all') {
events = events.filter((e) => e.source === this.filterMode);
}
return this.events.filter((e) => e.source === this.filterMode);
return events.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
}
private formatTime(timestamp: string): string {
@@ -378,11 +403,13 @@ export class TsviewActivityStream extends DeesElement {
private formatRelativeTime(timestamp: string): string {
const date = new Date(timestamp);
const now = new Date();
const diff = now.getTime() - date.getTime();
const diff = this.now - date.getTime();
if (diff < 60000) {
return 'just now';
if (diff < 1000) {
return 'now';
} else if (diff < 60000) {
const secs = Math.floor(diff / 1000);
return `${secs}s ago`;
} else if (diff < 3600000) {
const mins = Math.floor(diff / 60000);
return `${mins}m ago`;
@@ -534,6 +561,9 @@ export class TsviewActivityStream extends DeesElement {
: this.filteredEvents.map(
(event) => html`
<div class="event-item" @click=${() => this.handleEventClick(event)}>
<div class="event-time-col" title=${this.formatTime(event.timestamp)}>
${this.formatRelativeTime(event.timestamp)}
</div>
<div class="event-icon ${event.source}">
${event.source === 'mongodb' ? this.renderMongoIcon() : this.renderS3Icon()}
</div>
@@ -543,9 +573,6 @@ export class TsviewActivityStream extends DeesElement {
<span class="event-type ${this.getEventType(event)}">${this.getEventType(event)}</span>
${this.getEventTitle(event)}
</div>
<div class="event-time" title=${this.formatTime(event.timestamp)}>
${this.formatRelativeTime(event.timestamp)}
</div>
</div>
<div class="event-details">
<span class="event-path">${this.getEventDetails(event)}</span>

7
ts_web/elements/tsview-app.ts
View File

@@ -421,15 +421,18 @@ export class TsviewApp extends DeesElement {
async connectedCallback() {
super.connectedCallback();
await this.loadData();
// Initialize WebSocket connection for real-time updates
// Start WebSocket connection first (non-blocking) so it's in-flight
// before child components try to subscribe
this.initializeChangeStream();
await this.loadData();
}
private async initializeChangeStream() {
try {
await changeStreamService.connect();
console.log('[TsviewApp] ChangeStream connected');
// Subscribe to activity globally so events are buffered regardless of active tab
await changeStreamService.subscribeToActivity();
} catch (error) {
console.warn('[TsviewApp] Failed to connect to ChangeStream:', error);
}

27
ts_web/elements/tsview-mongo-browser.ts
View File

@@ -37,6 +37,7 @@ export class TsviewMongoBrowser extends DeesElement {
private accessor isStreamConnected: boolean = false;
private changeSubscription: plugins.smartrx.rxjs.Subscription | null = null;
private connectionSubscription: plugins.smartrx.rxjs.Subscription | null = null;
public static styles = [
cssManager.defaultStyles,
@@ -201,12 +202,20 @@ export class TsviewMongoBrowser extends DeesElement {
async connectedCallback() {
super.connectedCallback();
await this.loadStats();
this.subscribeToChanges();
// Subscription is handled by updated() when databaseName/collectionName are set.
// Only track connection status for UI indicator here.
this.connectionSubscription = changeStreamService.connectionStatus$.subscribe((status) => {
this.isStreamConnected = status === 'connected';
});
}
disconnectedCallback() {
super.disconnectedCallback();
this.unsubscribeFromChanges();
if (this.connectionSubscription) {
this.connectionSubscription.unsubscribe();
this.connectionSubscription = null;
}
}
updated(changedProperties: Map<string, unknown>) {
@@ -224,18 +233,16 @@ export class TsviewMongoBrowser extends DeesElement {
if (!this.databaseName || !this.collectionName) return;
try {
// Subscribe to collection changes
const success = await changeStreamService.subscribeToCollection(this.databaseName, this.collectionName);
this.isStreamConnected = success;
if (success) {
// Listen for changes
// Set up RxJS listener first so events aren't missed on reconnect
if (!this.changeSubscription) {
this.changeSubscription = changeStreamService
.getCollectionChanges(this.databaseName, this.collectionName)
.subscribe((event) => {
this.handleChange(event);
});
.subscribe((event) => this.handleChange(event));
}
// Subscribe on the server side (will auto-connect if needed)
const success = await changeStreamService.subscribeToCollection(this.databaseName, this.collectionName);
this.isStreamConnected = success;
} catch (error) {
console.warn('[MongoBrowser] Failed to subscribe to changes:', error);
this.isStreamConnected = false;

27
ts_web/elements/tsview-s3-browser.ts
View File

@@ -36,6 +36,7 @@ export class TsviewS3Browser extends DeesElement {
private accessor isStreamConnected: boolean = false;
private changeSubscription: plugins.smartrx.rxjs.Subscription | null = null;
private connectionSubscription: plugins.smartrx.rxjs.Subscription | null = null;
public static styles = [
cssManager.defaultStyles,
@@ -209,12 +210,20 @@ export class TsviewS3Browser extends DeesElement {
async connectedCallback() {
super.connectedCallback();
this.subscribeToChanges();
// Subscription is handled by updated() when bucketName is set.
// Only track connection status for UI indicator here.
this.connectionSubscription = changeStreamService.connectionStatus$.subscribe((status) => {
this.isStreamConnected = status === 'connected';
});
}
disconnectedCallback() {
super.disconnectedCallback();
this.unsubscribeFromChanges();
if (this.connectionSubscription) {
this.connectionSubscription.unsubscribe();
this.connectionSubscription = null;
}
}
private setViewType(type: TViewType) {
@@ -256,18 +265,16 @@ export class TsviewS3Browser extends DeesElement {
if (!this.bucketName) return;
try {
// Subscribe to bucket changes (with optional prefix)
const success = await changeStreamService.subscribeToBucket(this.bucketName, this.currentPrefix || undefined);
this.isStreamConnected = success;
if (success) {
// Listen for changes
// Set up RxJS listener first so events aren't missed on reconnect
if (!this.changeSubscription) {
this.changeSubscription = changeStreamService
.getBucketChanges(this.bucketName, this.currentPrefix || undefined)
.subscribe((event) => {
this.handleChange(event);
});
.subscribe((event) => this.handleChange(event));
}
// Subscribe on the server side (will auto-connect if needed)
const success = await changeStreamService.subscribeToBucket(this.bucketName, this.currentPrefix || undefined);
this.isStreamConnected = success;
} catch (error) {
console.warn('[S3Browser] Failed to subscribe to changes:', error);
this.isStreamConnected = false;

132
ts_web/elements/tsview-s3-columns.ts
View File

@@ -6,6 +6,25 @@ import { themeStyles } from '../styles/index.js';
const { html, css, cssManager, customElement, property, state, DeesElement } = plugins;
const { DeesContextmenu } = plugins.deesCatalog;
// FileSystem API types for drag-and-drop folder support
interface FileSystemEntry {
isFile: boolean;
isDirectory: boolean;
name: string;
}
interface FileSystemFileEntry extends FileSystemEntry {
file(successCallback: (file: File) => void, errorCallback?: (error: Error) => void): void;
}
interface FileSystemDirectoryEntry extends FileSystemEntry {
createReader(): FileSystemDirectoryReader;
}
interface FileSystemDirectoryReader {
readEntries(
successCallback: (entries: FileSystemEntry[]) => void,
errorCallback?: (error: Error) => void
): void;
}
interface IColumn {
prefix: string;
objects: IS3Object[];
@@ -364,10 +383,10 @@ export class TsviewS3Columns extends DeesElement {
}
updated(changedProperties: Map<string, unknown>) {
// Only reset columns when bucket changes or refresh is triggered
// Internal folder navigation is handled by selectFolder() which appends columns
if (changedProperties.has('bucketName') || changedProperties.has('refreshKey')) {
if (changedProperties.has('bucketName')) {
this.loadInitialColumn();
} else if (changedProperties.has('refreshKey')) {
this.refreshAllColumns();
}
}
@@ -391,6 +410,20 @@ export class TsviewS3Columns extends DeesElement {
this.loading = false;
}
private async refreshAllColumns() {
const updatedColumns = await Promise.all(
this.columns.map(async (col) => {
try {
const result = await apiService.listObjects(this.bucketName, col.prefix, '/');
return { ...col, objects: result.objects, prefixes: result.prefixes };
} catch {
return col;
}
})
);
this.columns = updatedColumns;
}
private async selectFolder(columnIndex: number, prefix: string) {
// Update selection in current column
this.columns = this.columns.map((col, i) => {
@@ -533,7 +566,7 @@ export class TsviewS3Columns extends DeesElement {
action: async () => this.openCreateDialog('file', prefix),
},
{
name: 'Upload Files',
name: 'Upload...',
iconName: 'lucide:upload',
action: async () => this.triggerFileUpload(prefix),
},
@@ -545,7 +578,12 @@ export class TsviewS3Columns extends DeesElement {
if (confirm(`Delete folder "${getFileName(prefix)}" and all its contents?`)) {
const success = await apiService.deletePrefix(this.bucketName, prefix);
if (success) {
await this.loadInitialColumn();
// Remove columns that were inside the deleted folder
this.columns = this.columns.slice(0, columnIndex + 1);
this.columns = this.columns.map((col, i) =>
i === columnIndex ? { ...col, selectedItem: null } : col
);
await this.refreshColumnByPrefix(this.columns[columnIndex].prefix);
}
}
},
@@ -589,7 +627,7 @@ export class TsviewS3Columns extends DeesElement {
if (confirm(`Delete file "${getFileName(key)}"?`)) {
const success = await apiService.deleteObject(this.bucketName, key);
if (success) {
await this.loadInitialColumn();
await this.refreshColumnByPrefix(this.columns[columnIndex].prefix);
}
}
},
@@ -616,7 +654,7 @@ export class TsviewS3Columns extends DeesElement {
action: async () => this.openCreateDialog('file', prefix),
},
{
name: 'Upload Files',
name: 'Upload...',
iconName: 'lucide:upload',
action: async () => this.triggerFileUpload(prefix),
},
@@ -664,6 +702,7 @@ export class TsviewS3Columns extends DeesElement {
this.fileInputElement = document.createElement('input');
this.fileInputElement.type = 'file';
this.fileInputElement.multiple = true;
(this.fileInputElement as any).webkitdirectory = true; // Enable folder selection
this.fileInputElement.style.display = 'none';
this.shadowRoot!.appendChild(this.fileInputElement);
}
@@ -692,7 +731,9 @@ export class TsviewS3Columns extends DeesElement {
this.uploadProgress = { current: i + 1, total: files.length };
try {
const base64 = await this.readFileAsBase64(file);
const key = targetPrefix + file.name;
// Use webkitRelativePath if available (folder upload), otherwise just filename
const relativePath = (file as any).webkitRelativePath || file.name;
const key = targetPrefix + relativePath;
const contentType = file.type || this.getContentType(file.name.split('.').pop()?.toLowerCase() || '');
await apiService.putObject(this.bucketName, key, base64, contentType);
} catch (err) {
@@ -705,6 +746,31 @@ export class TsviewS3Columns extends DeesElement {
await this.refreshColumnByPrefix(targetPrefix);
}
private async uploadFilesWithPaths(
files: { file: File; relativePath: string }[],
targetPrefix: string
) {
this.uploading = true;
this.uploadProgress = { current: 0, total: files.length };
for (let i = 0; i < files.length; i++) {
const { file, relativePath } = files[i];
this.uploadProgress = { current: i + 1, total: files.length };
try {
const base64 = await this.readFileAsBase64(file);
const key = targetPrefix + relativePath;
const contentType = file.type || this.getContentType(file.name.split('.').pop()?.toLowerCase() || '');
await apiService.putObject(this.bucketName, key, base64, contentType);
} catch (err) {
console.error(`Failed to upload ${relativePath}:`, err);
}
}
this.uploading = false;
this.uploadProgress = null;
await this.refreshColumnByPrefix(targetPrefix);
}
private readFileAsBase64(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
@@ -720,7 +786,7 @@ export class TsviewS3Columns extends DeesElement {
private async refreshColumnByPrefix(prefix: string) {
const columnIndex = this.columns.findIndex(col => col.prefix === prefix);
if (columnIndex === -1) {
await this.loadInitialColumn();
await this.refreshAllColumns();
return;
}
try {
@@ -729,7 +795,7 @@ export class TsviewS3Columns extends DeesElement {
i === columnIndex ? { ...col, objects: result.objects, prefixes: result.prefixes } : col
);
} catch {
await this.loadInitialColumn();
await this.refreshAllColumns();
}
}
@@ -760,18 +826,54 @@ export class TsviewS3Columns extends DeesElement {
}
}
private handleColumnDrop(e: DragEvent, columnIndex: number) {
private async handleColumnDrop(e: DragEvent, columnIndex: number) {
e.preventDefault();
e.stopPropagation();
this.dragCounters.clear();
this.dragOverColumnIndex = -1;
const files = e.dataTransfer?.files;
if (!files || files.length === 0) return;
const items = e.dataTransfer?.items;
if (!items || items.length === 0) return;
const targetPrefix = this.dragOverFolderPrefix ?? this.columns[columnIndex].prefix;
this.clearFolderHover();
this.uploadFiles(Array.from(files), targetPrefix);
// Collect all files (including from nested folders)
const allFiles: { file: File; relativePath: string }[] = [];
const processEntry = async (entry: FileSystemEntry, path: string): Promise<void> => {
if (entry.isFile) {
const fileEntry = entry as FileSystemFileEntry;
const file = await new Promise<File>((resolve, reject) => {
fileEntry.file(resolve, reject);
});
allFiles.push({ file, relativePath: path + file.name });
} else if (entry.isDirectory) {
const dirEntry = entry as FileSystemDirectoryEntry;
const reader = dirEntry.createReader();
const entries = await new Promise<FileSystemEntry[]>((resolve, reject) => {
reader.readEntries(resolve, reject);
});
for (const childEntry of entries) {
await processEntry(childEntry, path + entry.name + '/');
}
}
};
// Process all dropped items
for (let i = 0; i < items.length; i++) {
const item = items[i];
if (item.kind === 'file') {
const entry = (item as any).webkitGetAsEntry();
if (entry) {
await processEntry(entry, '');
}
}
}
if (allFiles.length > 0) {
await this.uploadFilesWithPaths(allFiles, targetPrefix);
}
}
private handleFolderDragEnter(e: DragEvent, folderPrefix: string) {
@@ -822,7 +924,7 @@ export class TsviewS3Columns extends DeesElement {
if (success) {
this.showCreateDialog = false;
await this.loadInitialColumn();
await this.refreshColumnByPrefix(this.createDialogPrefix);
}
}

136
ts_web/services/changestream.service.ts
View File

@@ -60,8 +60,13 @@ export class ChangeStreamService {
private typedSocket: plugins.typedsocket.TypedSocket | null = null;
private isConnected = false;
private isConnecting = false;
private connectPromise: Promise<void> | null = null;
private subscriptions: Map<string, ISubscription> = new Map();
// Buffer activity events so they survive tab switches
private activityBuffer: IActivityEvent[] = [];
private static readonly ACTIVITY_BUFFER_SIZE = 500;
// RxJS Subjects for UI components
public readonly mongoChanges$ = new plugins.smartrx.rxjs.Subject<IMongoChangeEvent>();
public readonly s3Changes$ = new plugins.smartrx.rxjs.Subject<IS3ChangeEvent>();
@@ -74,48 +79,75 @@ export class ChangeStreamService {
}
/**
* Connect to the WebSocket server
* Connect to the WebSocket server.
* If a connection is already in progress, waits for it to complete.
*/
public async connect(): Promise<void> {
if (this.isConnected || this.isConnecting) {
if (this.isConnected) {
return;
}
// If already connecting, wait for the existing attempt to finish
if (this.isConnecting && this.connectPromise) {
return this.connectPromise;
}
this.isConnecting = true;
this.connectionStatus$.next('connecting');
this.connectPromise = (async () => {
try {
// Create client router to handle server-initiated pushes
const clientRouter = new plugins.typedrequest.TypedRouter();
// Register handlers for server push events
this.registerPushHandlers(clientRouter);
// Connect to WebSocket server using current origin
this.typedSocket = await plugins.typedsocket.TypedSocket.createClient(
clientRouter,
plugins.typedsocket.TypedSocket.useWindowLocationOriginUrl()
);
this.isConnected = true;
this.isConnecting = false;
this.connectionStatus$.next('connected');
console.log('[ChangeStream] WebSocket connected');
// Handle reconnection events via statusSubject
this.typedSocket.statusSubject.subscribe((status) => {
if (status === 'disconnected') {
this.handleDisconnect();
} else if (status === 'connected') {
this.handleReconnect();
}
});
} catch (error) {
this.isConnecting = false;
this.connectPromise = null;
this.connectionStatus$.next('disconnected');
console.error('[ChangeStream] Failed to connect:', error);
throw error;
}
})();
return this.connectPromise;
}
/**
* Ensure a WebSocket connection is established.
* If not connected, triggers connect() and returns whether connection succeeded.
*/
private async ensureConnected(): Promise<boolean> {
if (this.isConnected && this.typedSocket) {
return true;
}
try {
// Create client router to handle server-initiated pushes
const clientRouter = new plugins.typedrequest.TypedRouter();
// Register handlers for server push events
this.registerPushHandlers(clientRouter);
// Connect to WebSocket server using current origin
this.typedSocket = await plugins.typedsocket.TypedSocket.createClient(
clientRouter,
plugins.typedsocket.TypedSocket.useWindowLocationOriginUrl()
);
this.isConnected = true;
this.isConnecting = false;
this.connectionStatus$.next('connected');
console.log('[ChangeStream] WebSocket connected');
// Handle reconnection events via statusSubject
this.typedSocket.statusSubject.subscribe((status) => {
if (status === 'disconnected') {
this.handleDisconnect();
} else if (status === 'connected') {
this.handleReconnect();
}
});
} catch (error) {
this.isConnecting = false;
this.connectionStatus$.next('disconnected');
console.error('[ChangeStream] Failed to connect:', error);
throw error;
await this.connect();
return this.isConnected;
} catch {
return false;
}
}
@@ -135,6 +167,7 @@ export class ChangeStreamService {
this.typedSocket = null;
this.isConnected = false;
this.connectPromise = null;
this.subscriptions.clear();
this.connectionStatus$.next('disconnected');
@@ -172,6 +205,10 @@ export class ChangeStreamService {
new plugins.typedrequest.TypedHandler<any>(
'pushActivityEvent',
async (data: { event: IActivityEvent }) => {
this.activityBuffer.push(data.event);
if (this.activityBuffer.length > ChangeStreamService.ACTIVITY_BUFFER_SIZE) {
this.activityBuffer = this.activityBuffer.slice(-ChangeStreamService.ACTIVITY_BUFFER_SIZE);
}
this.activityEvents$.next(data.event);
return { received: true };
}
@@ -228,8 +265,11 @@ export class ChangeStreamService {
*/
public async subscribeToCollection(database: string, collection: string): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
const connected = await this.ensureConnected();
if (!connected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
}
}
const key = `${database}/${collection}`;
@@ -307,8 +347,11 @@ export class ChangeStreamService {
*/
public async subscribeToBucket(bucket: string, prefix?: string): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
const connected = await this.ensureConnected();
if (!connected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
}
}
const key = prefix ? `${bucket}/${prefix}` : bucket;
@@ -387,8 +430,11 @@ export class ChangeStreamService {
*/
public async subscribeToActivity(): Promise<boolean> {
if (!this.typedSocket || !this.isConnected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
const connected = await this.ensureConnected();
if (!connected) {
console.warn('[ChangeStream] Not connected, cannot subscribe');
return false;
}
}
// Check if already subscribed
@@ -450,7 +496,10 @@ export class ChangeStreamService {
*/
public async getRecentActivity(limit: number = 100): Promise<IActivityEvent[]> {
if (!this.typedSocket || !this.isConnected) {
return [];
const connected = await this.ensureConnected();
if (!connected) {
return [];
}
}
try {
@@ -470,6 +519,13 @@ export class ChangeStreamService {
return this.subscriptions.has('activity:activity');
}
/**
* Get buffered activity events (captured regardless of UI subscriber)
*/
public getBufferedActivity(): IActivityEvent[] {
return [...this.activityBuffer];
}
// ===========================================
// Observables for UI Components
// ===========================================