diff --git a/changelog.md b/changelog.md index 8be5226..9043592 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,17 @@ # Changelog +## 2025-12-08 - 6.1.0 - feat(watcher.node) +Add automatic restart, periodic health checks, and safe event emission to Node watcher; improve logging and stat handling + +- NodeWatcher: introduced safeEmit() to isolate subscriber errors and prevent watcher crashes +- Auto-restart on failure with exponential backoff (1s → 30s) and up to 3 retry attempts per watched base path +- Periodic health checks (every 30s) to detect missing watched paths and trigger automatic restarts +- Handle unexpected FSWatcher 'close' events and restart watchers when they close silently +- Verbose lifecycle logging with `[smartwatch]` prefix for start/stop/health/restart events +- Clear restart tracking and stop health checks on watcher.stop() to ensure clean shutdown +- Improved statSafe() to normalize followSymlinks logic and log non-ENO errors as warnings +- Updated readme.hints.md documenting the new robustness features (v6.1.0+) + ## 2025-12-08 - 6.0.0 - BREAKING CHANGE(watchers) Replace polling-based write stabilization with debounce-based event coalescing and simplify watcher options diff --git a/readme.hints.md b/readme.hints.md index ace60f3..0bae412 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -71,6 +71,37 @@ The `WriteStabilizer` class replaces chokidar's built-in write stabilization: - **Deno**: Works on all versions with `Deno.watchFs()` - **Bun**: Uses Node.js compatibility layer +### Robustness Features (v6.1.0+) + +The Node.js watcher includes automatic recovery mechanisms: + +**Auto-restart on failure:** +- Watchers automatically restart when errors occur +- Exponential backoff (1s → 30s max) +- Maximum 3 retry attempts before giving up + +**Health check monitoring:** +- 30-second periodic health checks +- Detects when watched paths disappear +- Triggers automatic restart when issues detected + +**Error isolation:** +- Subscriber errors don't crash the watcher +- All events emitted via `safeEmit()` with try-catch + +**Verbose logging:** +- All lifecycle events logged with `[smartwatch]` prefix +- Helps debug watcher issues in production + +Example log output: +``` +[smartwatch] Starting watcher for 1 base path(s)... +[smartwatch] Started watching: ./test/assets/ +[smartwatch] Starting health check (every 30s) +[smartwatch] Watcher started with 1 active watcher(s) +[smartwatch] Health check: 1 watchers active +``` + ### Testing ```bash diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 8773374..5e7b97a 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartwatch', - version: '6.0.0', + version: '6.1.0', description: 'A cross-runtime file watcher with glob pattern support for Node.js, Deno, and Bun.' } diff --git a/ts/watchers/watcher.node.ts b/ts/watchers/watcher.node.ts index 24347b6..3b28382 100644 --- a/ts/watchers/watcher.node.ts +++ b/ts/watchers/watcher.node.ts @@ -14,10 +14,114 @@ export class NodeWatcher implements IWatcher { // Debounce: pending emits per file path private pendingEmits: Map = new Map(); + // Restart tracking + private restartDelays: Map = new Map(); + private restartAttempts: Map = new Map(); + private healthCheckInterval: NodeJS.Timeout | null = null; + + // Configuration constants + private static readonly MAX_RETRIES = 3; + private static readonly INITIAL_RESTART_DELAY = 1000; + private static readonly MAX_RESTART_DELAY = 30000; + private static readonly HEALTH_CHECK_INTERVAL = 30000; + public readonly events$ = new smartrx.rxjs.Subject(); constructor(private options: IWatcherOptions) {} + /** + * Safely emit an event, catching any subscriber errors + */ + private safeEmit(event: IWatchEvent): void { + try { + this.events$.next(event); + } catch (error) { + console.error('[smartwatch] Subscriber threw error (isolated):', error); + // Don't let subscriber errors kill the watcher + } + } + + /** + * Restart a watcher after an error with exponential backoff + */ + private async restartWatcher(basePath: string, error: Error): Promise { + const attempts = (this.restartAttempts.get(basePath) || 0) + 1; + this.restartAttempts.set(basePath, attempts); + + console.log(`[smartwatch] Watcher error for ${basePath}: ${error.message}`); + console.log(`[smartwatch] Restart attempt ${attempts}/${NodeWatcher.MAX_RETRIES}`); + + if (attempts > NodeWatcher.MAX_RETRIES) { + console.error(`[smartwatch] Max retries exceeded for ${basePath}, giving up`); + this.safeEmit({ + type: 'error', + path: basePath, + error: new Error(`Max restart retries (${NodeWatcher.MAX_RETRIES}) exceeded`) + }); + return; + } + + // Close failed watcher + const oldWatcher = this.watchers.get(basePath); + if (oldWatcher) { + try { + oldWatcher.close(); + } catch { + // Ignore close errors + } + this.watchers.delete(basePath); + } + + // Exponential backoff + const delay = this.restartDelays.get(basePath) || NodeWatcher.INITIAL_RESTART_DELAY; + console.log(`[smartwatch] Waiting ${delay}ms before restart...`); + await new Promise((resolve) => setTimeout(resolve, delay)); + this.restartDelays.set(basePath, Math.min(delay * 2, NodeWatcher.MAX_RESTART_DELAY)); + + try { + await this.watchPath(basePath, 0); + console.log(`[smartwatch] Successfully restarted watcher for ${basePath}`); + this.restartDelays.set(basePath, NodeWatcher.INITIAL_RESTART_DELAY); + this.restartAttempts.set(basePath, 0); + } catch (restartError) { + console.error(`[smartwatch] Restart failed for ${basePath}:`, restartError); + this.restartWatcher(basePath, restartError as Error); // Recursive retry + } + } + + /** + * Start periodic health checks to detect silent failures + */ + private startHealthCheck(): void { + console.log('[smartwatch] Starting health check (every 30s)'); + this.healthCheckInterval = setInterval(async () => { + console.log(`[smartwatch] Health check: ${this.watchers.size} watchers active`); + for (const [basePath] of this.watchers) { + const stats = await this.statSafe(basePath); + if (!stats && this._isWatching) { + console.error(`[smartwatch] Health check failed: ${basePath} no longer exists`); + this.safeEmit({ + type: 'error', + path: basePath, + error: new Error('Watched path no longer exists') + }); + this.restartWatcher(basePath, new Error('Watched path disappeared')); + } + } + }, NodeWatcher.HEALTH_CHECK_INTERVAL); + } + + /** + * Stop health check interval + */ + private stopHealthCheck(): void { + if (this.healthCheckInterval) { + clearInterval(this.healthCheckInterval); + this.healthCheckInterval = null; + console.log('[smartwatch] Stopped health check'); + } + } + /** * Check if a file is a temporary file created by editors */ @@ -40,6 +144,8 @@ export class NodeWatcher implements IWatcher { return; } + console.log(`[smartwatch] Starting watcher for ${this.options.basePaths.length} base path(s)...`); + try { // Start watching each base path for (const basePath of this.options.basePaths) { @@ -48,22 +154,31 @@ export class NodeWatcher implements IWatcher { this._isWatching = true; + // Start health check monitoring + this.startHealthCheck(); + // Perform initial scan to emit 'add' events for existing files for (const basePath of this.options.basePaths) { await this.scanDirectory(basePath, 0); } // Emit ready event - this.events$.next({ type: 'ready', path: '' }); + this.safeEmit({ type: 'ready', path: '' }); + console.log(`[smartwatch] Watcher started with ${this.watchers.size} active watcher(s)`); } catch (error: any) { - this.events$.next({ type: 'error', path: '', error }); + console.error('[smartwatch] Failed to start watcher:', error); + this.safeEmit({ type: 'error', path: '', error }); throw error; } } async stop(): Promise { + console.log('[smartwatch] Stopping watcher...'); this._isWatching = false; + // Stop health check monitoring + this.stopHealthCheck(); + // Cancel all pending debounced emits for (const timeout of this.pendingEmits.values()) { clearTimeout(timeout); @@ -71,11 +186,18 @@ export class NodeWatcher implements IWatcher { this.pendingEmits.clear(); // Close all watchers - for (const [, watcher] of this.watchers) { + for (const [watchPath, watcher] of this.watchers) { + console.log(`[smartwatch] Closing watcher for: ${watchPath}`); watcher.close(); } this.watchers.clear(); this.watchedFiles.clear(); + + // Clear restart tracking state + this.restartDelays.clear(); + this.restartAttempts.clear(); + + console.log('[smartwatch] Watcher stopped'); } /** @@ -105,13 +227,27 @@ export class NodeWatcher implements IWatcher { ); watcher.on('error', (error) => { - this.events$.next({ type: 'error', path: watchPath, error }); + console.error(`[smartwatch] FSWatcher error event on ${watchPath}:`, error); + this.safeEmit({ type: 'error', path: watchPath, error }); + if (this._isWatching) { + this.restartWatcher(watchPath, error); + } + }); + + // Handle 'close' event - fs.watch can close without error + watcher.on('close', () => { + console.warn(`[smartwatch] FSWatcher closed unexpectedly for ${watchPath}`); + if (this._isWatching) { + this.restartWatcher(watchPath, new Error('Watcher closed unexpectedly')); + } }); this.watchers.set(watchPath, watcher); + console.log(`[smartwatch] Started watching: ${watchPath}`); } } catch (error: any) { - this.events$.next({ type: 'error', path: watchPath, error }); + console.error(`[smartwatch] Failed to watch path ${watchPath}:`, error); + this.safeEmit({ type: 'error', path: watchPath, error }); } } @@ -162,12 +298,12 @@ export class NodeWatcher implements IWatcher { if (stats.isDirectory()) { if (!this.watchedFiles.has(fullPath)) { this.watchedFiles.add(fullPath); - this.events$.next({ type: 'addDir', path: fullPath, stats }); + this.safeEmit({ type: 'addDir', path: fullPath, stats }); } } else { const wasWatched = this.watchedFiles.has(fullPath); this.watchedFiles.add(fullPath); - this.events$.next({ + this.safeEmit({ type: wasWatched ? 'change' : 'add', path: fullPath, stats @@ -178,7 +314,7 @@ export class NodeWatcher implements IWatcher { if (this.watchedFiles.has(fullPath)) { const wasDir = this.isKnownDirectory(fullPath); this.watchedFiles.delete(fullPath); - this.events$.next({ + this.safeEmit({ type: wasDir ? 'unlinkDir' : 'unlink', path: fullPath }); @@ -191,18 +327,18 @@ export class NodeWatcher implements IWatcher { if (!wasWatched) { // This is actually an 'add' - file wasn't being watched before this.watchedFiles.add(fullPath); - this.events$.next({ type: 'add', path: fullPath, stats }); + this.safeEmit({ type: 'add', path: fullPath, stats }); } else { - this.events$.next({ type: 'change', path: fullPath, stats }); + this.safeEmit({ type: 'change', path: fullPath, stats }); } } else if (!stats && this.watchedFiles.has(fullPath)) { // File was deleted this.watchedFiles.delete(fullPath); - this.events$.next({ type: 'unlink', path: fullPath }); + this.safeEmit({ type: 'unlink', path: fullPath }); } } } catch (error: any) { - this.events$.next({ type: 'error', path: fullPath, error }); + this.safeEmit({ type: 'error', path: fullPath, error }); } } @@ -233,16 +369,16 @@ export class NodeWatcher implements IWatcher { if (entry.isDirectory()) { this.watchedFiles.add(fullPath); - this.events$.next({ type: 'addDir', path: fullPath, stats }); + this.safeEmit({ type: 'addDir', path: fullPath, stats }); await this.scanDirectory(fullPath, depth + 1); } else if (entry.isFile()) { this.watchedFiles.add(fullPath); - this.events$.next({ type: 'add', path: fullPath, stats }); + this.safeEmit({ type: 'add', path: fullPath, stats }); } } } catch (error: any) { if (error.code !== 'ENOENT' && error.code !== 'EACCES') { - this.events$.next({ type: 'error', path: dirPath, error }); + this.safeEmit({ type: 'error', path: dirPath, error }); } } } @@ -252,12 +388,16 @@ export class NodeWatcher implements IWatcher { */ private async statSafe(filePath: string): Promise { try { - if (this.options.followSymlinks) { - return await fs.promises.stat(filePath); - } else { - return await fs.promises.lstat(filePath); + return await (this.options.followSymlinks + ? fs.promises.stat(filePath) + : fs.promises.lstat(filePath)); + } catch (error: any) { + // Only silently return null for expected "file doesn't exist" errors + if (error.code === 'ENOENT' || error.code === 'ENOTDIR') { + return null; } - } catch { + // Log other errors (permission, I/O) but still return null + console.warn(`[smartwatch] statSafe warning for ${filePath}: ${error.code} - ${error.message}`); return null; } }