v6.1.1

changelog.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 2025-12-08 - 6.1.1 - fix(watchers/watcher.node)
+Improve Node watcher robustness: inode tracking, ENOSPC detection, enhanced health checks and temp-file handling
+
+- Track directory inodes (watchedInodes) and restart watchers if inode changes are detected (addresses stale watchers when directories are replaced).
+- Health check now validates inode stability and explicitly detects ENOSPC (inotify max_user_watches) errors, emitting errors and logging a recommended fix command.
+- Detect ENOSPC in FSWatcher error events and log guidance to increase inotify limits.
+- Clear inode tracking state on watcher stop to avoid stale state across restarts.
+- Improve temporary file handling and logging to avoid dropping events for atomic writes (only skip pure temp files and log skipped temp events).
+- Documentation (readme.hints.md) updated with robustness notes, known fs.watch limitations, and example logs.
+
+## 2025-12-08 - 6.1.0 - feat(watcher.node)
+Add automatic restart, periodic health checks, and safe event emission to Node watcher; improve logging and stat handling
+
+- NodeWatcher: introduced safeEmit() to isolate subscriber errors and prevent watcher crashes
+- Auto-restart on failure with exponential backoff (1s → 30s) and up to 3 retry attempts per watched base path
+- Periodic health checks (every 30s) to detect missing watched paths and trigger automatic restarts
+- Handle unexpected FSWatcher 'close' events and restart watchers when they close silently
+- Verbose lifecycle logging with `[smartwatch]` prefix for start/stop/health/restart events
+- Clear restart tracking and stop health checks on watcher.stop() to ensure clean shutdown
+- Improved statSafe() to normalize followSymlinks logic and log non-ENO errors as warnings
+- Updated readme.hints.md documenting the new robustness features (v6.1.0+)
+
 ## 2025-12-08 - 6.0.0 - BREAKING CHANGE(watchers)
 Replace polling-based write stabilization with debounce-based event coalescing and simplify watcher options
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartwatch",
-  "version": "6.0.0",
+  "version": "6.1.1",
   "private": false,
   "description": "A cross-runtime file watcher with glob pattern support for Node.js, Deno, and Bun.",
   "main": "dist_ts/index.js",

readme.hints.md

@@ -71,6 +71,57 @@ The `WriteStabilizer` class replaces chokidar's built-in write stabilization:
 - **Deno**: Works on all versions with `Deno.watchFs()`
 - **Bun**: Uses Node.js compatibility layer
 
+### Robustness Features (v6.1.0+)
+
+The Node.js watcher includes automatic recovery mechanisms based on learnings from [chokidar](https://github.com/paulmillr/chokidar) and known [fs.watch issues](https://github.com/nodejs/node/issues/47058):
+
+**Auto-restart on failure:**
+- Watchers automatically restart when errors occur
+- Exponential backoff (1s → 30s max)
+- Maximum 3 retry attempts before giving up
+
+**Inode tracking (critical for long-running watchers):**
+- `fs.watch()` watches the **inode**, not the path!
+- When directories are replaced (git checkout, atomic saves), the inode changes
+- Health check detects inode changes and restarts the watcher
+- This is the most common cause of "watcher stops working after some time"
+
+**Health check monitoring:**
+- 30-second periodic health checks
+- Detects when watched paths disappear
+- Detects inode changes (directory replacement)
+- Detects ENOSPC errors (inotify limit exceeded)
+
+**ENOSPC detection (Linux inotify limit):**
+- Detects when `/proc/sys/fs/inotify/max_user_watches` is exceeded
+- Logs fix command: `echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p`
+
+**Error isolation:**
+- Subscriber errors don't crash the watcher
+- All events emitted via `safeEmit()` with try-catch
+
+**Verbose logging:**
+- All lifecycle events logged with `[smartwatch]` prefix
+- Helps debug watcher issues in production
+
+Example log output:
+```
+[smartwatch] Starting watcher for 1 base path(s)...
+[smartwatch] Started watching: ./test/assets/
+[smartwatch] Starting health check (every 30s)
+[smartwatch] Watcher started with 1 active watcher(s)
+[smartwatch] Health check: 1 watchers active
+[smartwatch] Inode changed for ./src: 12345 -> 67890
+[smartwatch] fs.watch watches inode, not path - restarting watcher
+```
+
+### Known fs.watch Limitations
+
+1. **Watches inode, not path** - If a directory is replaced, watcher goes stale
+2. **inotify limits on Linux** - Default `max_user_watches` (8192) may be too low
+3. **No events for some atomic writes** - Some editors' save patterns may not trigger events
+4. **Platform differences** - Linux uses inotify, macOS uses FSEvents/kqueue
+
 ### Testing
 
 ```bash

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartwatch',
-  version: '6.0.0',
+  version: '6.1.1',
   description: 'A cross-runtime file watcher with glob pattern support for Node.js, Deno, and Bun.'
 }

ts/watchers/watcher.node.ts

@@ -14,10 +14,144 @@ export class NodeWatcher implements IWatcher {
   // Debounce: pending emits per file path
   private pendingEmits: Map<string, NodeJS.Timeout> = new Map();
 
+  // Restart tracking
+  private restartDelays: Map<string, number> = new Map();
+  private restartAttempts: Map<string, number> = new Map();
+  private healthCheckInterval: NodeJS.Timeout | null = null;
+
+  // Inode tracking - detect when directories are replaced (atomic saves, etc.)
+  // fs.watch watches the inode, not the path. If inode changes, we need to restart.
+  private watchedInodes: Map<string, bigint> = new Map();
+
+  // Configuration constants
+  private static readonly MAX_RETRIES = 3;
+  private static readonly INITIAL_RESTART_DELAY = 1000;
+  private static readonly MAX_RESTART_DELAY = 30000;
+  private static readonly HEALTH_CHECK_INTERVAL = 30000;
+
   public readonly events$ = new smartrx.rxjs.Subject<IWatchEvent>();
 
   constructor(private options: IWatcherOptions) {}
 
+  /**
+   * Safely emit an event, catching any subscriber errors
+   */
+  private safeEmit(event: IWatchEvent): void {
+    try {
+      this.events$.next(event);
+    } catch (error) {
+      console.error('[smartwatch] Subscriber threw error (isolated):', error);
+      // Don't let subscriber errors kill the watcher
+    }
+  }
+
+  /**
+   * Restart a watcher after an error with exponential backoff
+   */
+  private async restartWatcher(basePath: string, error: Error): Promise<void> {
+    const attempts = (this.restartAttempts.get(basePath) || 0) + 1;
+    this.restartAttempts.set(basePath, attempts);
+
+    console.log(`[smartwatch] Watcher error for ${basePath}: ${error.message}`);
+    console.log(`[smartwatch] Restart attempt ${attempts}/${NodeWatcher.MAX_RETRIES}`);
+
+    if (attempts > NodeWatcher.MAX_RETRIES) {
+      console.error(`[smartwatch] Max retries exceeded for ${basePath}, giving up`);
+      this.safeEmit({
+        type: 'error',
+        path: basePath,
+        error: new Error(`Max restart retries (${NodeWatcher.MAX_RETRIES}) exceeded`)
+      });
+      return;
+    }
+
+    // Close failed watcher
+    const oldWatcher = this.watchers.get(basePath);
+    if (oldWatcher) {
+      try {
+        oldWatcher.close();
+      } catch {
+        // Ignore close errors
+      }
+      this.watchers.delete(basePath);
+    }
+
+    // Exponential backoff
+    const delay = this.restartDelays.get(basePath) || NodeWatcher.INITIAL_RESTART_DELAY;
+    console.log(`[smartwatch] Waiting ${delay}ms before restart...`);
+    await new Promise((resolve) => setTimeout(resolve, delay));
+    this.restartDelays.set(basePath, Math.min(delay * 2, NodeWatcher.MAX_RESTART_DELAY));
+
+    try {
+      await this.watchPath(basePath, 0);
+      console.log(`[smartwatch] Successfully restarted watcher for ${basePath}`);
+      this.restartDelays.set(basePath, NodeWatcher.INITIAL_RESTART_DELAY);
+      this.restartAttempts.set(basePath, 0);
+    } catch (restartError) {
+      console.error(`[smartwatch] Restart failed for ${basePath}:`, restartError);
+      this.restartWatcher(basePath, restartError as Error); // Recursive retry
+    }
+  }
+
+  /**
+   * Start periodic health checks to detect silent failures
+   * Checks for:
+   * 1. Path no longer exists
+   * 2. Inode changed (directory was replaced - fs.watch watches inode, not path!)
+   */
+  private startHealthCheck(): void {
+    console.log('[smartwatch] Starting health check (every 30s)');
+    this.healthCheckInterval = setInterval(async () => {
+      console.log(`[smartwatch] Health check: ${this.watchers.size} watchers active`);
+      for (const [basePath] of this.watchers) {
+        try {
+          const stats = await fs.promises.stat(basePath);
+          const currentInode = stats.ino;
+          const previousInode = this.watchedInodes.get(basePath);
+
+          if (!stats) {
+            console.error(`[smartwatch] Health check failed: ${basePath} no longer exists`);
+            this.safeEmit({
+              type: 'error',
+              path: basePath,
+              error: new Error('Watched path no longer exists')
+            });
+            this.restartWatcher(basePath, new Error('Watched path disappeared'));
+          } else if (previousInode !== undefined && BigInt(currentInode) !== previousInode) {
+            // CRITICAL: Inode changed! fs.watch is now watching a stale inode.
+            // This happens when the directory is replaced (atomic operations, git checkout, etc.)
+            console.warn(`[smartwatch] Inode changed for ${basePath}: ${previousInode} -> ${currentInode}`);
+            console.warn('[smartwatch] fs.watch watches inode, not path - restarting watcher');
+            this.restartWatcher(basePath, new Error('Inode changed - directory was replaced'));
+          }
+        } catch (error: any) {
+          if (error.code === 'ENOENT') {
+            console.error(`[smartwatch] Health check failed: ${basePath} no longer exists`);
+            this.restartWatcher(basePath, new Error('Watched path disappeared'));
+          } else if (error.code === 'ENOSPC') {
+            // inotify watch limit exceeded - critical system issue
+            console.error(`[smartwatch] ENOSPC: inotify watch limit exceeded!`);
+            console.error('[smartwatch] Fix: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p');
+            this.safeEmit({ type: 'error', path: basePath, error });
+          } else {
+            console.error(`[smartwatch] Health check error for ${basePath}:`, error);
+          }
+        }
+      }
+    }, NodeWatcher.HEALTH_CHECK_INTERVAL);
+  }
+
+  /**
+   * Stop health check interval
+   */
+  private stopHealthCheck(): void {
+    if (this.healthCheckInterval) {
+      clearInterval(this.healthCheckInterval);
+      this.healthCheckInterval = null;
+      console.log('[smartwatch] Stopped health check');
+    }
+  }
+
   /**
    * Check if a file is a temporary file created by editors
    */
@@ -40,6 +174,8 @@ export class NodeWatcher implements IWatcher {
       return;
     }
 
+    console.log(`[smartwatch] Starting watcher for ${this.options.basePaths.length} base path(s)...`);
+
     try {
       // Start watching each base path
       for (const basePath of this.options.basePaths) {
@@ -48,22 +184,31 @@ export class NodeWatcher implements IWatcher {
 
       this._isWatching = true;
 
+      // Start health check monitoring
+      this.startHealthCheck();
+
       // Perform initial scan to emit 'add' events for existing files
       for (const basePath of this.options.basePaths) {
         await this.scanDirectory(basePath, 0);
       }
 
       // Emit ready event
-      this.events$.next({ type: 'ready', path: '' });
+      this.safeEmit({ type: 'ready', path: '' });
+      console.log(`[smartwatch] Watcher started with ${this.watchers.size} active watcher(s)`);
     } catch (error: any) {
-      this.events$.next({ type: 'error', path: '', error });
+      console.error('[smartwatch] Failed to start watcher:', error);
+      this.safeEmit({ type: 'error', path: '', error });
       throw error;
     }
   }
 
   async stop(): Promise<void> {
+    console.log('[smartwatch] Stopping watcher...');
     this._isWatching = false;
 
+    // Stop health check monitoring
+    this.stopHealthCheck();
+
     // Cancel all pending debounced emits
     for (const timeout of this.pendingEmits.values()) {
       clearTimeout(timeout);
@@ -71,11 +216,19 @@ export class NodeWatcher implements IWatcher {
     this.pendingEmits.clear();
 
     // Close all watchers
-    for (const [, watcher] of this.watchers) {
+    for (const [watchPath, watcher] of this.watchers) {
+      console.log(`[smartwatch] Closing watcher for: ${watchPath}`);
       watcher.close();
     }
     this.watchers.clear();
     this.watchedFiles.clear();
+
+    // Clear restart tracking state
+    this.restartDelays.clear();
+    this.restartAttempts.clear();
+    this.watchedInodes.clear();
+
+    console.log('[smartwatch] Watcher stopped');
   }
 
   /**
@@ -93,6 +246,10 @@ export class NodeWatcher implements IWatcher {
       }
 
       if (stats.isDirectory()) {
+        // Store inode for health check - fs.watch watches inode, not path!
+        // If inode changes (directory replaced), watcher becomes stale
+        this.watchedInodes.set(watchPath, BigInt(stats.ino));
+
         // Watch the directory with recursive option (Node.js 20+ supports this on all platforms)
         const watcher = fs.watch(
           watchPath,
@@ -104,14 +261,35 @@ export class NodeWatcher implements IWatcher {
           }
         );
 
-        watcher.on('error', (error) => {
-          this.events$.next({ type: 'error', path: watchPath, error });
+        watcher.on('error', (error: NodeJS.ErrnoException) => {
+          console.error(`[smartwatch] FSWatcher error event on ${watchPath}:`, error);
+
+          // Detect inotify watch limit exceeded - common cause of "stops working"
+          if (error.code === 'ENOSPC') {
+            console.error('[smartwatch] CRITICAL: inotify watch limit exceeded!');
+            console.error('[smartwatch] Fix with: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p');
+          }
+
+          this.safeEmit({ type: 'error', path: watchPath, error });
+          if (this._isWatching) {
+            this.restartWatcher(watchPath, error);
+          }
+        });
+
+        // Handle 'close' event - fs.watch can close without error
+        watcher.on('close', () => {
+          console.warn(`[smartwatch] FSWatcher closed unexpectedly for ${watchPath}`);
+          if (this._isWatching) {
+            this.restartWatcher(watchPath, new Error('Watcher closed unexpectedly'));
+          }
         });
 
         this.watchers.set(watchPath, watcher);
+        console.log(`[smartwatch] Started watching: ${watchPath}`);
       }
     } catch (error: any) {
-      this.events$.next({ type: 'error', path: watchPath, error });
+      console.error(`[smartwatch] Failed to watch path ${watchPath}:`, error);
+      this.safeEmit({ type: 'error', path: watchPath, error });
     }
   }
 
@@ -125,8 +303,14 @@ export class NodeWatcher implements IWatcher {
   ): void {
     const fullPath = path.join(basePath, filename);
 
-    // Skip temporary files created by editors (atomic saves)
+    // Skip temporary files - but ONLY pure temp files, not the target of atomic writes
+    // Atomic writes: editor writes to file.tmp.xxx then renames to file
+    // We need to detect the final file, so only skip files that ARE temp files
+    // and haven't been renamed to the real file yet
     if (this.isTemporaryFile(fullPath)) {
+      // For temp files, we still want to track if they get renamed TO a real file
+      // The 'rename' event fires for both source and target, so we'll catch the real file
+      console.log(`[smartwatch] Skipping temp file event: ${filename}`);
       return;
     }
 
@@ -162,12 +346,12 @@ export class NodeWatcher implements IWatcher {
           if (stats.isDirectory()) {
             if (!this.watchedFiles.has(fullPath)) {
               this.watchedFiles.add(fullPath);
-              this.events$.next({ type: 'addDir', path: fullPath, stats });
+              this.safeEmit({ type: 'addDir', path: fullPath, stats });
             }
           } else {
             const wasWatched = this.watchedFiles.has(fullPath);
             this.watchedFiles.add(fullPath);
-            this.events$.next({
+            this.safeEmit({
               type: wasWatched ? 'change' : 'add',
               path: fullPath,
               stats
@@ -178,7 +362,7 @@ export class NodeWatcher implements IWatcher {
           if (this.watchedFiles.has(fullPath)) {
             const wasDir = this.isKnownDirectory(fullPath);
             this.watchedFiles.delete(fullPath);
-            this.events$.next({
+            this.safeEmit({
               type: wasDir ? 'unlinkDir' : 'unlink',
               path: fullPath
             });
@@ -191,18 +375,18 @@ export class NodeWatcher implements IWatcher {
           if (!wasWatched) {
             // This is actually an 'add' - file wasn't being watched before
             this.watchedFiles.add(fullPath);
-            this.events$.next({ type: 'add', path: fullPath, stats });
+            this.safeEmit({ type: 'add', path: fullPath, stats });
           } else {
-            this.events$.next({ type: 'change', path: fullPath, stats });
+            this.safeEmit({ type: 'change', path: fullPath, stats });
           }
         } else if (!stats && this.watchedFiles.has(fullPath)) {
           // File was deleted
           this.watchedFiles.delete(fullPath);
-          this.events$.next({ type: 'unlink', path: fullPath });
+          this.safeEmit({ type: 'unlink', path: fullPath });
         }
       }
     } catch (error: any) {
-      this.events$.next({ type: 'error', path: fullPath, error });
+      this.safeEmit({ type: 'error', path: fullPath, error });
     }
   }
 
@@ -233,16 +417,16 @@ export class NodeWatcher implements IWatcher {
 
         if (entry.isDirectory()) {
           this.watchedFiles.add(fullPath);
-          this.events$.next({ type: 'addDir', path: fullPath, stats });
+          this.safeEmit({ type: 'addDir', path: fullPath, stats });
           await this.scanDirectory(fullPath, depth + 1);
         } else if (entry.isFile()) {
           this.watchedFiles.add(fullPath);
-          this.events$.next({ type: 'add', path: fullPath, stats });
+          this.safeEmit({ type: 'add', path: fullPath, stats });
         }
       }
     } catch (error: any) {
       if (error.code !== 'ENOENT' && error.code !== 'EACCES') {
-        this.events$.next({ type: 'error', path: dirPath, error });
+        this.safeEmit({ type: 'error', path: dirPath, error });
       }
     }
   }
@@ -252,12 +436,16 @@ export class NodeWatcher implements IWatcher {
    */
   private async statSafe(filePath: string): Promise<fs.Stats | null> {
     try {
-      if (this.options.followSymlinks) {
-        return await fs.promises.stat(filePath);
-      } else {
-        return await fs.promises.lstat(filePath);
+      return await (this.options.followSymlinks
+        ? fs.promises.stat(filePath)
+        : fs.promises.lstat(filePath));
+    } catch (error: any) {
+      // Only silently return null for expected "file doesn't exist" errors
+      if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
+        return null;
       }
-    } catch {
+      // Log other errors (permission, I/O) but still return null
+      console.warn(`[smartwatch] statSafe warning for ${filePath}: ${error.code} - ${error.message}`);
       return null;
     }
   }