diff --git a/changelog.md b/changelog.md index 08437b5..ed59233 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,14 @@ # Changelog +## 2025-12-11 - 6.3.0 - feat(watchers) +Integrate chokidar-based Node watcher, expose awaitWriteFinish options, and update docs/tests + +- Add chokidar dependency and implement NodeWatcher as a chokidar wrapper for Node.js/Bun +- Expose awaitWriteFinish, stabilityThreshold and pollInterval in IWatcherOptions and wire them into the NodeWatcher +- Update watcher factory to return NodeWatcher for Node/Bun and DenoWatcher for Deno +- Adjust tests to wait for chokidar readiness and to expect chokidar's atomic handling (delete+recreate -> change) +- Revise README and technical hints to document chokidar usage and cross-runtime behavior + ## 2025-12-11 - 6.2.5 - fix(watcher.node) Normalize paths and improve Node watcher robustness: restart/rescan on errors (including ENOSPC), clear stale state, and remove legacy throttler diff --git a/package.json b/package.json index a0f14b9..63847e1 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@push.rocks/smartenv": "^6.0.0", "@push.rocks/smartpromise": "^4.2.3", "@push.rocks/smartrx": "^3.0.10", + "chokidar": "^5.0.0", "picomatch": "^4.0.3" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4bf4cb3..95e3b5e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@push.rocks/smartrx': specifier: ^3.0.10 version: 3.0.10 + chokidar: + specifier: ^5.0.0 + version: 5.0.0 picomatch: specifier: ^4.0.3 version: 4.0.3 @@ -2199,6 +2202,10 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + chokidar@5.0.0: + resolution: {integrity: sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==} + engines: {node: '>= 20.19.0'} + chromium-bidi@5.1.0: resolution: {integrity: sha512-9MSRhWRVoRPDG0TgzkHrshFSJJNZzfY5UFqUMuksg7zL1yoZIZ3jLB0YAgHclbiAxPI86pBnwDX1tbzoiV8aFw==} peerDependencies: @@ -3631,6 +3638,10 @@ packages: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} + readdirp@5.0.0: + resolution: {integrity: sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==} + engines: {node: '>= 20.19.0'} + reflect-metadata@0.2.2: resolution: {integrity: sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==} @@ -7900,6 +7911,10 @@ snapshots: dependencies: readdirp: 4.1.2 + chokidar@5.0.0: + dependencies: + readdirp: 5.0.0 + chromium-bidi@5.1.0(devtools-protocol@0.0.1452169): dependencies: devtools-protocol: 0.0.1452169 @@ -9627,6 +9642,8 @@ snapshots: readdirp@4.1.2: {} + readdirp@5.0.0: {} + reflect-metadata@0.2.2: {} regenerator-runtime@0.14.1: {} diff --git a/readme.hints.md b/readme.hints.md index 7268ac3..3b0fd3d 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -1,14 +1,16 @@ -# smartchok - Technical Hints +# smartwatch - Technical Hints -## Native File Watching (v2.0.0+) +## Native File Watching (v6.0.0+) -The module now uses native file watching APIs instead of chokidar, providing cross-runtime support for Node.js, Deno, and Bun. +The module provides cross-runtime file watching support: +- **Node.js/Bun**: Uses [chokidar](https://github.com/paulmillr/chokidar) v5 +- **Deno**: Uses native `Deno.watchFs()` ### Exported Class -The package exports the `Smartwatch` class (not `Smartchok`): +The package exports the `Smartwatch` class: ```typescript -import { Smartwatch } from '@push.rocks/smartchok'; +import { Smartwatch } from '@push.rocks/smartwatch'; ``` ### Architecture @@ -16,198 +18,74 @@ import { Smartwatch } from '@push.rocks/smartchok'; ``` ts/ ├── smartwatch.classes.smartwatch.ts # Main Smartwatch class -├── smartwatch.plugins.ts # Dependencies (smartenv, picomatch, etc.) +├── smartwatch.plugins.ts # Dependencies ├── watchers/ │ ├── index.ts # Factory with runtime detection │ ├── interfaces.ts # IWatcher interface and types -│ ├── watcher.node.ts # Node.js/Bun implementation (fs.watch) -│ └── watcher.deno.ts # Deno implementation (Deno.watchFs) +│ ├── watcher.node.ts # Node.js/Bun: chokidar wrapper +│ └── watcher.deno.ts # Deno: Deno.watchFs wrapper └── utils/ └── write-stabilizer.ts # awaitWriteFinish polling implementation ``` ### Runtime Detection -Uses `@push.rocks/smartenv` v6.x for runtime detection: -- **Node.js/Bun**: Uses native `fs.watch()` with `{ recursive: true }` +Uses `@push.rocks/smartenv` for runtime detection: +- **Node.js/Bun**: Uses chokidar (battle-tested file watcher) - **Deno**: Uses `Deno.watchFs()` async iterable ### Dependencies -- **picomatch**: Glob pattern matching (zero deps, well-maintained) -- **@push.rocks/smartenv**: Runtime detection (Node.js, Deno, Bun) +- **chokidar**: Battle-tested file watcher for Node.js/Bun +- **picomatch**: Glob pattern matching (zero deps) +- **@push.rocks/smartenv**: Runtime detection - **@push.rocks/smartrx**: RxJS Subject/Observable management - **@push.rocks/smartpromise**: Deferred promise utilities - **@push.rocks/lik**: Stringmap for pattern storage -### Why picomatch? +### Chokidar Features (Node.js/Bun) -Native file watching APIs don't support glob patterns. Picomatch provides glob pattern matching with: -- Zero dependencies -- 164M+ weekly downloads -- Excellent security profile -- Full glob syntax support +The Node.js watcher (`ts/watchers/watcher.node.ts`) is a thin ~100 line wrapper around chokidar v5: + +```typescript +chokidar.watch(paths, { + persistent: true, + ignoreInitial: false, + followSymlinks: options.followSymlinks, + depth: options.depth, + atomic: true, // Handles atomic writes (delete+recreate, temp+rename) + awaitWriteFinish: { stabilityThreshold: 300, pollInterval: 100 }, +}); +``` + +**Chokidar handles all edge cases:** +- Atomic writes (temp file + rename pattern) → emits single 'change' event +- Delete + recreate detection → emits single 'change' event +- Inode tracking +- Cross-platform differences (inotify, FSEvents, etc.) +- Debouncing +- Write stabilization +- ENOSPC (inotify limit) errors ### Event Handling -Native events are normalized to a consistent interface: +Events are normalized across all runtimes: -| Node.js/Bun Event | Deno Event | Normalized Event | -|-------------------|------------|------------------| -| `rename` (file exists) | `create` | `add` | -| `rename` (file gone) | `remove` | `unlink` | -| `change` | `modify` | `change` | - -### awaitWriteFinish Implementation - -The `WriteStabilizer` class replaces chokidar's built-in write stabilization: -- Polls file size until stable (configurable threshold: 300ms default) -- Configurable poll interval (100ms default) -- Handles file deletion during write detection +| Event | Description | +|-------|-------------| +| `add` | File added | +| `change` | File modified | +| `unlink` | File removed | +| `addDir` | Directory added | +| `unlinkDir` | Directory removed | +| `ready` | Initial scan complete | +| `error` | Error occurred | ### Platform Requirements -- **Node.js 20+**: Required for native recursive watching on all platforms +- **Node.js 20+**: Required for chokidar v5 - **Deno**: Works on all versions with `Deno.watchFs()` -- **Bun**: Uses Node.js compatibility layer - -### Architecture (v6.3.0+) - Chokidar-Inspired - -The Node.js watcher has been refactored with elegant patterns inspired by [chokidar](https://github.com/paulmillr/chokidar): - -**DirEntry Class:** -- Tracks directory contents with proper disposal -- Encapsulates file tracking and inode management -- `dispose()` method freezes object to catch use-after-cleanup bugs - -**Path Normalization (v6.3.1+):** -- ALL paths are normalized to absolute at entry points -- Prevents relative/absolute path mismatch bugs -- `watchPath()`, `handleFsEvent()`, `scanDirectory()` all resolve paths - -**Restart Rescan (v6.3.1+):** -- When watcher restarts, it rescans the directory -- Catches files created during the restart window -- Clears stale DirEntry data before rescan -- Clears pending unlink timeouts to prevent stale events - -**ENOSPC Handling (v6.3.1+):** -- inotify limit errors now trigger watcher restart -- Previously only logged error without recovery - -**Atomic Write Handling:** -- Unlink events are queued with 100ms delay -- If add event arrives for same path within delay, unlink is cancelled -- Emits single `change` event instead of `unlink` + `add` -- Handles editor atomic saves elegantly - -**Closer Registry:** -- Maps watch paths to cleanup functions -- Ensures proper resource cleanup on stop -- `addCloser()` / `runClosers()` pattern - -**Event Constants Object:** -```typescript -const EV = { - ADD: 'add', - CHANGE: 'change', - UNLINK: 'unlink', - ADD_DIR: 'addDir', - UNLINK_DIR: 'unlinkDir', - READY: 'ready', - ERROR: 'error', -} as const; -``` - -**Configuration Constants:** -```typescript -const CONFIG = { - MAX_RETRIES: 3, - INITIAL_RESTART_DELAY: 1000, - MAX_RESTART_DELAY: 30000, - HEALTH_CHECK_INTERVAL: 30000, - ATOMIC_DELAY: 100, - TEMP_FILE_DELAY: 50, -} as const; -``` - -### Robustness Features (v6.1.0+) - -The Node.js watcher includes automatic recovery mechanisms based on learnings from [chokidar](https://github.com/paulmillr/chokidar) and known [fs.watch issues](https://github.com/nodejs/node/issues/47058): - -**Auto-restart on failure:** -- Watchers automatically restart when errors occur -- Exponential backoff (1s → 30s max) -- Maximum 3 retry attempts before giving up -- **v6.2.0+**: Race condition guards prevent orphan watchers when `stop()` is called during restart - -**Inode tracking (critical for long-running watchers):** -- `fs.watch()` watches the **inode**, not the path! -- When directories are replaced (git checkout, atomic saves), the inode changes -- Health check detects inode changes and restarts the watcher -- **v6.2.0+**: File-level inode tracking detects delete+recreate (common editor save pattern) -- This is the most common cause of "watcher stops working after some time" - -**Health check monitoring:** -- 30-second periodic health checks -- Detects when watched paths disappear -- Detects inode changes (directory replacement) -- Detects ENOSPC errors (inotify limit exceeded) -- **v6.2.0+**: Protected against dual-restart race conditions (health check + error handler) - -**ENOSPC detection (Linux inotify limit):** -- Detects when `/proc/sys/fs/inotify/max_user_watches` is exceeded -- Logs fix command: `echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p` - -**Error isolation:** -- Subscriber errors don't crash the watcher -- All events emitted via `safeEmit()` with try-catch - -**Untracked file handling (v6.2.0+):** -- Files created after initial scan are properly detected -- Untracked file deletions emit `unlink` events instead of being silently dropped - -**Event Deferral During Initial Scan (v6.2.2+):** -- Events are queued until initial scan completes -- Prevents race conditions where events arrive before `watchedFiles` is populated -- Deferred events are processed after scan completes - -**Event Sequence Tracking (v6.2.2+):** -- Debounce now tracks ALL events in sequence, not just the last one -- Prevents losing intermediate events (e.g., add→change→delete no longer loses add) -- Intelligent processing of event sequences: - - Delete+recreate with inode change → emits `unlink` then `add` - - Rapid create+delete → emits both events - - Multiple changes → single `change` event (debouncing) - -**Post-Stop Event Guards (v6.2.2+):** -- `handleFsEvent()` returns early if watcher is stopped -- Pending emits are cleared BEFORE setting `_isWatching = false` -- Prevents orphaned timeouts and events after `stop()` - -**Verbose logging:** -- All lifecycle events logged with `[smartwatch]` prefix -- Event sequences logged for debugging complex scenarios -- Helps debug watcher issues in production - -Example log output: -``` -[smartwatch] Starting watcher for 1 base path(s)... -[smartwatch] Started watching: ./test/assets/ -[smartwatch] Starting health check (every 30s) -[smartwatch] Watcher started with 1 active watcher(s) -[smartwatch] Health check: 1 watchers active -[smartwatch] Processing event sequence for ./src/file.ts: [rename, rename, change] -[smartwatch] File inode changed (delete+recreate): ./src/file.ts -[smartwatch] Previous inode: 12345, current: 67890 -``` - -### Known fs.watch Limitations - -1. **Watches inode, not path** - If a directory is replaced, watcher goes stale -2. **inotify limits on Linux** - Default `max_user_watches` (8192) may be too low -3. **No events for some atomic writes** - Some editors' save patterns may not trigger events -4. **Platform differences** - Linux uses inotify, macOS uses FSEvents/kqueue +- **Bun**: Uses Node.js compatibility layer with chokidar ### Testing @@ -217,15 +95,15 @@ pnpm test Test files: - **test.basic.ts** - Core functionality (add, change, unlink events) -- **test.inode.ts** - Inode change detection, atomic writes +- **test.inode.ts** - Atomic write detection (delete+recreate, temp+rename) - **test.stress.ts** - Rapid modifications, many files, interleaved operations Tests verify: - Creating Smartwatch instance - Adding glob patterns - Receiving 'add', 'change', 'unlink' events -- Inode change detection (delete+recreate pattern) -- Atomic write pattern (temp file + rename) +- Atomic write detection (delete+recreate → change event) +- Temp file + rename pattern detection - Rapid file modifications (debouncing) - Many files created rapidly - Interleaved add/change/delete operations diff --git a/test/test.basic.ts b/test/test.basic.ts index cc6edb5..b793888 100644 --- a/test/test.basic.ts +++ b/test/test.basic.ts @@ -74,11 +74,15 @@ tap.test('should add paths and start watching', async () => { testSmartwatch.add([`${TEST_DIR}/**/*.txt`]); await testSmartwatch.start(); expect(testSmartwatch.status).toEqual('watching'); + // Wait for chokidar to be ready + await delay(500); }); tap.test('should detect ADD event for new files', async () => { const addObservable = await testSmartwatch.getObservableFor('add'); - const eventPromise = waitForEvent(addObservable); + + // Subscribe FIRST, then create file + const eventPromise = waitForFileEvent(addObservable, 'add-test.txt'); // Create a new file const testFile = path.join(TEST_DIR, 'add-test.txt'); @@ -87,9 +91,9 @@ tap.test('should detect ADD event for new files', async () => { const [filePath] = await eventPromise; expect(filePath).toInclude('add-test.txt'); - // Cleanup - wait for atomic delay to complete (100ms debounce + 100ms atomic) + // Cleanup await fs.promises.unlink(testFile); - await delay(250); + await delay(200); }); tap.test('should detect CHANGE event for modified files', async () => { @@ -98,10 +102,10 @@ tap.test('should detect CHANGE event for modified files', async () => { await smartfile.memory.toFs('initial content', testFile); // Wait for add event to complete - await delay(200); + await delay(300); const changeObservable = await testSmartwatch.getObservableFor('change'); - const eventPromise = waitForEvent(changeObservable); + const eventPromise = waitForFileEvent(changeObservable, 'change-test.txt'); // Modify the file await smartfile.memory.toFs('modified content', testFile); @@ -109,9 +113,9 @@ tap.test('should detect CHANGE event for modified files', async () => { const [filePath] = await eventPromise; expect(filePath).toInclude('change-test.txt'); - // Cleanup - wait for atomic delay to complete + // Cleanup await fs.promises.unlink(testFile); - await delay(250); + await delay(200); }); tap.test('should detect UNLINK event for deleted files', async () => { @@ -120,7 +124,7 @@ tap.test('should detect UNLINK event for deleted files', async () => { await smartfile.memory.toFs('to be deleted', testFile); // Wait for add event to complete - await delay(200); + await delay(300); const unlinkObservable = await testSmartwatch.getObservableFor('unlink'); diff --git a/test/test.inode.ts b/test/test.inode.ts index c329d9b..07d4430 100644 --- a/test/test.inode.ts +++ b/test/test.inode.ts @@ -16,21 +16,25 @@ const TEST_DIR = './test/assets'; // Helper to delay const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -// Helper to wait for an event with timeout -async function waitForEvent( +// Helper to wait for an event with timeout (filters by filename) +async function waitForFileEvent( observable: smartrx.rxjs.Observable, + expectedFile: string, timeoutMs: number = 5000 ): Promise { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { subscription.unsubscribe(); - reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`)); + reject(new Error(`Timeout waiting for event on ${expectedFile} after ${timeoutMs}ms`)); }, timeoutMs); const subscription = observable.subscribe((value) => { - clearTimeout(timeout); - subscription.unsubscribe(); - resolve(value); + const [filePath] = value; + if (filePath.includes(expectedFile)) { + clearTimeout(timeout); + subscription.unsubscribe(); + resolve(value); + } }); }); } @@ -45,27 +49,31 @@ tap.test('setup: start watcher', async () => { testSmartwatch = new smartwatch.Smartwatch([`${TEST_DIR}/**/*.txt`]); await testSmartwatch.start(); expect(testSmartwatch.status).toEqual('watching'); + // Wait for chokidar to be ready + await delay(500); }); -tap.test('should detect delete+recreate (inode change scenario)', async () => { - // This simulates what many editors do: delete file, create new file +tap.test('should detect delete+recreate as change event (atomic handling)', async () => { + // Chokidar with atomic: true handles delete+recreate as a single change event + // This is the expected behavior for editor save patterns const testFile = path.join(TEST_DIR, 'inode-test.txt'); + // Clean up any leftover file from previous runs + try { await fs.promises.unlink(testFile); } catch {} + await delay(100); + // Create initial file await smartfile.memory.toFs('initial content', testFile); - await delay(200); + await delay(300); // Get the initial inode const initialStats = await fs.promises.stat(testFile); const initialInode = initialStats.ino; console.log(`[test] Initial inode: ${initialInode}`); - // With event sequence tracking, delete+recreate emits: unlink, then add - // This is more accurate than just emitting 'change' - const unlinkObservable = await testSmartwatch.getObservableFor('unlink'); - const addObservable = await testSmartwatch.getObservableFor('add'); - const unlinkPromise = waitForEvent(unlinkObservable, 3000); - const addPromise = waitForEvent(addObservable, 3000); + // Chokidar's atomic handling will emit a single 'change' event + const changeObservable = await testSmartwatch.getObservableFor('change'); + const eventPromise = waitForFileEvent(changeObservable, 'inode-test.txt', 3000); // Delete and recreate (this creates a new inode) await fs.promises.unlink(testFile); @@ -77,14 +85,14 @@ tap.test('should detect delete+recreate (inode change scenario)', async () => { console.log(`[test] New inode: ${newInode}`); expect(newInode).not.toEqual(initialInode); - // Should detect both unlink and add events for delete+recreate - const [[unlinkPath], [addPath]] = await Promise.all([unlinkPromise, addPromise]); - expect(unlinkPath).toInclude('inode-test.txt'); - expect(addPath).toInclude('inode-test.txt'); - console.log(`[test] Detected unlink + add events for delete+recreate`); + // Chokidar detects this as a change (atomic write pattern) + const [filePath] = await eventPromise; + expect(filePath).toInclude('inode-test.txt'); + console.log(`[test] Detected change event for delete+recreate (atomic handling)`); // Cleanup await fs.promises.unlink(testFile); + await delay(200); }); tap.test('should detect atomic write pattern (temp file + rename)', async () => { @@ -96,10 +104,10 @@ tap.test('should detect atomic write pattern (temp file + rename)', async () => // Create initial file await smartfile.memory.toFs('initial content', testFile); - await delay(200); + await delay(300); const changeObservable = await testSmartwatch.getObservableFor('change'); - const eventPromise = waitForEvent(changeObservable, 3000); + const eventPromise = waitForFileEvent(changeObservable, 'atomic-test.txt', 3000); // Atomic write: create temp file then rename await smartfile.memory.toFs('atomic content', tempFile); diff --git a/test/test.stress.ts b/test/test.stress.ts index 742e283..9849d9f 100644 --- a/test/test.stress.ts +++ b/test/test.stress.ts @@ -44,6 +44,8 @@ tap.test('setup: start watcher', async () => { testSmartwatch = new smartwatch.Smartwatch([`${TEST_DIR}/**/*.txt`]); await testSmartwatch.start(); expect(testSmartwatch.status).toEqual('watching'); + // Wait for chokidar to be ready + await delay(500); }); tap.test('STRESS: rapid file modifications', async () => { diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index c2cc14e..0eede44 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartwatch', - version: '6.2.5', + version: '6.3.0', description: 'A cross-runtime file watcher with glob pattern support for Node.js, Deno, and Bun.' } diff --git a/ts/watchers/interfaces.ts b/ts/watchers/interfaces.ts index cbb47ae..ea28fd5 100644 --- a/ts/watchers/interfaces.ts +++ b/ts/watchers/interfaces.ts @@ -28,6 +28,12 @@ export interface IWatcherOptions { followSymlinks: boolean; /** Debounce time in ms - events for the same file within this window are coalesced */ debounceMs: number; + /** Whether to wait for writes to stabilize before emitting events */ + awaitWriteFinish?: boolean; + /** How long file size must remain constant before emitting event (ms) */ + stabilityThreshold?: number; + /** How often to poll file size during write detection (ms) */ + pollInterval?: number; } /** diff --git a/ts/watchers/watcher.node.ts b/ts/watchers/watcher.node.ts index 7dd220a..78b8ef7 100644 --- a/ts/watchers/watcher.node.ts +++ b/ts/watchers/watcher.node.ts @@ -1,196 +1,83 @@ import * as fs from 'fs'; import * as path from 'path'; import * as smartrx from '@push.rocks/smartrx'; -import type { IWatcher, IWatcherOptions, IWatchEvent, TWatchEventType } from './interfaces.js'; - -// ============================================================================= -// Constants -// ============================================================================= - -/** Event type constants - inspired by chokidar's pattern */ -const EV = { - ADD: 'add', - CHANGE: 'change', - UNLINK: 'unlink', - ADD_DIR: 'addDir', - UNLINK_DIR: 'unlinkDir', - READY: 'ready', - ERROR: 'error', -} as const; - -/** Configuration constants */ -const CONFIG = { - MAX_RETRIES: 3, - INITIAL_RESTART_DELAY: 1000, - MAX_RESTART_DELAY: 30000, - HEALTH_CHECK_INTERVAL: 30000, - ATOMIC_DELAY: 100, - TEMP_FILE_DELAY: 50, -} as const; - -// ============================================================================= -// DirEntry Class - Elegant directory content tracking (inspired by chokidar) -// ============================================================================= - +import * as chokidar from 'chokidar'; +import type { IWatcher, IWatcherOptions, IWatchEvent } from './interfaces.js'; /** - * Tracks contents of a watched directory with proper disposal - */ -class DirEntry { - private _path: string; - private _items: Set; - private _inodes: Map; - - constructor(dirPath: string) { - this._path = dirPath; - this._items = new Set(); - this._inodes = new Map(); - } - - get path(): string { - return this._path; - } - - add(item: string, inode?: bigint): void { - if (item === '.' || item === '..') return; - this._items.add(item); - if (inode !== undefined) { - this._inodes.set(item, inode); - } - } - - remove(item: string): void { - this._items.delete(item); - this._inodes.delete(item); - } - - has(item: string): boolean { - return this._items.has(item); - } - - getInode(item: string): bigint | undefined { - return this._inodes.get(item); - } - - setInode(item: string, inode: bigint): void { - this._inodes.set(item, inode); - } - - getChildren(): string[] { - return [...this._items]; - } - - get size(): number { - return this._items.size; - } - - dispose(): void { - this._items.clear(); - this._inodes.clear(); - this._path = ''; - // Freeze to catch accidental use after disposal - Object.freeze(this); - } -} - -// ============================================================================= -// NodeWatcher Class -// ============================================================================= - -/** - * Node.js/Bun file watcher using native fs.watch API + * Node.js/Bun file watcher using chokidar * - * Architecture inspired by chokidar with additional robustness features: - * - Event deferral during initial scan - * - Event sequence tracking for debounce - * - Atomic write handling (unlink→add becomes change) - * - Inode tracking for delete+recreate detection - * - Health check monitoring - * - Auto-restart with exponential backoff + * Chokidar handles all the edge cases: + * - Atomic writes (temp file + rename) + * - Inode tracking + * - Cross-platform differences + * - Debouncing + * - Write stabilization */ export class NodeWatcher implements IWatcher { - // Core state - private watchers: Map = new Map(); - private watched: Map = new Map(); + private watcher: chokidar.FSWatcher | null = null; private _isWatching = false; - // Event stream public readonly events$ = new smartrx.rxjs.Subject(); - // Atomic write handling - pending unlinks that may become changes - private pendingUnlinks: Map = new Map(); - - // Debounce with event sequence tracking - private pendingEmits: Map; - }> = new Map(); - - // Restart management - private restartDelays: Map = new Map(); - private restartAttempts: Map = new Map(); - private restartAbortControllers: Map = new Map(); - private restartingPaths: Set = new Set(); - - // Health monitoring - private healthCheckInterval: NodeJS.Timeout | null = null; - private watchedInodes: Map = new Map(); - - // Initial scan state - private initialScanComplete = false; - private deferredEvents: Array<{ basePath: string; filename: string; eventType: string }> = []; - - // Closer registry - inspired by chokidar for clean resource management - private closers: Map void>> = new Map(); - constructor(private options: IWatcherOptions) {} get isWatching(): boolean { return this._isWatching; } - // =========================================================================== - // Public API - // =========================================================================== - async start(): Promise { if (this._isWatching) return; - console.log(`[smartwatch] Starting watcher for ${this.options.basePaths.length} base path(s)...`); + console.log(`[smartwatch] Starting chokidar watcher for ${this.options.basePaths.length} base path(s)...`); try { - // Reset state - this.initialScanComplete = false; - this.deferredEvents = []; + // Resolve all paths to absolute + const absolutePaths = this.options.basePaths.map(p => path.resolve(p)); - // Start watching each base path (events will be deferred) - for (const basePath of this.options.basePaths) { - await this.watchPath(basePath); - } + this.watcher = chokidar.watch(absolutePaths, { + persistent: true, + ignoreInitial: false, + followSymlinks: this.options.followSymlinks, + depth: this.options.depth, + atomic: true, // Handle atomic writes + awaitWriteFinish: this.options.awaitWriteFinish ? { + stabilityThreshold: this.options.stabilityThreshold || 300, + pollInterval: this.options.pollInterval || 100, + } : false, + }); + + // Wire up all events + this.watcher + .on('add', (filePath: string, stats?: fs.Stats) => { + this.safeEmit({ type: 'add', path: filePath, stats }); + }) + .on('change', (filePath: string, stats?: fs.Stats) => { + this.safeEmit({ type: 'change', path: filePath, stats }); + }) + .on('unlink', (filePath: string) => { + this.safeEmit({ type: 'unlink', path: filePath }); + }) + .on('addDir', (filePath: string, stats?: fs.Stats) => { + this.safeEmit({ type: 'addDir', path: filePath, stats }); + }) + .on('unlinkDir', (filePath: string) => { + this.safeEmit({ type: 'unlinkDir', path: filePath }); + }) + .on('error', (error: Error) => { + console.error('[smartwatch] Chokidar error:', error); + this.safeEmit({ type: 'error', path: '', error }); + }) + .on('ready', () => { + console.log('[smartwatch] Chokidar ready - initial scan complete'); + this.safeEmit({ type: 'ready', path: '' }); + }); this._isWatching = true; - this.startHealthCheck(); - - // Initial scan populates watched entries - for (const basePath of this.options.basePaths) { - await this.scanDirectory(basePath, 0); - } - - // Process deferred events - this.initialScanComplete = true; - if (this.deferredEvents.length > 0) { - console.log(`[smartwatch] Processing ${this.deferredEvents.length} deferred events`); - for (const event of this.deferredEvents) { - this.handleFsEvent(event.basePath, event.filename, event.eventType); - } - this.deferredEvents = []; - } - - this.safeEmit({ type: EV.READY, path: '' }); - console.log(`[smartwatch] Watcher started with ${this.watchers.size} active watcher(s)`); + console.log('[smartwatch] Watcher started'); } catch (error: any) { console.error('[smartwatch] Failed to start watcher:', error); - this.safeEmit({ type: EV.ERROR, path: '', error }); + this.safeEmit({ type: 'error', path: '', error }); throw error; } } @@ -198,58 +85,15 @@ export class NodeWatcher implements IWatcher { async stop(): Promise { console.log('[smartwatch] Stopping watcher...'); - // Cancel pending emits first (before flag changes) - for (const pending of this.pendingEmits.values()) { - clearTimeout(pending.timeout); + if (this.watcher) { + await this.watcher.close(); + this.watcher = null; } - this.pendingEmits.clear(); - // Cancel pending unlinks - for (const pending of this.pendingUnlinks.values()) { - clearTimeout(pending.timeout); - } - this.pendingUnlinks.clear(); - - // Now set flag this._isWatching = false; - - this.stopHealthCheck(); - - // Abort pending restarts - for (const [watchPath, controller] of this.restartAbortControllers) { - console.log(`[smartwatch] Aborting pending restart for: ${watchPath}`); - controller.abort(); - } - this.restartAbortControllers.clear(); - - // Close all watchers and run closers - for (const [watchPath, watcher] of this.watchers) { - console.log(`[smartwatch] Closing watcher for: ${watchPath}`); - watcher.close(); - this.runClosers(watchPath); - } - - // Clear all state - this.watchers.clear(); - this.watched.forEach(entry => entry.dispose()); - this.watched.clear(); - this.restartDelays.clear(); - this.restartAttempts.clear(); - this.watchedInodes.clear(); - this.restartingPaths.clear(); - this.closers.clear(); - - // Reset scan state - this.initialScanComplete = false; - this.deferredEvents = []; - console.log('[smartwatch] Watcher stopped'); } - // =========================================================================== - // Event Emission - // =========================================================================== - /** Safely emit an event, isolating subscriber errors */ private safeEmit(event: IWatchEvent): void { try { @@ -258,548 +102,4 @@ export class NodeWatcher implements IWatcher { console.error('[smartwatch] Subscriber threw error (isolated):', error); } } - - // =========================================================================== - // Closer Registry - Clean resource management - // =========================================================================== - - private addCloser(watchPath: string, closer: () => void): void { - let list = this.closers.get(watchPath); - if (!list) { - list = []; - this.closers.set(watchPath, list); - } - list.push(closer); - } - - private runClosers(watchPath: string): void { - const list = this.closers.get(watchPath); - if (list) { - list.forEach(closer => closer()); - this.closers.delete(watchPath); - } - } - - // =========================================================================== - // Directory Entry Management - // =========================================================================== - - private getWatchedDir(dirPath: string): DirEntry { - const resolved = path.resolve(dirPath); - let entry = this.watched.get(resolved); - if (!entry) { - entry = new DirEntry(resolved); - this.watched.set(resolved, entry); - } - return entry; - } - - private isTracked(filePath: string): boolean { - const dir = path.dirname(filePath); - const base = path.basename(filePath); - const entry = this.watched.get(path.resolve(dir)); - return entry?.has(base) ?? false; - } - - private trackFile(filePath: string, inode?: bigint): void { - const dir = path.dirname(filePath); - const base = path.basename(filePath); - this.getWatchedDir(dir).add(base, inode); - } - - private untrackFile(filePath: string): void { - const dir = path.dirname(filePath); - const base = path.basename(filePath); - const entry = this.watched.get(path.resolve(dir)); - entry?.remove(base); - } - - private getFileInode(filePath: string): bigint | undefined { - const dir = path.dirname(filePath); - const base = path.basename(filePath); - const entry = this.watched.get(path.resolve(dir)); - return entry?.getInode(base); - } - - // =========================================================================== - // Temp File Handling - // =========================================================================== - - private isTemporaryFile(filePath: string): boolean { - const basename = path.basename(filePath); - return ( - basename.includes('.tmp.') || - basename.endsWith('.swp') || - basename.endsWith('.swx') || - basename.endsWith('~') || - basename.startsWith('.#') - ); - } - - /** - * Extract real file path from temp file (Claude Code atomic writes) - * Pattern: file.ts.tmp.PID.TIMESTAMP -> file.ts - */ - private getTempFileTarget(tempPath: string): string | null { - const basename = path.basename(tempPath); - - // Claude Code: file.ts.tmp.PID.TIMESTAMP - const claudeMatch = basename.match(/^(.+)\.tmp\.\d+\.\d+$/); - if (claudeMatch) { - return path.join(path.dirname(tempPath), claudeMatch[1]); - } - - // Generic: file.ts.tmp.something - const genericMatch = basename.match(/^(.+)\.tmp\.[^.]+$/); - if (genericMatch) { - return path.join(path.dirname(tempPath), genericMatch[1]); - } - - return null; - } - - // =========================================================================== - // Watch Path Setup - // =========================================================================== - - private async watchPath(watchPath: string): Promise { - // Normalize path to absolute - critical for consistent lookups - watchPath = path.resolve(watchPath); - - try { - const stats = await this.statSafe(watchPath); - if (!stats?.isDirectory()) return; - - // Store inode for health check (fs.watch watches inode, not path!) - this.watchedInodes.set(watchPath, BigInt(stats.ino)); - - const watcher = fs.watch( - watchPath, - { recursive: true, persistent: true }, - (eventType, filename) => { - if (filename) { - this.handleFsEvent(watchPath, filename, eventType); - } - } - ); - - watcher.on('error', (error: NodeJS.ErrnoException) => { - console.error(`[smartwatch] FSWatcher error on ${watchPath}:`, error); - - if (error.code === 'ENOSPC') { - console.error('[smartwatch] CRITICAL: inotify watch limit exceeded!'); - console.error('[smartwatch] Fix: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p'); - } - - this.safeEmit({ type: EV.ERROR, path: watchPath, error }); - if (this._isWatching) { - this.restartWatcher(watchPath, error); - } - }); - - watcher.on('close', () => { - if (this._isWatching) { - console.warn(`[smartwatch] FSWatcher closed unexpectedly for ${watchPath}`); - this.restartWatcher(watchPath, new Error('Watcher closed unexpectedly')); - } - }); - - this.watchers.set(watchPath, watcher); - - // Register closer - this.addCloser(watchPath, () => { - try { watcher.close(); } catch {} - }); - - console.log(`[smartwatch] Started watching: ${watchPath}`); - } catch (error: any) { - console.error(`[smartwatch] Failed to watch path ${watchPath}:`, error); - this.safeEmit({ type: EV.ERROR, path: watchPath, error }); - } - } - - // =========================================================================== - // Event Handling - // =========================================================================== - - private handleFsEvent( - basePath: string, - filename: string, - eventType: 'rename' | 'change' | string - ): void { - // Guard against post-stop events - if (!this._isWatching) return; - - // Defer events until initial scan completes - if (!this.initialScanComplete) { - this.deferredEvents.push({ basePath, filename, eventType }); - return; - } - - // Normalize to absolute path - critical for consistent lookups - const fullPath = path.resolve(path.join(basePath, filename)); - - // Handle temp files from atomic writes - if (this.isTemporaryFile(fullPath)) { - console.log(`[smartwatch] Detected temp file event: ${filename}`); - const realPath = this.getTempFileTarget(fullPath); - if (realPath) { - console.log(`[smartwatch] Checking corresponding real file: ${realPath}`); - setTimeout(() => { - if (this._isWatching) { - this.handleFsEvent(basePath, path.relative(basePath, realPath), 'change'); - } - }, CONFIG.TEMP_FILE_DELAY); - } - return; - } - - // Track event sequence for intelligent debouncing - const existing = this.pendingEmits.get(fullPath); - if (existing) { - clearTimeout(existing.timeout); - existing.events.push(eventType as 'rename' | 'change'); - existing.timeout = setTimeout(() => { - const pending = this.pendingEmits.get(fullPath); - if (pending) { - this.pendingEmits.delete(fullPath); - this.emitFileEvent(fullPath, pending.events); - } - }, this.options.debounceMs); - } else { - const timeout = setTimeout(() => { - const pending = this.pendingEmits.get(fullPath); - if (pending) { - this.pendingEmits.delete(fullPath); - this.emitFileEvent(fullPath, pending.events); - } - }, this.options.debounceMs); - - this.pendingEmits.set(fullPath, { - timeout, - events: [eventType as 'rename' | 'change'], - }); - } - } - - /** - * Emit file event after debounce with atomic write handling - * - * Atomic write pattern (inspired by chokidar): - * - unlink event queued with delay - * - if add arrives for same path, transform to change - */ - private async emitFileEvent( - fullPath: string, - eventSequence: Array<'rename' | 'change'> - ): Promise { - try { - const stats = await this.statSafe(fullPath); - const wasTracked = this.isTracked(fullPath); - const previousInode = this.getFileInode(fullPath); - - // Analyze event sequence - const hasRename = eventSequence.includes('rename'); - const renameCount = eventSequence.filter(e => e === 'rename').length; - - if (eventSequence.length > 1) { - console.log(`[smartwatch] Processing event sequence for ${fullPath}: [${eventSequence.join(', ')}]`); - } - - if (stats) { - // File EXISTS - const currentInode = BigInt(stats.ino); - const inodeChanged = previousInode !== undefined && previousInode !== currentInode; - - if (stats.isDirectory()) { - if (!wasTracked) { - this.trackFile(fullPath); - this.safeEmit({ type: EV.ADD_DIR, path: fullPath, stats }); - } - } else { - // Update tracking - this.trackFile(fullPath, currentInode); - - // Check for pending unlink → transform to change (atomic write pattern) - const pendingUnlink = this.pendingUnlinks.get(fullPath); - if (pendingUnlink) { - clearTimeout(pendingUnlink.timeout); - this.pendingUnlinks.delete(fullPath); - console.log(`[smartwatch] Atomic write detected (unlink→add→change): ${fullPath}`); - this.safeEmit({ type: EV.CHANGE, path: fullPath, stats }); - return; - } - - if (!wasTracked) { - this.safeEmit({ type: EV.ADD, path: fullPath, stats }); - } else if (inodeChanged) { - console.log(`[smartwatch] File inode changed (delete+recreate): ${fullPath}`); - console.log(`[smartwatch] Previous inode: ${previousInode}, current: ${currentInode}`); - - if (renameCount >= 2) { - // Multiple renames with inode change = delete+recreate - this.safeEmit({ type: EV.UNLINK, path: fullPath }); - this.safeEmit({ type: EV.ADD, path: fullPath, stats }); - } else { - // Single rename with inode change = atomic save - this.safeEmit({ type: EV.CHANGE, path: fullPath, stats }); - } - } else { - // Debounce already handles rapid events - no extra throttle needed - this.safeEmit({ type: EV.CHANGE, path: fullPath, stats }); - } - } - } else { - // File does NOT exist - handle unlink - const wasDir = this.isKnownDirectory(fullPath); - - if (wasTracked) { - this.untrackFile(fullPath); - - if (renameCount >= 2 && !wasDir) { - // Rapid create+delete - console.log(`[smartwatch] File created and deleted rapidly: ${fullPath}`); - this.safeEmit({ type: EV.ADD, path: fullPath }); - this.safeEmit({ type: EV.UNLINK, path: fullPath }); - } else { - // Queue unlink with delay for atomic write detection - this.queueUnlink(fullPath, wasDir); - } - } else { - if (renameCount >= 2) { - console.log(`[smartwatch] Untracked file created and deleted: ${fullPath}`); - this.safeEmit({ type: EV.ADD, path: fullPath }); - this.safeEmit({ type: EV.UNLINK, path: fullPath }); - } else if (hasRename) { - console.log(`[smartwatch] Untracked file deleted: ${fullPath}`); - this.queueUnlink(fullPath, false); - } - } - } - } catch (error: any) { - this.safeEmit({ type: EV.ERROR, path: fullPath, error }); - } - } - - /** - * Queue an unlink event with delay for atomic write detection - * If add event arrives within delay, unlink is cancelled and change is emitted - */ - private queueUnlink(fullPath: string, isDir: boolean): void { - const event: IWatchEvent = { - type: isDir ? EV.UNLINK_DIR : EV.UNLINK, - path: fullPath, - }; - - const timeout = setTimeout(() => { - const pending = this.pendingUnlinks.get(fullPath); - if (pending) { - this.pendingUnlinks.delete(fullPath); - this.safeEmit(pending.event); - } - }, CONFIG.ATOMIC_DELAY); - - this.pendingUnlinks.set(fullPath, { timeout, event }); - } - - // =========================================================================== - // Directory Scanning - // =========================================================================== - - private async scanDirectory(dirPath: string, depth: number): Promise { - // Normalize path to absolute - critical for consistent lookups - dirPath = path.resolve(dirPath); - - if (depth > this.options.depth) return; - - try { - const entries = await fs.promises.readdir(dirPath, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = path.join(dirPath, entry.name); - - if (this.isTemporaryFile(fullPath)) continue; - - const stats = await this.statSafe(fullPath); - if (!stats) continue; - - if (entry.isDirectory()) { - this.trackFile(fullPath); - this.safeEmit({ type: EV.ADD_DIR, path: fullPath, stats }); - await this.scanDirectory(fullPath, depth + 1); - } else if (entry.isFile()) { - this.trackFile(fullPath, BigInt(stats.ino)); - this.safeEmit({ type: EV.ADD, path: fullPath, stats }); - } - } - } catch (error: any) { - if (error.code !== 'ENOENT' && error.code !== 'EACCES') { - this.safeEmit({ type: EV.ERROR, path: dirPath, error }); - } - } - } - - // =========================================================================== - // Health Check & Auto-Restart - // =========================================================================== - - private startHealthCheck(): void { - console.log('[smartwatch] Starting health check (every 30s)'); - this.healthCheckInterval = setInterval(async () => { - console.log(`[smartwatch] Health check: ${this.watchers.size} watchers active`); - - for (const [basePath] of this.watchers) { - try { - const stats = await fs.promises.stat(basePath); - const currentInode = BigInt(stats.ino); - const previousInode = this.watchedInodes.get(basePath); - - if (previousInode !== undefined && currentInode !== previousInode) { - console.warn(`[smartwatch] Inode changed for ${basePath}: ${previousInode} -> ${currentInode}`); - console.warn('[smartwatch] fs.watch watches inode, not path - restarting watcher'); - this.restartWatcher(basePath, new Error('Inode changed - directory was replaced')); - } - } catch (error: any) { - if (error.code === 'ENOENT') { - console.error(`[smartwatch] Health check failed: ${basePath} no longer exists`); - this.restartWatcher(basePath, new Error('Watched path disappeared')); - } else if (error.code === 'ENOSPC') { - console.error('[smartwatch] ENOSPC: inotify watch limit exceeded!'); - console.error('[smartwatch] Fix: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p'); - this.safeEmit({ type: EV.ERROR, path: basePath, error }); - // Trigger restart - watcher may be broken after ENOSPC - this.restartWatcher(basePath, error); - } else { - console.error(`[smartwatch] Health check error for ${basePath}:`, error); - } - } - } - }, CONFIG.HEALTH_CHECK_INTERVAL); - } - - private stopHealthCheck(): void { - if (this.healthCheckInterval) { - clearInterval(this.healthCheckInterval); - this.healthCheckInterval = null; - console.log('[smartwatch] Stopped health check'); - } - } - - private async restartWatcher(basePath: string, error: Error): Promise { - // Guard against concurrent restarts - if (this.restartingPaths.has(basePath)) { - console.log(`[smartwatch] Restart already in progress for ${basePath}, skipping`); - return; - } - this.restartingPaths.add(basePath); - - try { - const attempts = (this.restartAttempts.get(basePath) || 0) + 1; - this.restartAttempts.set(basePath, attempts); - - console.log(`[smartwatch] Watcher error for ${basePath}: ${error.message}`); - console.log(`[smartwatch] Restart attempt ${attempts}/${CONFIG.MAX_RETRIES}`); - - if (attempts > CONFIG.MAX_RETRIES) { - console.error(`[smartwatch] Max retries exceeded for ${basePath}, giving up`); - this.safeEmit({ - type: EV.ERROR, - path: basePath, - error: new Error(`Max restart retries (${CONFIG.MAX_RETRIES}) exceeded`), - }); - return; - } - - // Close old watcher - const oldWatcher = this.watchers.get(basePath); - if (oldWatcher) { - try { oldWatcher.close(); } catch {} - this.watchers.delete(basePath); - } - - // Clear pending unlinks for this base path (prevent stale events) - for (const [unlinkedPath, pending] of this.pendingUnlinks) { - if (unlinkedPath.startsWith(basePath)) { - clearTimeout(pending.timeout); - this.pendingUnlinks.delete(unlinkedPath); - } - } - - // Clear stale DirEntry data (will be repopulated by rescan) - for (const [dirPath, entry] of this.watched) { - if (dirPath === basePath || dirPath.startsWith(basePath + path.sep)) { - entry.dispose(); - this.watched.delete(dirPath); - } - } - - // Exponential backoff with abort support - const delay = this.restartDelays.get(basePath) || CONFIG.INITIAL_RESTART_DELAY; - console.log(`[smartwatch] Waiting ${delay}ms before restart...`); - - const abortController = new AbortController(); - this.restartAbortControllers.set(basePath, abortController); - - try { - await new Promise((resolve, reject) => { - const timeout = setTimeout(resolve, delay); - abortController.signal.addEventListener('abort', () => { - clearTimeout(timeout); - reject(new Error('Restart aborted by stop()')); - }); - }); - } catch { - console.log(`[smartwatch] Restart aborted for ${basePath}`); - return; - } finally { - this.restartAbortControllers.delete(basePath); - } - - if (!this._isWatching) { - console.log('[smartwatch] Watcher stopped during restart delay, aborting'); - return; - } - - this.restartDelays.set(basePath, Math.min(delay * 2, CONFIG.MAX_RESTART_DELAY)); - - try { - await this.watchPath(basePath); - // Rescan to catch files created during restart window - await this.scanDirectory(basePath, 0); - console.log(`[smartwatch] Successfully restarted watcher for ${basePath}`); - this.restartDelays.set(basePath, CONFIG.INITIAL_RESTART_DELAY); - this.restartAttempts.set(basePath, 0); - } catch (restartError) { - console.error(`[smartwatch] Restart failed for ${basePath}:`, restartError); - this.restartingPaths.delete(basePath); - this.restartWatcher(basePath, restartError as Error); - return; - } - } finally { - this.restartingPaths.delete(basePath); - } - } - - // =========================================================================== - // Utilities - // =========================================================================== - - private async statSafe(filePath: string): Promise { - try { - return await (this.options.followSymlinks - ? fs.promises.stat(filePath) - : fs.promises.lstat(filePath)); - } catch (error: any) { - if (error.code === 'ENOENT' || error.code === 'ENOTDIR') { - return null; - } - console.warn(`[smartwatch] statSafe warning for ${filePath}: ${error.code} - ${error.message}`); - return null; - } - } - - private isKnownDirectory(filePath: string): boolean { - const resolved = path.resolve(filePath); - return this.watched.has(resolved); - } }