feat(context): Introduce smart context system: analyzer, lazy loader, cache and README/docs improvements

ts/context/lazy-file-loader.ts

@@ -0,0 +1,191 @@
+import * as plugins from '../plugins.js';
+import * as fs from 'fs';
+import type { IFileMetadata, IFileInfo } from './types.js';
+
+/**
+ * LazyFileLoader handles efficient file loading by:
+ * - Scanning files for metadata without loading contents
+ * - Providing fast file size and token estimates
+ * - Loading contents only when requested
+ * - Parallel loading of selected files
+ */
+export class LazyFileLoader {
+  private projectRoot: string;
+  private metadataCache: Map<string, IFileMetadata> = new Map();
+
+  /**
+   * Creates a new LazyFileLoader
+   * @param projectRoot - Root directory of the project
+   */
+  constructor(projectRoot: string) {
+    this.projectRoot = projectRoot;
+  }
+
+  /**
+   * Scans files in given globs and creates metadata without loading contents
+   * @param globs - File patterns to scan (e.g., ['ts/**\/*.ts', 'test/**\/*.ts'])
+   * @returns Array of file metadata
+   */
+  public async scanFiles(globs: string[]): Promise<IFileMetadata[]> {
+    const metadata: IFileMetadata[] = [];
+
+    for (const globPattern of globs) {
+      try {
+        const smartFiles = await plugins.smartfile.fs.fileTreeToObject(this.projectRoot, globPattern);
+        const fileArray = Array.isArray(smartFiles) ? smartFiles : [smartFiles];
+
+        for (const smartFile of fileArray) {
+          try {
+            const meta = await this.getMetadata(smartFile.path);
+            metadata.push(meta);
+          } catch (error) {
+            // Skip files that can't be read
+            console.warn(`Failed to get metadata for ${smartFile.path}:`, error.message);
+          }
+        }
+      } catch (error) {
+        // Skip patterns that don't match any files
+        console.warn(`No files found for pattern ${globPattern}`);
+      }
+    }
+
+    return metadata;
+  }
+
+  /**
+   * Gets metadata for a single file without loading contents
+   * @param filePath - Absolute path to the file
+   * @returns File metadata
+   */
+  public async getMetadata(filePath: string): Promise<IFileMetadata> {
+    // Check cache first
+    if (this.metadataCache.has(filePath)) {
+      const cached = this.metadataCache.get(filePath)!;
+      const currentStats = await fs.promises.stat(filePath);
+
+      // Return cached if file hasn't changed
+      if (cached.mtime === Math.floor(currentStats.mtimeMs)) {
+        return cached;
+      }
+    }
+
+    // Get file stats
+    const stats = await fs.promises.stat(filePath);
+    const relativePath = plugins.path.relative(this.projectRoot, filePath);
+
+    // Estimate tokens: rough estimate of ~4 characters per token
+    // This is faster than reading and tokenizing the entire file
+    const estimatedTokens = Math.ceil(stats.size / 4);
+
+    const metadata: IFileMetadata = {
+      path: filePath,
+      relativePath,
+      size: stats.size,
+      mtime: Math.floor(stats.mtimeMs),
+      estimatedTokens,
+    };
+
+    // Cache the metadata
+    this.metadataCache.set(filePath, metadata);
+
+    return metadata;
+  }
+
+  /**
+   * Loads file contents for selected files in parallel
+   * @param metadata - Array of file metadata to load
+   * @param tokenizer - Function to calculate accurate token count
+   * @returns Array of complete file info with contents
+   */
+  public async loadFiles(
+    metadata: IFileMetadata[],
+    tokenizer: (content: string) => number
+  ): Promise<IFileInfo[]> {
+    // Load files in parallel
+    const loadPromises = metadata.map(async (meta) => {
+      try {
+        const contents = await plugins.smartfile.fs.toStringSync(meta.path);
+        const tokenCount = tokenizer(contents);
+
+        const fileInfo: IFileInfo = {
+          path: meta.path,
+          relativePath: meta.relativePath,
+          contents,
+          tokenCount,
+          importanceScore: meta.importanceScore,
+        };
+
+        return fileInfo;
+      } catch (error) {
+        console.warn(`Failed to load file ${meta.path}:`, error.message);
+        return null;
+      }
+    });
+
+    // Wait for all loads to complete and filter out failures
+    const results = await Promise.all(loadPromises);
+    return results.filter((r): r is IFileInfo => r !== null);
+  }
+
+  /**
+   * Loads a single file with contents
+   * @param filePath - Absolute path to the file
+   * @param tokenizer - Function to calculate accurate token count
+   * @returns Complete file info with contents
+   */
+  public async loadFile(
+    filePath: string,
+    tokenizer: (content: string) => number
+  ): Promise<IFileInfo> {
+    const meta = await this.getMetadata(filePath);
+    const contents = await plugins.smartfile.fs.toStringSync(filePath);
+    const tokenCount = tokenizer(contents);
+    const relativePath = plugins.path.relative(this.projectRoot, filePath);
+
+    return {
+      path: filePath,
+      relativePath,
+      contents,
+      tokenCount,
+      importanceScore: meta.importanceScore,
+    };
+  }
+
+  /**
+   * Updates importance scores for metadata entries
+   * @param scores - Map of file paths to importance scores
+   */
+  public updateImportanceScores(scores: Map<string, number>): void {
+    for (const [path, score] of scores) {
+      const meta = this.metadataCache.get(path);
+      if (meta) {
+        meta.importanceScore = score;
+      }
+    }
+  }
+
+  /**
+   * Clears the metadata cache
+   */
+  public clearCache(): void {
+    this.metadataCache.clear();
+  }
+
+  /**
+   * Gets total estimated tokens for all cached metadata
+   */
+  public getTotalEstimatedTokens(): number {
+    let total = 0;
+    for (const meta of this.metadataCache.values()) {
+      total += meta.estimatedTokens;
+    }
+    return total;
+  }
+
+  /**
+   * Gets cached metadata entries
+   */
+  public getCachedMetadata(): IFileMetadata[] {
+    return Array.from(this.metadataCache.values());
+  }
+}