tsdoc/ts/context/lazy-file-loader.ts

import * as plugins from '../plugins.js';
import * as fs from 'fs';
import type { IFileMetadata, IFileInfo } from './types.js';

/**
 * LazyFileLoader handles efficient file loading by:
 * - Scanning files for metadata without loading contents
 * - Providing fast file size and token estimates
 * - Loading contents only when requested
 * - Parallel loading of selected files
 */
export class LazyFileLoader {
  private projectRoot: string;
  private metadataCache: Map<string, IFileMetadata> = new Map();

  /**
   * Creates a new LazyFileLoader
   * @param projectRoot - Root directory of the project
   */
  constructor(projectRoot: string) {
    this.projectRoot = projectRoot;
  }

  /**
   * Scans files in given globs and creates metadata without loading contents
   * @param globs - File patterns to scan (e.g., ['ts/**\/*.ts', 'test/**\/*.ts'])
   * @returns Array of file metadata
   */
  public async scanFiles(globs: string[]): Promise<IFileMetadata[]> {
    const metadata: IFileMetadata[] = [];

    for (const globPattern of globs) {
      try {
        const smartFiles = await plugins.smartfile.fs.fileTreeToObject(this.projectRoot, globPattern);
        const fileArray = Array.isArray(smartFiles) ? smartFiles : [smartFiles];

        for (const smartFile of fileArray) {
          try {
            const meta = await this.getMetadata(smartFile.path);
            metadata.push(meta);
          } catch (error) {
            // Skip files that can't be read
            console.warn(`Failed to get metadata for ${smartFile.path}:`, error.message);
          }
        }
      } catch (error) {
        // Skip patterns that don't match any files
        console.warn(`No files found for pattern ${globPattern}`);
      }
    }

    return metadata;
  }

  /**
   * Gets metadata for a single file without loading contents
   * @param filePath - Absolute path to the file
   * @returns File metadata
   */
  public async getMetadata(filePath: string): Promise<IFileMetadata> {
    // Check cache first
    if (this.metadataCache.has(filePath)) {
      const cached = this.metadataCache.get(filePath)!;
      const currentStats = await fs.promises.stat(filePath);

      // Return cached if file hasn't changed
      if (cached.mtime === Math.floor(currentStats.mtimeMs)) {
        return cached;
      }
    }

    // Get file stats
    const stats = await fs.promises.stat(filePath);
    const relativePath = plugins.path.relative(this.projectRoot, filePath);

    // Estimate tokens: rough estimate of ~4 characters per token
    // This is faster than reading and tokenizing the entire file
    const estimatedTokens = Math.ceil(stats.size / 4);

    const metadata: IFileMetadata = {
      path: filePath,
      relativePath,
      size: stats.size,
      mtime: Math.floor(stats.mtimeMs),
      estimatedTokens,
    };

    // Cache the metadata
    this.metadataCache.set(filePath, metadata);

    return metadata;
  }

  /**
   * Loads file contents for selected files in parallel
   * @param metadata - Array of file metadata to load
   * @param tokenizer - Function to calculate accurate token count
   * @returns Array of complete file info with contents
   */
  public async loadFiles(
    metadata: IFileMetadata[],
    tokenizer: (content: string) => number
  ): Promise<IFileInfo[]> {
    // Load files in parallel
    const loadPromises = metadata.map(async (meta) => {
      try {
        const contents = await plugins.smartfile.fs.toStringSync(meta.path);
        const tokenCount = tokenizer(contents);

        const fileInfo: IFileInfo = {
          path: meta.path,
          relativePath: meta.relativePath,
          contents,
          tokenCount,
          importanceScore: meta.importanceScore,
        };

        return fileInfo;
      } catch (error) {
        console.warn(`Failed to load file ${meta.path}:`, error.message);
        return null;
      }
    });

    // Wait for all loads to complete and filter out failures
    const results = await Promise.all(loadPromises);
    return results.filter((r): r is IFileInfo => r !== null);
  }

  /**
   * Loads a single file with contents
   * @param filePath - Absolute path to the file
   * @param tokenizer - Function to calculate accurate token count
   * @returns Complete file info with contents
   */
  public async loadFile(
    filePath: string,
    tokenizer: (content: string) => number
  ): Promise<IFileInfo> {
    const meta = await this.getMetadata(filePath);
    const contents = await plugins.smartfile.fs.toStringSync(filePath);
    const tokenCount = tokenizer(contents);
    const relativePath = plugins.path.relative(this.projectRoot, filePath);

    return {
      path: filePath,
      relativePath,
      contents,
      tokenCount,
      importanceScore: meta.importanceScore,
    };
  }

  /**
   * Updates importance scores for metadata entries
   * @param scores - Map of file paths to importance scores
   */
  public updateImportanceScores(scores: Map<string, number>): void {
    for (const [path, score] of scores) {
      const meta = this.metadataCache.get(path);
      if (meta) {
        meta.importanceScore = score;
      }
    }
  }

  /**
   * Clears the metadata cache
   */
  public clearCache(): void {
    this.metadataCache.clear();
  }

  /**
   * Gets total estimated tokens for all cached metadata
   */
  public getTotalEstimatedTokens(): number {
    let total = 0;
    for (const meta of this.metadataCache.values()) {
      total += meta.estimatedTokens;
    }
    return total;
  }

  /**
   * Gets cached metadata entries
   */
  public getCachedMetadata(): IFileMetadata[] {
    return Array.from(this.metadataCache.values());
  }
}