import * as plugins from '../plugins.js'; import * as fs from 'fs'; import type { IFileMetadata, IFileInfo } from './types.js'; /** * LazyFileLoader handles efficient file loading by: * - Scanning files for metadata without loading contents * - Providing fast file size and token estimates * - Loading contents only when requested * - Parallel loading of selected files */ export class LazyFileLoader { private projectRoot: string; private metadataCache: Map = new Map(); /** * Creates a new LazyFileLoader * @param projectRoot - Root directory of the project */ constructor(projectRoot: string) { this.projectRoot = projectRoot; } /** * Scans files in given globs and creates metadata without loading contents * @param globs - File patterns to scan (e.g., ['ts/**\/*.ts', 'test/**\/*.ts']) * @returns Array of file metadata */ public async scanFiles(globs: string[]): Promise { const metadata: IFileMetadata[] = []; for (const globPattern of globs) { try { const smartFiles = await plugins.smartfile.fs.fileTreeToObject(this.projectRoot, globPattern); const fileArray = Array.isArray(smartFiles) ? smartFiles : [smartFiles]; for (const smartFile of fileArray) { try { const meta = await this.getMetadata(smartFile.path); metadata.push(meta); } catch (error) { // Skip files that can't be read console.warn(`Failed to get metadata for ${smartFile.path}:`, error.message); } } } catch (error) { // Skip patterns that don't match any files console.warn(`No files found for pattern ${globPattern}`); } } return metadata; } /** * Gets metadata for a single file without loading contents * @param filePath - Absolute path to the file * @returns File metadata */ public async getMetadata(filePath: string): Promise { // Check cache first if (this.metadataCache.has(filePath)) { const cached = this.metadataCache.get(filePath)!; const currentStats = await fs.promises.stat(filePath); // Return cached if file hasn't changed if (cached.mtime === Math.floor(currentStats.mtimeMs)) { return cached; } } // Get file stats const stats = await fs.promises.stat(filePath); const relativePath = plugins.path.relative(this.projectRoot, filePath); // Estimate tokens: rough estimate of ~4 characters per token // This is faster than reading and tokenizing the entire file const estimatedTokens = Math.ceil(stats.size / 4); const metadata: IFileMetadata = { path: filePath, relativePath, size: stats.size, mtime: Math.floor(stats.mtimeMs), estimatedTokens, }; // Cache the metadata this.metadataCache.set(filePath, metadata); return metadata; } /** * Loads file contents for selected files in parallel * @param metadata - Array of file metadata to load * @param tokenizer - Function to calculate accurate token count * @returns Array of complete file info with contents */ public async loadFiles( metadata: IFileMetadata[], tokenizer: (content: string) => number ): Promise { // Load files in parallel const loadPromises = metadata.map(async (meta) => { try { const contents = await plugins.smartfile.fs.toStringSync(meta.path); const tokenCount = tokenizer(contents); const fileInfo: IFileInfo = { path: meta.path, relativePath: meta.relativePath, contents, tokenCount, importanceScore: meta.importanceScore, }; return fileInfo; } catch (error) { console.warn(`Failed to load file ${meta.path}:`, error.message); return null; } }); // Wait for all loads to complete and filter out failures const results = await Promise.all(loadPromises); return results.filter((r): r is IFileInfo => r !== null); } /** * Loads a single file with contents * @param filePath - Absolute path to the file * @param tokenizer - Function to calculate accurate token count * @returns Complete file info with contents */ public async loadFile( filePath: string, tokenizer: (content: string) => number ): Promise { const meta = await this.getMetadata(filePath); const contents = await plugins.smartfile.fs.toStringSync(filePath); const tokenCount = tokenizer(contents); const relativePath = plugins.path.relative(this.projectRoot, filePath); return { path: filePath, relativePath, contents, tokenCount, importanceScore: meta.importanceScore, }; } /** * Updates importance scores for metadata entries * @param scores - Map of file paths to importance scores */ public updateImportanceScores(scores: Map): void { for (const [path, score] of scores) { const meta = this.metadataCache.get(path); if (meta) { meta.importanceScore = score; } } } /** * Clears the metadata cache */ public clearCache(): void { this.metadataCache.clear(); } /** * Gets total estimated tokens for all cached metadata */ public getTotalEstimatedTokens(): number { let total = 0; for (const meta of this.metadataCache.values()) { total += meta.estimatedTokens; } return total; } /** * Gets cached metadata entries */ public getCachedMetadata(): IFileMetadata[] { return Array.from(this.metadataCache.values()); } }