import * as plugins from '../plugins.js'; export class ProjectContext { public static async fromDir(dirArg: string) {} // INSTANCE public projectDir: string; private tokenCount: number = 0; private contextString: string = ''; constructor(projectDirArg: string) { this.projectDir = projectDirArg; } public async gatherFiles() { const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath( plugins.path.join(this.projectDir, 'package.json'), this.projectDir, ); const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath( plugins.path.join(this.projectDir, 'readme.md'), this.projectDir, ); const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath( plugins.path.join(this.projectDir, 'readme.hints.md'), this.projectDir, ); const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath( plugins.path.join(this.projectDir, 'npmextra.json'), this.projectDir, ); const smartfilesMod = await plugins.smartfile.fs.fileTreeToObject( this.projectDir, 'ts*/**/*.ts', ); const smartfilesTest = await plugins.smartfile.fs.fileTreeToObject( this.projectDir, 'test/**/*.ts', ); return { smartfilePackageJSON, smartfilesReadme, smartfilesReadmeHints, smartfilesNpmextraJSON, smartfilesMod, smartfilesTest, }; } public async convertFilesToContext(filesArg: plugins.smartfile.SmartFile[]) { filesArg.map((fileArg) => { // console.log(` -> ${fileArg.relative}`); }); return filesArg .map((smartfile) => { return ` ====== START OF FILE ${smartfile.relative} ====== ${smartfile.contents.toString()} ====== END OF FILE ${smartfile.relative} ====== `; }) .join('\n'); } /** * Calculate the token count for a string using the GPT tokenizer * @param text The text to count tokens for * @param model The model to use for token counting (default: gpt-3.5-turbo) * @returns The number of tokens in the text */ public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number { try { // Use the gpt-tokenizer library to count tokens const tokens = plugins.gptTokenizer.encode(text); return tokens.length; } catch (error) { console.error('Error counting tokens:', error); // Provide a rough estimate (4 chars per token) if tokenization fails return Math.ceil(text.length / 4); } } private async buildContext(dirArg: string) { const files = await this.gatherFiles(); let context = await this.convertFilesToContext([ files.smartfilePackageJSON, files.smartfilesReadme, files.smartfilesReadmeHints, files.smartfilesNpmextraJSON, ...files.smartfilesMod, ...files.smartfilesTest, ]); // Count tokens in the context this.contextString = context; this.tokenCount = this.countTokens(context); // console.log(context); return context; } /** * Get the token count for the current context * @returns The number of tokens in the context */ public getTokenCount(): number { return this.tokenCount; } /** * Get both the context string and its token count * @returns An object containing the context string and token count */ public getContextWithTokenCount(): { context: string; tokenCount: number } { return { context: this.contextString, tokenCount: this.tokenCount }; } public async update() { const result = await this.buildContext(this.projectDir); return result; } }