feat(docs): Update project metadata and documentation to reflect comprehensive AI-enhanced features and improved installation and usage instructions

ts/aidocs_classes/commit.ts

@@ -31,15 +31,27 @@ export class Commit {
       'pnpm-lock.yaml',
       'package-lock.json',
     ]);
-    const projectContext = new ProjectContext(this.projectDir);
-    let contextString = await projectContext.update();
-    contextString = `
-${contextString}
-
-Below is the diff of the uncommitted changes. If nothing is changed, there are no changes:
-
-${diffStringArray[0] ? diffStringArray.join('\n\n') : 'No changes.'}
-    `;
+    // Use the new TaskContextFactory for optimized context
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    await taskContextFactory.initialize();
+    
+    // Generate context specifically for commit task
+    const contextResult = await taskContextFactory.createContextForCommit(
+      diffStringArray[0] ? diffStringArray.join('\n\n') : 'No changes.'
+    );
+    
+    // Get the optimized context string
+    let contextString = contextResult.context;
+    
+    // Log token usage statistics
+    console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
+    
+    // Check for token overflow against model limits
+    const MODEL_TOKEN_LIMIT = 200000; // o4-mini
+    if (contextResult.tokenCount > MODEL_TOKEN_LIMIT * 0.9) {
+      console.log(`⚠️ Warning: Context size (${contextResult.tokenCount} tokens) is close to or exceeds model limit (${MODEL_TOKEN_LIMIT} tokens).`);
+      console.log(`The model may not be able to process all information effectively.`);
+    }
 
     let result = await this.aiDocsRef.openaiInstance.chat({
       systemMessage: `

ts/aidocs_classes/description.ts

@@ -18,9 +18,16 @@ export class Description {
   }
 
   public async build() {
-    // we can now assemble the directory structure.
-    const projectContext = new ProjectContext(this.projectDir);
-    const contextString = await projectContext.update();
+    // Use the new TaskContextFactory for optimized context
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    await taskContextFactory.initialize();
+    
+    // Generate context specifically for description task
+    const contextResult = await taskContextFactory.createContextForDescription();
+    const contextString = contextResult.context;
+    
+    // Log token usage statistics
+    console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
 
     let result = await this.aiDocsRef.openaiInstance.chat({
       systemMessage: `
@@ -48,7 +55,11 @@ Don't wrap the JSON in three ticks json!!!
       result.message.replace('```json', '').replace('```', ''),
     );
 
-    const npmextraJson = (await projectContext.gatherFiles()).smartfilesNpmextraJSON;
+    // Create a standard ProjectContext instance for file operations
+    const projectContext = new ProjectContext(this.projectDir);
+    const files = await projectContext.gatherFiles();
+    
+    const npmextraJson = files.smartfilesNpmextraJSON;
     const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString());
 
     npmextraJsonContent.gitzone.module.description = resultObject.description;
@@ -58,7 +69,7 @@ Don't wrap the JSON in three ticks json!!!
     await npmextraJson.write();
 
     // do the same with packageJson
-    const packageJson = (await projectContext.gatherFiles()).smartfilePackageJSON;
+    const packageJson = files.smartfilePackageJSON;
     const packageJsonContent = JSON.parse(packageJson.contents.toString());
     packageJsonContent.description = resultObject.description;
     packageJsonContent.keywords = resultObject.keywords;

ts/aidocs_classes/projectcontext.ts

@@ -5,6 +5,8 @@ export class ProjectContext {
 
   // INSTANCE
   public projectDir: string;
+  private tokenCount: number = 0;
+  private contextString: string = '';
 
   constructor(projectDirArg: string) {
     this.projectDir = projectDirArg;
@@ -63,6 +65,24 @@ ${smartfile.contents.toString()}
       .join('\n');
   }
 
+  /**
+   * Calculate the token count for a string using the GPT tokenizer
+   * @param text The text to count tokens for
+   * @param model The model to use for token counting (default: gpt-3.5-turbo)
+   * @returns The number of tokens in the text
+   */
+  public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
+    try {
+      // Use the gpt-tokenizer library to count tokens
+      const tokens = plugins.gptTokenizer.encode(text);
+      return tokens.length;
+    } catch (error) {
+      console.error('Error counting tokens:', error);
+      // Provide a rough estimate (4 chars per token) if tokenization fails
+      return Math.ceil(text.length / 4);
+    }
+  }
+
   private async buildContext(dirArg: string) {
     const files = await this.gatherFiles();
     let context = await this.convertFilesToContext([
@@ -73,10 +93,33 @@ ${smartfile.contents.toString()}
       ...files.smartfilesMod,
       ...files.smartfilesTest,
     ]);
+    // Count tokens in the context
+    this.contextString = context;
+    this.tokenCount = this.countTokens(context);
+    
     // console.log(context);
     return context;
   }
 
+  /**
+   * Get the token count for the current context
+   * @returns The number of tokens in the context
+   */
+  public getTokenCount(): number {
+    return this.tokenCount;
+  }
+
+  /**
+   * Get both the context string and its token count
+   * @returns An object containing the context string and token count
+   */
+  public getContextWithTokenCount(): { context: string; tokenCount: number } {
+    return {
+      context: this.contextString,
+      tokenCount: this.tokenCount
+    };
+  }
+
   public async update() {
     const result = await this.buildContext(this.projectDir);
     return result;

ts/aidocs_classes/readme.ts

@@ -17,11 +17,19 @@ export class Readme {
   public async build() {
     let finalReadmeString = ``;
 
-    // we can now assemble the directory structure.
-    const projectContext = new ProjectContext(this.projectDir);
-    const contextString = await projectContext.update();
+    // Use the new TaskContextFactory for optimized context
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    await taskContextFactory.initialize();
+    
+    // Generate context specifically for readme task
+    const contextResult = await taskContextFactory.createContextForReadme();
+    const contextString = contextResult.context;
+    
+    // Log token usage statistics
+    console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
 
     // lets first check legal before introducung any cost
+    const projectContext = new ProjectContext(this.projectDir);
     const npmExtraJson = JSON.parse(
       (await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString()
     );