feat(docs): Update project metadata and documentation to reflect comprehensive AI-enhanced features and improved installation and usage instructions

ts/aidocs_classes/projectcontext.ts

@@ -5,6 +5,8 @@ export class ProjectContext {
 
   // INSTANCE
   public projectDir: string;
+  private tokenCount: number = 0;
+  private contextString: string = '';
 
   constructor(projectDirArg: string) {
     this.projectDir = projectDirArg;
@@ -63,6 +65,24 @@ ${smartfile.contents.toString()}
       .join('\n');
   }
 
+  /**
+   * Calculate the token count for a string using the GPT tokenizer
+   * @param text The text to count tokens for
+   * @param model The model to use for token counting (default: gpt-3.5-turbo)
+   * @returns The number of tokens in the text
+   */
+  public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
+    try {
+      // Use the gpt-tokenizer library to count tokens
+      const tokens = plugins.gptTokenizer.encode(text);
+      return tokens.length;
+    } catch (error) {
+      console.error('Error counting tokens:', error);
+      // Provide a rough estimate (4 chars per token) if tokenization fails
+      return Math.ceil(text.length / 4);
+    }
+  }
+
   private async buildContext(dirArg: string) {
     const files = await this.gatherFiles();
     let context = await this.convertFilesToContext([
@@ -73,10 +93,33 @@ ${smartfile.contents.toString()}
       ...files.smartfilesMod,
       ...files.smartfilesTest,
     ]);
+    // Count tokens in the context
+    this.contextString = context;
+    this.tokenCount = this.countTokens(context);
+    
     // console.log(context);
     return context;
   }
 
+  /**
+   * Get the token count for the current context
+   * @returns The number of tokens in the context
+   */
+  public getTokenCount(): number {
+    return this.tokenCount;
+  }
+
+  /**
+   * Get both the context string and its token count
+   * @returns An object containing the context string and token count
+   */
+  public getContextWithTokenCount(): { context: string; tokenCount: number } {
+    return {
+      context: this.contextString,
+      tokenCount: this.tokenCount
+    };
+  }
+
   public async update() {
     const result = await this.buildContext(this.projectDir);
     return result;