update

ts/aidocs_classes/projectcontext.ts

@@ -64,21 +64,14 @@ ${smartfile.contents.toString()}
   }
 
   /**
-   * Calculate the token count for a string using the GPT tokenizer
-   * @param text The text to count tokens for
-   * @param model The model to use for token counting (default: gpt-3.5-turbo)
-   * @returns The number of tokens in the text
+   * Estimate token count for a string
+   * Uses a rough estimate of 4 characters per token
+   * @param text The text to estimate tokens for
+   * @returns Estimated number of tokens
    */
-  public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
-    try {
-      // Use the gpt-tokenizer library to count tokens
-      const tokens = plugins.gptTokenizer.encode(text);
-      return tokens.length;
-    } catch (error) {
-      console.error('Error counting tokens:', error);
-      // Provide a rough estimate (4 chars per token) if tokenization fails
-      return Math.ceil(text.length / 4);
-    }
+  public countTokens(text: string): number {
+    // Rough estimate: ~4 characters per token for English text
+    return Math.ceil(text.length / 4);
   }
 
   private async buildContext(dirArg: string) {