1.8.3

changelog.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 2025-11-04 - 1.8.3 - fix(context)
+Prevent enormous git diffs and OOM during context building by adding exclusion patterns, truncation, and diagnostic logging
+
+- Add comprehensive git diff exclusion globs (locks, build artifacts, maps, bundles, IDE folders, logs, caches) when collecting uncommitted diffs to avoid noisy/huge diffs
+- Pass glob patterns directly to smartgit.getUncommittedDiff for efficient server-side matching
+- Emit diagnostic statistics for diffs (files changed, total characters, estimated tokens, number of exclusion patterns) and warn on unusually large diffs
+- Introduce pre-tokenization safety checks in iterative context builder: truncate raw diff text if it exceeds MAX_DIFF_CHARS and throw a clear error if token count still exceeds MAX_DIFF_TOKENS
+- Format and log token counts using locale-aware formatting for clarity
+- Improve robustness of commit context generation to reduce risk of OOM / model-limit overruns
+
+## 2025-11-03 - 1.8.0 - feat(context)
+Wire OpenAI provider through task context factory and add git-diff support to iterative context builder
+
+- Pass AiDoc.openaiInstance through TaskContextFactory into IterativeContextBuilder to reuse the same OpenAI provider and avoid reinitialization.
+- IterativeContextBuilder now accepts an optional OpenAiProvider and an additionalContext string; when provided, git diffs (or other extra context) are prepended to the AI context and token counts are updated.
+- createContextForCommit now forwards the git diff into the iterative builder so commit-specific context includes the diff.
+- Updated aidocs_classes (commit, description, readme) to supply the existing openaiInstance when creating the TaskContextFactory.
+
 ## 2025-11-03 - 1.7.0 - feat(IterativeContextBuilder)
 Add iterative AI-driven context builder and integrate into task factory; add tests and iterative configuration
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@git.zone/tsdoc",
-  "version": "1.7.0",
+  "version": "1.8.3",
   "private": false,
   "description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
   "type": "module",

pnpm-lock.yaml

@@ -3799,8 +3799,8 @@ packages:
     resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==}
     engines: {node: '>=4'}
 
-  minimatch@10.0.3:
-    resolution: {integrity: sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==}
+  minimatch@10.1.1:
+    resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==}
     engines: {node: 20 || >=22}
 
   minimatch@3.1.2:
@@ -9797,7 +9797,7 @@ snapshots:
     dependencies:
       foreground-child: 3.3.1
       jackspeak: 4.1.1
-      minimatch: 10.0.3
+      minimatch: 10.1.1
       minipass: 7.1.2
       package-json-from-dist: 1.0.1
       path-scurry: 2.0.0
@@ -10680,7 +10680,7 @@ snapshots:
 
   min-indent@1.0.1: {}
 
-  minimatch@10.0.3:
+  minimatch@10.1.1:
     dependencies:
       '@isaacs/brace-expansion': 5.0.0
 

test/test.aidoc.nonci.ts

@@ -33,7 +33,10 @@ tap.test('should build commit object', async () => {
   expect(commitObject).toHaveProperty('recommendedNextVersionLevel');
   expect(commitObject).toHaveProperty('recommendedNextVersionScope');
   expect(commitObject).toHaveProperty('recommendedNextVersionMessage');
+});
 
-})
+tap.test('should stop AIdocs', async () => {
+  await aidocs.stop();
+});
 
 tap.start();

test/test.ts

@@ -1,8 +0,0 @@
-import { expect, tap } from '@push.rocks/tapbundle';
-import * as tsdoc from '../ts/index.js';
-
-tap.test('first test', async () => {
-  console.log('test');
-});
-
-tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@git.zone/tsdoc',
-  version: '1.7.0',
+  version: '1.8.3',
   description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
 }

ts/aidocs_classes/commit.ts

@@ -27,12 +27,76 @@ export class Commit {
       smartgitInstance,
       this.projectDir
     );
-    const diffStringArray = await gitRepo.getUncommittedDiff([
+
+    // Define comprehensive exclusion patterns
+    // smartgit@3.3.0+ supports glob patterns natively
+    const excludePatterns = [
+      // Lock files
       'pnpm-lock.yaml',
       'package-lock.json',
-    ]);
+      'npm-shrinkwrap.json',
+      'yarn.lock',
+      'deno.lock',
+      'bun.lockb',
+
+      // Build artifacts (main culprit for large diffs!)
+      'dist/**',
+      'dist_*/**',           // dist_ts, dist_web, etc.
+      'build/**',
+      '.next/**',
+      'out/**',
+      'public/dist/**',
+
+      // Compiled/bundled files
+      '**/*.js.map',
+      '**/*.d.ts.map',
+      '**/*.min.js',
+      '**/*.bundle.js',
+      '**/*.chunk.js',
+
+      // IDE/Editor directories
+      '.claude/**',
+      '.cursor/**',
+      '.vscode/**',
+      '.idea/**',
+      '**/*.swp',
+      '**/*.swo',
+
+      // Logs and caches
+      '.nogit/**',
+      '**/*.log',
+      '.cache/**',
+      '.rpt2_cache/**',
+      'coverage/**',
+      '.nyc_output/**',
+    ];
+
+    // Pass glob patterns directly to smartgit - it handles matching internally
+    const diffStringArray = await gitRepo.getUncommittedDiff(excludePatterns);
+
+    // Diagnostic logging for diff statistics
+    if (diffStringArray.length > 0) {
+      const totalChars = diffStringArray.join('\n\n').length;
+      const estimatedTokens = Math.ceil(totalChars / 4);
+
+      console.log(`📊 Git diff statistics:`);
+      console.log(`   Files changed: ${diffStringArray.length}`);
+      console.log(`   Total characters: ${totalChars.toLocaleString()}`);
+      console.log(`   Estimated tokens: ${estimatedTokens.toLocaleString()}`);
+      console.log(`   Exclusion patterns: ${excludePatterns.length}`);
+
+      if (estimatedTokens > 50000) {
+        console.warn(`⚠️  WARNING: Unusually large diff (${estimatedTokens.toLocaleString()} tokens)`);
+        console.warn(`   This may indicate build artifacts or large files in the diff.`);
+        console.warn(`   Consider reviewing uncommitted changes or improving exclusion patterns.`);
+      }
+    }
+
     // Use the new TaskContextFactory for optimized context
-    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
+      this.projectDir,
+      this.aiDocsRef.openaiInstance
+    );
     await taskContextFactory.initialize();
     
     // Generate context specifically for commit task

ts/aidocs_classes/description.ts

@@ -19,7 +19,10 @@ export class Description {
 
   public async build() {
     // Use the new TaskContextFactory for optimized context
-    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
+      this.projectDir,
+      this.aiDocsRef.openaiInstance
+    );
     await taskContextFactory.initialize();
     
     // Generate context specifically for description task

ts/aidocs_classes/readme.ts

@@ -18,7 +18,10 @@ export class Readme {
     let finalReadmeString = ``;
 
     // Use the new TaskContextFactory for optimized context
-    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
+      this.projectDir,
+      this.aiDocsRef.openaiInstance
+    );
     await taskContextFactory.initialize();
     
     // Generate context specifically for readme task

ts/classes.aidoc.ts

@@ -64,7 +64,7 @@ export class AiDoc {
         await this.npmextraKV.writeKey('OPENAI_TOKEN', this.openaiToken);
       }
     }
-    if (!this.openaiToken) {
+    if (!this.openaiToken && this.npmextraKV) {
       this.openaiToken = await this.npmextraKV.readKey('OPENAI_TOKEN');
     }
 
@@ -76,7 +76,11 @@ export class AiDoc {
   }
 
   public async stop() {
-    await this.openaiInstance.stop();
+    if (this.openaiInstance) {
+      await this.openaiInstance.stop();
+    }
+    // No explicit cleanup needed for npmextraKV or aidocInteract
+    // They don't keep event loop alive
   }
 
   public async buildReadme(projectDirArg: string) {

ts/context/iterative-context-builder.ts

@@ -28,17 +28,24 @@ export class IterativeContextBuilder {
   private config: Required<IIterativeConfig>;
   private tokenBudget: number = 190000;
   private openaiInstance: plugins.smartai.OpenAiProvider;
+  private externalOpenaiInstance?: plugins.smartai.OpenAiProvider;
 
   /**
    * Creates a new IterativeContextBuilder
    * @param projectRoot - Root directory of the project
    * @param config - Iterative configuration
+   * @param openaiInstance - Optional pre-configured OpenAI provider instance
    */
-  constructor(projectRoot: string, config?: Partial<IIterativeConfig>) {
+  constructor(
+    projectRoot: string,
+    config?: Partial<IIterativeConfig>,
+    openaiInstance?: plugins.smartai.OpenAiProvider
+  ) {
     this.projectRoot = projectRoot;
     this.lazyLoader = new LazyFileLoader(projectRoot);
     this.cache = new ContextCache(projectRoot);
     this.analyzer = new ContextAnalyzer(projectRoot);
+    this.externalOpenaiInstance = openaiInstance;
 
     // Default configuration
     this.config = {
@@ -60,24 +67,30 @@ export class IterativeContextBuilder {
     await configManager.initialize(this.projectRoot);
     this.tokenBudget = configManager.getMaxTokens();
 
-    // Initialize OpenAI instance
-    const qenvInstance = new plugins.qenv.Qenv();
-    const openaiToken = await qenvInstance.getEnvVarOnDemand('OPENAI_TOKEN');
-    if (!openaiToken) {
-      throw new Error('OPENAI_TOKEN environment variable is required for iterative context building');
+    // Use external OpenAI instance if provided, otherwise create a new one
+    if (this.externalOpenaiInstance) {
+      this.openaiInstance = this.externalOpenaiInstance;
+    } else {
+      // Initialize OpenAI instance from environment
+      const qenvInstance = new plugins.qenv.Qenv();
+      const openaiToken = await qenvInstance.getEnvVarOnDemand('OPENAI_TOKEN');
+      if (!openaiToken) {
+        throw new Error('OPENAI_TOKEN environment variable is required for iterative context building');
+      }
+      this.openaiInstance = new plugins.smartai.OpenAiProvider({
+        openaiToken,
+      });
+      await this.openaiInstance.start();
     }
-    this.openaiInstance = new plugins.smartai.OpenAiProvider({
-      openaiToken,
-    });
-    await this.openaiInstance.start();
   }
 
   /**
    * Build context iteratively using AI decision making
    * @param taskType - Type of task being performed
+   * @param additionalContext - Optional additional context (e.g., git diff for commit tasks)
    * @returns Complete iterative context result
    */
-  public async buildContextIteratively(taskType: TaskType): Promise<IIterativeContextResult> {
+  public async buildContextIteratively(taskType: TaskType, additionalContext?: string): Promise<IIterativeContextResult> {
     const startTime = Date.now();
     logger.log('info', '🤖 Starting iterative context building...');
     logger.log('info', `   Task: ${taskType}, Budget: ${this.tokenBudget} tokens, Max iterations: ${this.config.maxIterations}`);
@@ -100,6 +113,49 @@ export class IterativeContextBuilder {
     let loadedContent = '';
     const includedFiles: IFileInfo[] = [];
 
+    // If additional context (e.g., git diff) is provided, prepend it
+    if (additionalContext) {
+      // CRITICAL SAFETY: Check raw string size BEFORE tokenization to prevent OOM
+      const MAX_DIFF_CHARS = 500000; // ~125k tokens max (conservative 4 chars/token ratio)
+      const MAX_DIFF_TOKENS = 150000; // Hard token limit for safety
+
+      // First check: raw character count
+      if (additionalContext.length > MAX_DIFF_CHARS) {
+        const originalSize = additionalContext.length;
+        logger.log('warn', `⚠️  Git diff too large (${originalSize.toLocaleString()} chars > ${MAX_DIFF_CHARS.toLocaleString()} limit)`);
+        logger.log('warn', `   This likely includes build artifacts (dist/, *.js.map, bundles, etc.)`);
+        logger.log('warn', `   Truncating to first ${MAX_DIFF_CHARS.toLocaleString()} characters.`);
+        logger.log('warn', `   Consider: git stash build files, improve .gitignore, or review uncommitted changes.`);
+
+        additionalContext = additionalContext.substring(0, MAX_DIFF_CHARS) +
+          '\n\n[... DIFF TRUNCATED - exceeded size limit of ' + MAX_DIFF_CHARS.toLocaleString() + ' chars ...]';
+      }
+
+      const diffSection = `
+====== GIT DIFF ======
+
+${additionalContext}
+
+====== END OF GIT DIFF ======
+`;
+
+      // Second check: actual token count after truncation
+      const diffTokens = this.countTokens(diffSection);
+
+      if (diffTokens > MAX_DIFF_TOKENS) {
+        logger.log('error', `❌ Git diff still too large after truncation (${diffTokens.toLocaleString()} tokens > ${MAX_DIFF_TOKENS.toLocaleString()} limit)`);
+        throw new Error(
+          `Git diff size (${diffTokens.toLocaleString()} tokens) exceeds maximum (${MAX_DIFF_TOKENS.toLocaleString()} tokens). ` +
+          `This indicates massive uncommitted changes, likely build artifacts. ` +
+          `Please commit or stash dist/, build/, or other generated files.`
+        );
+      }
+
+      loadedContent = diffSection;
+      totalTokensUsed += diffTokens;
+      logger.log('info', `📝 Added git diff to context (${diffTokens.toLocaleString()} tokens)`);
+    }
+
     // Phase 3: Iterative file selection and loading
     for (let iteration = 1; iteration <= this.config.maxIterations; iteration++) {
       const iterationStart = Date.now();

ts/context/task-context-factory.ts

@@ -9,14 +9,17 @@ import type { IIterativeContextResult, TaskType } from './types.js';
 export class TaskContextFactory {
   private projectDir: string;
   private configManager: ConfigManager;
+  private openaiInstance?: any; // OpenAI provider instance
 
   /**
    * Create a new TaskContextFactory
    * @param projectDirArg The project directory
+   * @param openaiInstance Optional pre-configured OpenAI provider instance
    */
-  constructor(projectDirArg: string) {
+  constructor(projectDirArg: string, openaiInstance?: any) {
     this.projectDir = projectDirArg;
     this.configManager = ConfigManager.getInstance();
+    this.openaiInstance = openaiInstance;
   }
 
   /**
@@ -32,7 +35,8 @@ export class TaskContextFactory {
   public async createContextForReadme(): Promise<IIterativeContextResult> {
     const iterativeBuilder = new IterativeContextBuilder(
       this.projectDir,
-      this.configManager.getIterativeConfig()
+      this.configManager.getIterativeConfig(),
+      this.openaiInstance
     );
     await iterativeBuilder.initialize();
     return await iterativeBuilder.buildContextIteratively('readme');
@@ -44,7 +48,8 @@ export class TaskContextFactory {
   public async createContextForDescription(): Promise<IIterativeContextResult> {
     const iterativeBuilder = new IterativeContextBuilder(
       this.projectDir,
-      this.configManager.getIterativeConfig()
+      this.configManager.getIterativeConfig(),
+      this.openaiInstance
     );
     await iterativeBuilder.initialize();
     return await iterativeBuilder.buildContextIteratively('description');
@@ -52,16 +57,16 @@ export class TaskContextFactory {
 
   /**
    * Create context for commit message generation
-   * @param gitDiff Optional git diff to include (currently not used in iterative mode)
+   * @param gitDiff Optional git diff to include in the context
    */
   public async createContextForCommit(gitDiff?: string): Promise<IIterativeContextResult> {
     const iterativeBuilder = new IterativeContextBuilder(
       this.projectDir,
-      this.configManager.getIterativeConfig()
+      this.configManager.getIterativeConfig(),
+      this.openaiInstance
     );
     await iterativeBuilder.initialize();
-    // Note: git diff could be incorporated into the iterative prompts if needed
-    return await iterativeBuilder.buildContextIteratively('commit');
+    return await iterativeBuilder.buildContextIteratively('commit', gitDiff);
   }
 
   /**