v1.10.0

changelog.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 2025-12-02 - 1.10.0 - feat(diff-processor)
+Improve diff sampling and file prioritization: increase inclusion thresholds, expand sampled context, and boost priority for interface/type and entry-point files
+
+- Raise small/medium file thresholds used by DiffProcessor (smallFileLines 50 -> 300, mediumFileLines 200 -> 800) so more source files are included fully or summarized rather than treated as large metadata-only files
+- Increase sample window for medium files (sampleHeadLines/sampleTailLines 20 -> 75) to provide more context when summarizing diffs
+- Boost importance scoring for interfaces/type files and entry points (adds +20 for interfaces/.types and +15 for index/mod entry files) to prioritize critical API surface in diff processing
+- Keep other prioritization rules intact (source/test/config/docs/build heuristics), and align the aidoc commit DiffProcessor usage with the new defaults
+
+## 2025-11-04 - 1.9.2 - fix(deps)
+Update dependencies and devDependencies to newer versions (bump multiple packages)
+
+- Bumped devDependencies: @git.zone/tsbuild 2.6.8 -> 2.7.1, @git.zone/tsrun 1.2.46 -> 1.6.2, @git.zone/tstest 2.3.6 -> 2.7.0
+- Bumped runtime dependencies: @push.rocks/smartai 0.5.11 -> 0.8.0, @push.rocks/smartcli 4.0.11 -> 4.0.19, @push.rocks/smartgit 3.2.1 -> 3.3.1, @push.rocks/smartlog 3.1.9 -> 3.1.10, gpt-tokenizer 3.0.1 -> 3.2.0, typedoc 0.28.12 -> 0.28.14, typescript 5.9.2 -> 5.9.3
+- No source code changes in this commit; dependency-only updates. Run the test suite and CI to verify compatibility.
+
+## 2025-11-04 - 1.9.1 - fix(iterative-context-builder)
+Rely on DiffProcessor for git diff pre-processing; remove raw char truncation, raise diff token safety, and improve logging
+
+- Removed raw character-based truncation of additionalContext — diffs are expected to be pre-processed by DiffProcessor instead of blind substring truncation.
+- Now validates pre-processed diff token count only and treats DiffProcessor as the primary sampler (DiffProcessor typically uses a ~100k token budget).
+- Increased MAX_DIFF_TOKENS safety net to 200,000 to cover edge cases and avoid false positives; updated logs to reflect pre-processed diffs.
+- Improved error messaging to indicate a likely DiffProcessor misconfiguration when pre-processed diffs exceed the safety limit.
+- Updated informational logs to state that a pre-processed git diff was added to context.
+
 ## 2025-11-04 - 1.9.0 - feat(context)
 Add intelligent DiffProcessor to summarize and prioritize git diffs and integrate it into the commit context pipeline
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@git.zone/tsdoc",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "private": false,
   "description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
   "type": "module",
@@ -19,9 +19,9 @@
     "buildDocs": "tsdoc"
   },
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.6.8",
-    "@git.zone/tsrun": "^1.2.46",
-    "@git.zone/tstest": "^2.3.6",
+    "@git.zone/tsbuild": "^2.7.1",
+    "@git.zone/tsrun": "^1.6.2",
+    "@git.zone/tstest": "^2.7.0",
     "@types/node": "^22.15.17"
   },
   "dependencies": {
@@ -29,20 +29,20 @@
     "@push.rocks/early": "^4.0.3",
     "@push.rocks/npmextra": "^5.3.3",
     "@push.rocks/qenv": "^6.1.3",
-    "@push.rocks/smartai": "^0.5.11",
-    "@push.rocks/smartcli": "^4.0.11",
+    "@push.rocks/smartai": "^0.8.0",
+    "@push.rocks/smartcli": "^4.0.19",
     "@push.rocks/smartdelay": "^3.0.5",
     "@push.rocks/smartfile": "^11.2.7",
-    "@push.rocks/smartgit": "^3.2.1",
+    "@push.rocks/smartgit": "^3.3.1",
     "@push.rocks/smartinteract": "^2.0.15",
-    "@push.rocks/smartlog": "^3.1.9",
+    "@push.rocks/smartlog": "^3.1.10",
     "@push.rocks/smartlog-destination-local": "^9.0.2",
     "@push.rocks/smartpath": "^6.0.0",
     "@push.rocks/smartshell": "^3.3.0",
     "@push.rocks/smarttime": "^4.0.6",
-    "gpt-tokenizer": "^3.0.1",
-    "typedoc": "^0.28.12",
-    "typescript": "^5.9.2"
+    "gpt-tokenizer": "^3.2.0",
+    "typedoc": "^0.28.14",
+    "typescript": "^5.9.3"
   },
   "files": [
     "ts/**/*",

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@git.zone/tsdoc',
-  version: '1.9.0',
+  version: '1.10.0',
   description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
 }

ts/aidocs_classes/commit.ts

@@ -92,10 +92,10 @@ export class Commit {
       // Use DiffProcessor to intelligently handle large diffs
       const diffProcessor = new DiffProcessor({
         maxDiffTokens: 100000,      // Reserve 100k tokens for diffs
-        smallFileLines: 50,          // Include files <= 50 lines fully
-        mediumFileLines: 200,        // Summarize files <= 200 lines
-        sampleHeadLines: 20,         // Show first 20 lines
-        sampleTailLines: 20,         // Show last 20 lines
+        smallFileLines: 300,         // Most source files are under 300 lines
+        mediumFileLines: 800,        // Only very large files get head/tail treatment
+        sampleHeadLines: 75,         // When sampling, show more context
+        sampleTailLines: 75,         // When sampling, show more context
       });
 
       const processedDiff = diffProcessor.processDiffs(diffStringArray);

ts/context/diff-processor.ts

@@ -239,8 +239,20 @@ export class DiffProcessor {
       return 10;
     }
 
-    // Everything else - default priority
-    return 50;
+    // Start with default priority
+    let score = 50;
+
+    // Boost interface/type files - they're usually small but critical
+    if (filepath.includes('interfaces/') || filepath.includes('.types.')) {
+      score += 20;
+    }
+
+    // Boost entry points
+    if (filepath.endsWith('index.ts') || filepath.endsWith('mod.ts')) {
+      score += 15;
+    }
+
+    return score;
   }
 
   /**

ts/context/iterative-context-builder.ts

@@ -115,21 +115,9 @@ export class IterativeContextBuilder {
 
     // If additional context (e.g., git diff) is provided, prepend it
     if (additionalContext) {
-      // CRITICAL SAFETY: Check raw string size BEFORE tokenization to prevent OOM
-      const MAX_DIFF_CHARS = 500000; // ~125k tokens max (conservative 4 chars/token ratio)
-      const MAX_DIFF_TOKENS = 150000; // Hard token limit for safety
-
-      // First check: raw character count
-      if (additionalContext.length > MAX_DIFF_CHARS) {
-        const originalSize = additionalContext.length;
-        logger.log('warn', `⚠️  Git diff too large (${originalSize.toLocaleString()} chars > ${MAX_DIFF_CHARS.toLocaleString()} limit)`);
-        logger.log('warn', `   This likely includes build artifacts (dist/, *.js.map, bundles, etc.)`);
-        logger.log('warn', `   Truncating to first ${MAX_DIFF_CHARS.toLocaleString()} characters.`);
-        logger.log('warn', `   Consider: git stash build files, improve .gitignore, or review uncommitted changes.`);
-
-        additionalContext = additionalContext.substring(0, MAX_DIFF_CHARS) +
-          '\n\n[... DIFF TRUNCATED - exceeded size limit of ' + MAX_DIFF_CHARS.toLocaleString() + ' chars ...]';
-      }
+      // NOTE: additionalContext is expected to be pre-processed by DiffProcessor
+      // which intelligently samples large diffs to stay within token budget (100k default)
+      const MAX_DIFF_TOKENS = 200000; // Safety net for edge cases (DiffProcessor uses 100k budget)
 
       const diffSection = `
 ====== GIT DIFF ======
@@ -139,21 +127,22 @@ ${additionalContext}
 ====== END OF GIT DIFF ======
 `;
 
-      // Second check: actual token count after truncation
+      // Validate token count (should already be under budget from DiffProcessor)
       const diffTokens = this.countTokens(diffSection);
 
       if (diffTokens > MAX_DIFF_TOKENS) {
-        logger.log('error', `❌ Git diff still too large after truncation (${diffTokens.toLocaleString()} tokens > ${MAX_DIFF_TOKENS.toLocaleString()} limit)`);
+        logger.log('error', `❌ Pre-processed git diff exceeds safety limit (${diffTokens.toLocaleString()} tokens > ${MAX_DIFF_TOKENS.toLocaleString()} limit)`);
+        logger.log('error', `   This should not happen - DiffProcessor should have limited to ~100k tokens.`);
+        logger.log('error', `   Please check DiffProcessor configuration and output.`);
         throw new Error(
-          `Git diff size (${diffTokens.toLocaleString()} tokens) exceeds maximum (${MAX_DIFF_TOKENS.toLocaleString()} tokens). ` +
-          `This indicates massive uncommitted changes, likely build artifacts. ` +
-          `Please commit or stash dist/, build/, or other generated files.`
+          `Pre-processed git diff size (${diffTokens.toLocaleString()} tokens) exceeds safety limit (${MAX_DIFF_TOKENS.toLocaleString()} tokens). ` +
+          `This indicates a bug in DiffProcessor or misconfiguration.`
         );
       }
 
       loadedContent = diffSection;
       totalTokensUsed += diffTokens;
-      logger.log('info', `📝 Added git diff to context (${diffTokens.toLocaleString()} tokens)`);
+      logger.log('info', `📝 Added pre-processed git diff to context (${diffTokens.toLocaleString()} tokens)`);
     }
 
     // Phase 3: Iterative file selection and loading