feat(commit): add token budgeting and dynamic diff token calculation to avoid OpenAI context limit issues

changelog.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 2026-01-04 - 1.12.0 - feat(commit)
+add token budgeting and dynamic diff token calculation to avoid OpenAI context limit issues
+
+- Introduce TOKEN_BUDGET constants and calculateMaxDiffTokens() in ts/aidocs_classes/commit.ts
+- Use dynamic maxDiffTokens for DiffProcessor and validate/log warnings when estimated tokens approach limits
+- Add token budgeting notes to readme.hints.md (guidance for splitting large commits and adjusting overhead)
+- Bump dependencies/devDependencies: @git.zone/tstest ^3.1.4, @types/node ^25.0.3, @git.zone/tspublish ^1.11.0, @push.rocks/smartfs ^1.3.1
+
 ## 2025-12-16 - 1.11.4 - fix(aidocs_classes)
 clarify recommendedNextVersionMessage field to require only the description body without the type(scope) prefix
 

package.json

@@ -21,11 +21,11 @@
   "devDependencies": {
     "@git.zone/tsbuild": "^4.0.2",
     "@git.zone/tsrun": "^2.0.1",
-    "@git.zone/tstest": "^3.1.3",
+    "@git.zone/tstest": "^3.1.4",
-    "@types/node": "^25.0.2"
+    "@types/node": "^25.0.3"
   },
   "dependencies": {
-    "@git.zone/tspublish": "^1.10.3",
+    "@git.zone/tspublish": "^1.11.0",
     "@push.rocks/early": "^4.0.4",
     "@push.rocks/npmextra": "^5.3.3",
     "@push.rocks/qenv": "^6.1.3",
@@ -34,7 +34,7 @@
     "@push.rocks/smartcli": "^4.0.19",
     "@push.rocks/smartdelay": "^3.0.5",
     "@push.rocks/smartfile": "^13.1.2",
-    "@push.rocks/smartfs": "^1.2.0",
+    "@push.rocks/smartfs": "^1.3.1",
     "@push.rocks/smartgit": "^3.3.1",
     "@push.rocks/smartinteract": "^2.0.16",
     "@push.rocks/smartlog": "^3.1.10",

readme.hints.md

@@ -2,4 +2,13 @@
 * alternatively can be used through npx, if installed locally
 * cli parameters are concluded from ./ts/cli.ts
 * this module is not intended for API use.
-* Read carefully through the TypeScript files. Don't make stuff up.
+* Read carefully through the TypeScript files. Don't make stuff up.
+
+## Token Budgeting (commit.ts)
+* OpenAI has a 272,000 token context limit
+* The smartagent infrastructure adds ~180,000 tokens of overhead (system messages, tool descriptions, conversation history)
+* TOKEN_BUDGET constants in commit.ts control the available tokens for diff content
+* Dynamic calculation: 272K - 10K (safety) - 180K (overhead) - 2K (prompt) = 80K tokens for diff
+* If token limit errors occur, consider:
+  - Splitting large commits into smaller ones
+  - Adjusting SMARTAGENT_OVERHEAD if actual overhead is different

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@git.zone/tsdoc',
-  version: '1.11.4',
+  version: '1.12.0',
   description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
 }

ts/aidocs_classes/commit.ts

@@ -4,6 +4,25 @@ import { ProjectContext } from './projectcontext.js';
 import { DiffProcessor } from '../classes.diffprocessor.js';
 import { logger } from '../logging.js';
 
+// Token budget configuration for OpenAI API limits
+const TOKEN_BUDGET = {
+  OPENAI_CONTEXT_LIMIT: 272000,   // OpenAI's configured limit
+  SAFETY_MARGIN: 10000,            // Buffer to avoid hitting exact limit
+  SMARTAGENT_OVERHEAD: 180000,     // System msgs, tools, history, formatting
+  TASK_PROMPT_OVERHEAD: 2000,      // Task prompt template size
+} as const;
+
+/**
+ * Calculate max tokens available for diff content based on total budget
+ */
+function calculateMaxDiffTokens(): number {
+  const available = TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT
+    - TOKEN_BUDGET.SAFETY_MARGIN
+    - TOKEN_BUDGET.SMARTAGENT_OVERHEAD
+    - TOKEN_BUDGET.TASK_PROMPT_OVERHEAD;
+  return Math.max(available, 30000);
+}
+
 export interface INextCommitObject {
   recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project
   recommendedNextVersionScope: string; // the recommended scope name of the next version, like "core" or "cli", or specific class names.
@@ -90,9 +109,13 @@ export class Commit {
       console.log(`   Estimated tokens: ${estimatedTokens.toLocaleString()}`);
       console.log(`   Exclusion patterns: ${excludePatterns.length}`);
 
+      // Calculate available tokens for diff based on total budget
+      const maxDiffTokens = calculateMaxDiffTokens();
+      console.log(`📊 Token budget: ${maxDiffTokens.toLocaleString()} tokens for diff (limit: ${TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT.toLocaleString()}, overhead: ${(TOKEN_BUDGET.SMARTAGENT_OVERHEAD + TOKEN_BUDGET.TASK_PROMPT_OVERHEAD).toLocaleString()})`);
+
       // Use DiffProcessor to intelligently handle large diffs
       const diffProcessor = new DiffProcessor({
-        maxDiffTokens: 100000,      // Reserve 100k tokens for diffs
+        maxDiffTokens,               // Dynamic based on total budget
         smallFileLines: 300,         // Most source files are under 300 lines
         mediumFileLines: 800,        // Only very large files get head/tail treatment
         sampleHeadLines: 75,         // When sampling, show more context
@@ -111,6 +134,16 @@ export class Commit {
       if (estimatedTokens > 50000) {
         console.log(`✅ DiffProcessor reduced token usage: ${estimatedTokens.toLocaleString()} → ${processedDiff.totalTokens.toLocaleString()}`);
       }
+
+      // Validate total tokens won't exceed limit
+      const totalEstimatedTokens = processedDiff.totalTokens
+        + TOKEN_BUDGET.SMARTAGENT_OVERHEAD
+        + TOKEN_BUDGET.TASK_PROMPT_OVERHEAD;
+
+      if (totalEstimatedTokens > TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT - TOKEN_BUDGET.SAFETY_MARGIN) {
+        console.log(`⚠️  Warning: Estimated tokens (${totalEstimatedTokens.toLocaleString()}) approaching limit`);
+        console.log(`   Consider splitting into smaller commits`);
+      }
     } else {
       processedDiffString = 'No changes.';
     }