v1.12.0

changelog.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 2026-01-04 - 1.12.0 - feat(commit)
+add token budgeting and dynamic diff token calculation to avoid OpenAI context limit issues
+
+- Introduce TOKEN_BUDGET constants and calculateMaxDiffTokens() in ts/aidocs_classes/commit.ts
+- Use dynamic maxDiffTokens for DiffProcessor and validate/log warnings when estimated tokens approach limits
+- Add token budgeting notes to readme.hints.md (guidance for splitting large commits and adjusting overhead)
+- Bump dependencies/devDependencies: @git.zone/tstest ^3.1.4, @types/node ^25.0.3, @git.zone/tspublish ^1.11.0, @push.rocks/smartfs ^1.3.1
+
+## 2025-12-16 - 1.11.4 - fix(aidocs_classes)
+clarify recommendedNextVersionMessage field to require only the description body without the type(scope) prefix
+
+- Updated inline documentation in ts/aidocs_classes/commit.ts to explicitly state that recommendedNextVersionMessage must be only the description body (example: 'bump dependency to ^1.2.6') and not include the type(scope) prefix.
+- Removes ambiguity in the example text and improves guidance for commit message generation.
+
 ## 2025-12-15 - 1.11.0 - feat(commit)
 Integrate DualAgentOrchestrator for commit message generation and improve diff/context handling
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@git.zone/tsdoc",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "private": false,
   "description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
   "type": "module",
@@ -21,20 +21,20 @@
   "devDependencies": {
     "@git.zone/tsbuild": "^4.0.2",
     "@git.zone/tsrun": "^2.0.1",
-    "@git.zone/tstest": "^3.1.3",
-    "@types/node": "^25.0.2"
+    "@git.zone/tstest": "^3.1.4",
+    "@types/node": "^25.0.3"
   },
   "dependencies": {
-    "@git.zone/tspublish": "^1.10.3",
+    "@git.zone/tspublish": "^1.11.0",
     "@push.rocks/early": "^4.0.4",
     "@push.rocks/npmextra": "^5.3.3",
     "@push.rocks/qenv": "^6.1.3",
-    "@push.rocks/smartagent": "file:../../push.rocks/smartagent",
+    "@push.rocks/smartagent": "1.2.5",
     "@push.rocks/smartai": "^0.8.0",
     "@push.rocks/smartcli": "^4.0.19",
     "@push.rocks/smartdelay": "^3.0.5",
     "@push.rocks/smartfile": "^13.1.2",
-    "@push.rocks/smartfs": "^1.2.0",
+    "@push.rocks/smartfs": "^1.3.1",
     "@push.rocks/smartgit": "^3.3.1",
     "@push.rocks/smartinteract": "^2.0.16",
     "@push.rocks/smartlog": "^3.1.10",

readme.hints.md

@@ -2,4 +2,13 @@
 * alternatively can be used through npx, if installed locally
 * cli parameters are concluded from ./ts/cli.ts
 * this module is not intended for API use.
-* Read carefully through the TypeScript files. Don't make stuff up.
+* Read carefully through the TypeScript files. Don't make stuff up.
+
+## Token Budgeting (commit.ts)
+* OpenAI has a 272,000 token context limit
+* The smartagent infrastructure adds ~180,000 tokens of overhead (system messages, tool descriptions, conversation history)
+* TOKEN_BUDGET constants in commit.ts control the available tokens for diff content
+* Dynamic calculation: 272K - 10K (safety) - 180K (overhead) - 2K (prompt) = 80K tokens for diff
+* If token limit errors occur, consider:
+  - Splitting large commits into smaller ones
+  - Adjusting SMARTAGENT_OVERHEAD if actual overhead is different

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@git.zone/tsdoc',
-  version: '1.11.0',
+  version: '1.12.0',
   description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
 }

ts/aidocs_classes/commit.ts

@@ -4,6 +4,25 @@ import { ProjectContext } from './projectcontext.js';
 import { DiffProcessor } from '../classes.diffprocessor.js';
 import { logger } from '../logging.js';
 
+// Token budget configuration for OpenAI API limits
+const TOKEN_BUDGET = {
+  OPENAI_CONTEXT_LIMIT: 272000,   // OpenAI's configured limit
+  SAFETY_MARGIN: 10000,            // Buffer to avoid hitting exact limit
+  SMARTAGENT_OVERHEAD: 180000,     // System msgs, tools, history, formatting
+  TASK_PROMPT_OVERHEAD: 2000,      // Task prompt template size
+} as const;
+
+/**
+ * Calculate max tokens available for diff content based on total budget
+ */
+function calculateMaxDiffTokens(): number {
+  const available = TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT
+    - TOKEN_BUDGET.SAFETY_MARGIN
+    - TOKEN_BUDGET.SMARTAGENT_OVERHEAD
+    - TOKEN_BUDGET.TASK_PROMPT_OVERHEAD;
+  return Math.max(available, 30000);
+}
+
 export interface INextCommitObject {
   recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project
   recommendedNextVersionScope: string; // the recommended scope name of the next version, like "core" or "cli", or specific class names.
@@ -90,9 +109,13 @@ export class Commit {
       console.log(`   Estimated tokens: ${estimatedTokens.toLocaleString()}`);
       console.log(`   Exclusion patterns: ${excludePatterns.length}`);
 
+      // Calculate available tokens for diff based on total budget
+      const maxDiffTokens = calculateMaxDiffTokens();
+      console.log(`📊 Token budget: ${maxDiffTokens.toLocaleString()} tokens for diff (limit: ${TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT.toLocaleString()}, overhead: ${(TOKEN_BUDGET.SMARTAGENT_OVERHEAD + TOKEN_BUDGET.TASK_PROMPT_OVERHEAD).toLocaleString()})`);
+
       // Use DiffProcessor to intelligently handle large diffs
       const diffProcessor = new DiffProcessor({
-        maxDiffTokens: 100000,      // Reserve 100k tokens for diffs
+        maxDiffTokens,               // Dynamic based on total budget
         smallFileLines: 300,         // Most source files are under 300 lines
         mediumFileLines: 800,        // Only very large files get head/tail treatment
         sampleHeadLines: 75,         // When sampling, show more context
@@ -111,12 +134,21 @@ export class Commit {
       if (estimatedTokens > 50000) {
         console.log(`✅ DiffProcessor reduced token usage: ${estimatedTokens.toLocaleString()} → ${processedDiff.totalTokens.toLocaleString()}`);
       }
+
+      // Validate total tokens won't exceed limit
+      const totalEstimatedTokens = processedDiff.totalTokens
+        + TOKEN_BUDGET.SMARTAGENT_OVERHEAD
+        + TOKEN_BUDGET.TASK_PROMPT_OVERHEAD;
+
+      if (totalEstimatedTokens > TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT - TOKEN_BUDGET.SAFETY_MARGIN) {
+        console.log(`⚠️  Warning: Estimated tokens (${totalEstimatedTokens.toLocaleString()}) approaching limit`);
+        console.log(`   Consider splitting into smaller commits`);
+      }
     } else {
       processedDiffString = 'No changes.';
     }
 
     // Use DualAgentOrchestrator for commit message generation
-    // Note: No filesystem tool needed - the diff already contains all change information
     const commitOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
       smartAiInstance: this.aiDocsRef.smartAiInstance,
       defaultProvider: 'openai',
@@ -125,43 +157,65 @@ export class Commit {
       guardianPolicyPrompt: `
 You validate commit messages for semantic versioning compliance.
 
-APPROVE if:
+APPROVE tool calls for:
+- Reading package.json or source files to understand project context
+- Using tree to see project structure
+- Listing directory contents
+
+REJECT tool calls for:
+- Reading files outside the project directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+APPROVE final output if:
 - Version level (fix/feat/BREAKING CHANGE) matches the scope of changes in the diff
 - Commit message is clear, professional, and follows conventional commit conventions
 - No personal information, licensing details, or AI mentions (Claude/Codex) included
 - JSON structure is valid with all required fields
 - Scope accurately reflects the changed modules/files
 
-REJECT with specific feedback if:
+REJECT final output if:
 - Version level doesn't match the scope of changes (e.g., "feat" for a typo fix should be "fix")
 - Message is vague, unprofessional, or contains sensitive information
 - JSON is malformed or missing required fields
 `,
     });
 
+    // Register scoped filesystem tool for agent exploration
+    commitOrchestrator.registerScopedFilesystemTool(this.projectDir, [
+      '.nogit/**',
+      'node_modules/**',
+      '.git/**',
+      'dist/**',
+      'dist_*/**',
+    ]);
+
     await commitOrchestrator.start();
 
     const commitTaskPrompt = `
 You create a commit message for a git commit.
 Project directory: ${this.projectDir}
 
+You have access to a filesystem tool to explore the project if needed:
+- Use tree to see project structure
+- Use read to read package.json or source files for context
+
 Analyze the git diff below to understand what changed and generate a commit message.
 
 You should not include any licensing information or personal information.
 Never mention CLAUDE code, or codex.
 
-Important: Answer only in valid JSON.
+Your final output (inside the task_complete tags) must be ONLY valid JSON - the raw JSON object, nothing else.
+No explanations, no summaries, no markdown - just the JSON object that can be parsed with JSON.parse().
 
-Your answer should be parseable with JSON.parse() without modifying anything.
+Here is the structure of the JSON you must return:
 
-Here is the structure of the JSON you should return:
-
-interface {
-  recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level
-  recommendedNextVersionScope: string; // scope name like "core", "cli", or specific class names
-  recommendedNextVersionMessage: string; // the commit message (don't include fix/feat prefix)
-  recommendedNextVersionDetails: string[]; // detailed bullet points for the changelog
-  recommendedNextVersion: string; // the recommended next version x.x.x
+{
+  "recommendedNextVersionLevel": "fix" | "feat" | "BREAKING CHANGE",
+  "recommendedNextVersionScope": "string",
+  "recommendedNextVersionMessage": "string (ONLY the description body WITHOUT the type(scope): prefix - e.g. 'bump dependency to ^1.2.6' NOT 'fix(deps): bump dependency to ^1.2.6')",
+  "recommendedNextVersionDetails": ["string"],
+  "recommendedNextVersion": "x.x.x"
 }
 
 For recommendedNextVersionDetails, only add entries that have obvious value to the reader.
@@ -170,7 +224,7 @@ Here is the git diff showing what changed:
 
 ${processedDiffString}
 
-Generate the commit message based on these changes.
+Analyze these changes and output the JSON commit message object.
 `;
 
     const commitResult = await commitOrchestrator.run(commitTaskPrompt);
@@ -180,9 +234,19 @@ Generate the commit message based on these changes.
       throw new Error(`Commit message generation failed: ${commitResult.status}`);
     }
 
-    const resultObject: INextCommitObject = JSON.parse(
-      commitResult.result.replace('```json', '').replace('```', '')
-    );
+    // Extract JSON from result - handle cases where AI adds text around it
+    let jsonString = commitResult.result
+      .replace(/```json\n?/gi, '')
+      .replace(/```\n?/gi, '');
+
+    // Try to find JSON object in the result
+    const jsonMatch = jsonString.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) {
+      throw new Error(`Could not find JSON object in result: ${jsonString.substring(0, 100)}...`);
+    }
+    jsonString = jsonMatch[0];
+
+    const resultObject: INextCommitObject = JSON.parse(jsonString);
 
     const previousChangelogPath = plugins.path.join(this.projectDir, 'changelog.md');
     let previousChangelog: plugins.smartfile.SmartFile;