git.zone/tsdoc
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(commit): add token budgeting and dynamic diff token calculation to avoid OpenAI context limit issues

Browse Source
This commit is contained in:
Jürgen Kunz
2026-01-04 15:20:27 +00:00
parent 5d0411a5ba
commit 21006b41d0
6 changed files with 1270 additions and 926 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@git.zone/tsdoc',
version: '1.11.4',
version: '1.12.0',
description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
}

35
ts/aidocs_classes/commit.ts
View File

@@ -4,6 +4,25 @@ import { ProjectContext } from './projectcontext.js';
import { DiffProcessor } from '../classes.diffprocessor.js';
import { logger } from '../logging.js';
// Token budget configuration for OpenAI API limits
const TOKEN_BUDGET = {
OPENAI_CONTEXT_LIMIT: 272000, // OpenAI's configured limit
SAFETY_MARGIN: 10000, // Buffer to avoid hitting exact limit
SMARTAGENT_OVERHEAD: 180000, // System msgs, tools, history, formatting
TASK_PROMPT_OVERHEAD: 2000, // Task prompt template size
} as const;
/**
* Calculate max tokens available for diff content based on total budget
*/
function calculateMaxDiffTokens(): number {
const available = TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT
- TOKEN_BUDGET.SAFETY_MARGIN
- TOKEN_BUDGET.SMARTAGENT_OVERHEAD
- TOKEN_BUDGET.TASK_PROMPT_OVERHEAD;
return Math.max(available, 30000);
}
export interface INextCommitObject {
recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project
recommendedNextVersionScope: string; // the recommended scope name of the next version, like "core" or "cli", or specific class names.
@@ -90,9 +109,13 @@ export class Commit {
console.log(` Estimated tokens: ${estimatedTokens.toLocaleString()}`);
console.log(` Exclusion patterns: ${excludePatterns.length}`);
// Calculate available tokens for diff based on total budget
const maxDiffTokens = calculateMaxDiffTokens();
console.log(`📊 Token budget: ${maxDiffTokens.toLocaleString()} tokens for diff (limit: ${TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT.toLocaleString()}, overhead: ${(TOKEN_BUDGET.SMARTAGENT_OVERHEAD + TOKEN_BUDGET.TASK_PROMPT_OVERHEAD).toLocaleString()})`);
// Use DiffProcessor to intelligently handle large diffs
const diffProcessor = new DiffProcessor({
maxDiffTokens: 100000, // Reserve 100k tokens for diffs
maxDiffTokens, // Dynamic based on total budget
smallFileLines: 300, // Most source files are under 300 lines
mediumFileLines: 800, // Only very large files get head/tail treatment
sampleHeadLines: 75, // When sampling, show more context
@@ -111,6 +134,16 @@ export class Commit {
if (estimatedTokens > 50000) {
console.log(`✅ DiffProcessor reduced token usage: ${estimatedTokens.toLocaleString()} → ${processedDiff.totalTokens.toLocaleString()}`);
}
// Validate total tokens won't exceed limit
const totalEstimatedTokens = processedDiff.totalTokens
+ TOKEN_BUDGET.SMARTAGENT_OVERHEAD
+ TOKEN_BUDGET.TASK_PROMPT_OVERHEAD;
if (totalEstimatedTokens > TOKEN_BUDGET.OPENAI_CONTEXT_LIMIT - TOKEN_BUDGET.SAFETY_MARGIN) {
console.log(`⚠️ Warning: Estimated tokens (${totalEstimatedTokens.toLocaleString()}) approaching limit`);
console.log(` Consider splitting into smaller commits`);
}
} else {
processedDiffString = 'No changes.';
}