git.zone/tsdoc
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(context): Prevent enormous git diffs and OOM during context building by adding exclusion patterns, truncation, and diagnostic logging

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-04 01:37:15 +00:00
parent 11e65b92ec
commit 3f22fc91ae
5 changed files with 104 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
ts/context/iterative-context-builder.ts
View File

@@ -115,6 +115,22 @@ export class IterativeContextBuilder {
// If additional context (e.g., git diff) is provided, prepend it
if (additionalContext) {
// CRITICAL SAFETY: Check raw string size BEFORE tokenization to prevent OOM
const MAX_DIFF_CHARS = 500000; // ~125k tokens max (conservative 4 chars/token ratio)
const MAX_DIFF_TOKENS = 150000; // Hard token limit for safety
// First check: raw character count
if (additionalContext.length > MAX_DIFF_CHARS) {
const originalSize = additionalContext.length;
logger.log('warn', `⚠️ Git diff too large (${originalSize.toLocaleString()} chars > ${MAX_DIFF_CHARS.toLocaleString()} limit)`);
logger.log('warn', ` This likely includes build artifacts (dist/, *.js.map, bundles, etc.)`);
logger.log('warn', ` Truncating to first ${MAX_DIFF_CHARS.toLocaleString()} characters.`);
logger.log('warn', ` Consider: git stash build files, improve .gitignore, or review uncommitted changes.`);
additionalContext = additionalContext.substring(0, MAX_DIFF_CHARS) +
'\n\n[... DIFF TRUNCATED - exceeded size limit of ' + MAX_DIFF_CHARS.toLocaleString() + ' chars ...]';
}
const diffSection = `
====== GIT DIFF ======
@@ -122,10 +138,22 @@ ${additionalContext}
====== END OF GIT DIFF ======
`;
loadedContent = diffSection;
// Second check: actual token count after truncation
const diffTokens = this.countTokens(diffSection);
if (diffTokens > MAX_DIFF_TOKENS) {
logger.log('error', `❌ Git diff still too large after truncation (${diffTokens.toLocaleString()} tokens > ${MAX_DIFF_TOKENS.toLocaleString()} limit)`);
throw new Error(
`Git diff size (${diffTokens.toLocaleString()} tokens) exceeds maximum (${MAX_DIFF_TOKENS.toLocaleString()} tokens). ` +
`This indicates massive uncommitted changes, likely build artifacts. ` +
`Please commit or stash dist/, build/, or other generated files.`
);
}
loadedContent = diffSection;
totalTokensUsed += diffTokens;
logger.log('info', `📝 Added git diff to context (${diffTokens} tokens)`);
logger.log('info', `📝 Added git diff to context (${diffTokens.toLocaleString()} tokens)`);
}
// Phase 3: Iterative file selection and loading