git.zone/tsdoc
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(context): Improve context building, caching and test robustness

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-03 11:04:21 +00:00
parent d46fd1590e
commit 0a9d535df4
8 changed files with 69 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@git.zone/tsdoc',
version: '1.6.0',
version: '1.6.1',
description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
}

3
ts/context/config-manager.ts
View File

@@ -85,7 +85,6 @@ export class ConfigManager {
directory: undefined // Will be set to .nogit/context-cache by ContextCache
},
analyzer: {
enabled: true,
useAIRefinement: false, // Disabled by default for now
aiModel: 'haiku'
},
@@ -306,7 +305,7 @@ export class ConfigManager {
* Get analyzer configuration
*/
public getAnalyzerConfig(): IAnalyzerConfig {
return this.config.analyzer || { enabled: true, useAIRefinement: false, aiModel: 'haiku' };
return this.config.analyzer || { useAIRefinement: false, aiModel: 'haiku' };
}
/**

271
ts/context/enhanced-context.ts
View File

@@ -69,181 +69,7 @@ export class EnhancedContext {
public setTokenBudget(maxTokens: number): void {
this.tokenBudget = maxTokens;
}
/**
* Gather files from the project
* @param includePaths Optional paths to include
* @param excludePaths Optional paths to exclude
*/
public async gatherFiles(includePaths?: string[], excludePaths?: string[]): Promise<Record<string, plugins.smartfile.SmartFile | plugins.smartfile.SmartFile[]>> {
const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'package.json'),
this.projectDir,
);
const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'readme.md'),
this.projectDir,
);
const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'readme.hints.md'),
this.projectDir,
);
const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'npmextra.json'),
this.projectDir,
);
// Use provided include paths or default to all TypeScript files
const includeGlobs = includePaths?.map(path => `${path}/**/*.ts`) || ['ts*/**/*.ts'];
// Get TypeScript files
const smartfilesModPromises = includeGlobs.map(glob =>
plugins.smartfile.fs.fileTreeToObject(this.projectDir, glob)
);
const smartfilesModArrays = await Promise.all(smartfilesModPromises);
// Flatten the arrays
const smartfilesMod: plugins.smartfile.SmartFile[] = [];
smartfilesModArrays.forEach(array => {
smartfilesMod.push(...array);
});
// Get test files if not excluded
let smartfilesTest: plugins.smartfile.SmartFile[] = [];
if (!excludePaths?.includes('test/')) {
smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(
this.projectDir,
'test/**/*.ts',
);
}
return {
smartfilePackageJSON,
smartfilesReadme,
smartfilesReadmeHints,
smartfilesNpmextraJSON,
smartfilesMod,
smartfilesTest,
};
}
/**
* Convert files to context string
* @param files The files to convert
* @param mode The context mode to use
*/
public async convertFilesToContext(
files: plugins.smartfile.SmartFile[],
mode: ContextMode = this.contextMode
): Promise<string> {
// Reset context result
this.contextResult = {
context: '',
tokenCount: 0,
includedFiles: [],
trimmedFiles: [],
excludedFiles: [],
tokenSavings: 0
};
let totalTokenCount = 0;
let totalOriginalTokens = 0;
// Convert SmartFile objects to IFileMetadata for analysis
const metadata: IFileMetadata[] = files.map(sf => ({
path: sf.path,
relativePath: sf.relative,
size: sf.contents.toString().length,
mtime: Date.now(), // SmartFile doesn't expose mtime, use current time
estimatedTokens: this.countTokens(sf.contents.toString()),
importanceScore: 0
}));
// Analyze files using ContextAnalyzer to get smart prioritization
// (Note: This requires task type which we'll pass from buildContext)
// For now, sort files by estimated tokens (smaller files first for better efficiency)
const sortedFiles = [...files].sort((a, b) => {
const aTokens = this.countTokens(a.contents.toString());
const bTokens = this.countTokens(b.contents.toString());
return aTokens - bTokens;
});
const processedFiles: string[] = [];
for (const smartfile of sortedFiles) {
// Calculate original token count
const originalContent = smartfile.contents.toString();
const originalTokenCount = this.countTokens(originalContent);
totalOriginalTokens += originalTokenCount;
// Apply trimming based on mode
let processedContent = originalContent;
if (mode !== 'full') {
processedContent = this.trimmer.trimFile(
smartfile.relative,
originalContent,
mode
);
}
// Calculate new token count
const processedTokenCount = this.countTokens(processedContent);
// Check if we have budget for this file
if (totalTokenCount + processedTokenCount > this.tokenBudget) {
// We don't have budget for this file
this.contextResult.excludedFiles.push({
path: smartfile.path,
contents: originalContent,
relativePath: smartfile.relative,
tokenCount: originalTokenCount
});
continue;
}
// Format the file for context
const formattedContent = `
====== START OF FILE ${smartfile.relative} ======
${processedContent}
====== END OF FILE ${smartfile.relative} ======
`;
processedFiles.push(formattedContent);
totalTokenCount += processedTokenCount;
// Track file in appropriate list
const fileInfo: IFileInfo = {
path: smartfile.path,
contents: processedContent,
relativePath: smartfile.relative,
tokenCount: processedTokenCount
};
if (mode === 'full' || processedContent === originalContent) {
this.contextResult.includedFiles.push(fileInfo);
} else {
this.contextResult.trimmedFiles.push(fileInfo);
this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
}
}
// Join all processed files
const context = processedFiles.join('\n');
// Update context result
this.contextResult.context = context;
this.contextResult.tokenCount = totalTokenCount;
return context;
}
/**
* Convert files to context with smart analysis and prioritization
* @param metadata - File metadata to analyze
@@ -393,87 +219,44 @@ ${processedContent}
}
/**
* Build context for the project
* @param taskType Optional task type for task-specific context
* Build context for the project using smart analysis
* @param taskType Task type for context-aware prioritization (defaults to 'description')
*/
public async buildContext(taskType?: TaskType): Promise<IContextResult> {
// Initialize if needed
if (this.tokenBudget === 0) {
await this.initialize();
}
// Get task-specific configuration if a task type is provided
if (taskType) {
const taskConfig = this.configManager.getTaskConfig(taskType);
if (taskConfig.mode) {
this.setContextMode(taskConfig.mode);
}
// Smart context building always requires a task type for optimal prioritization
// Default to 'description' if not provided
const effectiveTaskType = taskType || 'description';
// Get task-specific configuration
const taskConfig = this.configManager.getTaskConfig(effectiveTaskType);
if (taskConfig.mode) {
this.setContextMode(taskConfig.mode);
}
// Check if analyzer is enabled in config
const analyzerConfig = this.configManager.getAnalyzerConfig();
const useAnalyzer = analyzerConfig.enabled && taskType;
if (useAnalyzer) {
// Use new smart context building with lazy loading and analysis
const taskConfig = this.configManager.getTaskConfig(taskType!);
// Build globs for scanning
const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [
'ts/**/*.ts',
'ts*/**/*.ts'
];
// Build globs for scanning
const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [
'ts/**/*.ts',
'ts*/**/*.ts'
];
// Add config files
const configGlobs = [
'package.json',
'readme.md',
'readme.hints.md',
'npmextra.json'
];
// Add config files
const configGlobs = [
'package.json',
'readme.md',
'readme.hints.md',
'npmextra.json'
];
// Scan files for metadata (fast, doesn't load contents)
const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]);
// Scan files for metadata (fast, doesn't load contents)
const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]);
// Use analyzer to build context with smart prioritization
await this.convertFilesToContextWithAnalysis(metadata, taskType!, this.contextMode);
} else {
// Fall back to old method for backward compatibility
const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
const files = await this.gatherFiles(
taskConfig?.includePaths,
taskConfig?.excludePaths
);
// Convert files to context
// Create an array of all files to process
const allFiles: plugins.smartfile.SmartFile[] = [];
// Add individual files
if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
// Add arrays of files
if (files.smartfilesMod) {
if (Array.isArray(files.smartfilesMod)) {
allFiles.push(...files.smartfilesMod);
} else {
allFiles.push(files.smartfilesMod);
}
}
if (files.smartfilesTest) {
if (Array.isArray(files.smartfilesTest)) {
allFiles.push(...files.smartfilesTest);
} else {
allFiles.push(files.smartfilesTest);
}
}
await this.convertFilesToContext(allFiles);
}
// Use smart analyzer to build context with intelligent prioritization
await this.convertFilesToContextWithAnalysis(metadata, effectiveTaskType, this.contextMode);
return this.contextResult;
}

5
ts/context/types.ts
View File

@@ -84,11 +84,10 @@ export interface ICacheConfig {
/**
* Analyzer configuration
* Note: Smart analysis is always enabled; this config only controls advanced options
*/
export interface IAnalyzerConfig {
/** Whether analyzer is enabled */
enabled?: boolean;
/** Whether to use AI refinement for selection */
/** Whether to use AI refinement for selection (advanced, disabled by default) */
useAIRefinement?: boolean;
/** AI model to use for refinement */
aiModel?: string;