git.zone/tsdoc
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(context): Introduce smart context system: analyzer, lazy loader, cache and README/docs improvements

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-02 23:07:59 +00:00
parent fe5121ec9c
commit 1d7317f063
14 changed files with 3031 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

286
ts/context/enhanced-context.ts
View File

@@ -1,7 +1,10 @@
import * as plugins from '../plugins.js';
import type { ContextMode, IContextResult, IFileInfo, TaskType } from './types.js';
import type { ContextMode, IContextResult, IFileInfo, TaskType, IFileMetadata } from './types.js';
import { ContextTrimmer } from './context-trimmer.js';
import { ConfigManager } from './config-manager.js';
import { LazyFileLoader } from './lazy-file-loader.js';
import { ContextCache } from './context-cache.js';
import { ContextAnalyzer } from './context-analyzer.js';
/**
* Enhanced ProjectContext that supports context optimization strategies
@@ -10,6 +13,9 @@ export class EnhancedContext {
private projectDir: string;
private trimmer: ContextTrimmer;
private configManager: ConfigManager;
private lazyLoader: LazyFileLoader;
private cache: ContextCache;
private analyzer: ContextAnalyzer;
private contextMode: ContextMode = 'trimmed';
private tokenBudget: number = 190000; // Default for o4-mini
private contextResult: IContextResult = {
@@ -29,6 +35,13 @@ export class EnhancedContext {
this.projectDir = projectDirArg;
this.configManager = ConfigManager.getInstance();
this.trimmer = new ContextTrimmer(this.configManager.getTrimConfig());
this.lazyLoader = new LazyFileLoader(projectDirArg);
this.cache = new ContextCache(projectDirArg, this.configManager.getCacheConfig());
this.analyzer = new ContextAnalyzer(
projectDirArg,
this.configManager.getPrioritizationWeights(),
this.configManager.getTierConfig()
);
}
/**
@@ -38,6 +51,7 @@ export class EnhancedContext {
await this.configManager.initialize(this.projectDir);
this.tokenBudget = this.configManager.getMaxTokens();
this.trimmer.updateConfig(this.configManager.getTrimConfig());
await this.cache.init();
}
/**
@@ -138,13 +152,28 @@ export class EnhancedContext {
let totalTokenCount = 0;
let totalOriginalTokens = 0;
// Sort files by importance (for now just a simple alphabetical sort)
// Later this could be enhanced with more sophisticated prioritization
const sortedFiles = [...files].sort((a, b) => a.relative.localeCompare(b.relative));
// Convert SmartFile objects to IFileMetadata for analysis
const metadata: IFileMetadata[] = files.map(sf => ({
path: sf.path,
relativePath: sf.relative,
size: sf.contents.toString().length,
mtime: Date.now(), // SmartFile doesn't expose mtime, use current time
estimatedTokens: this.countTokens(sf.contents.toString()),
importanceScore: 0
}));
// Analyze files using ContextAnalyzer to get smart prioritization
// (Note: This requires task type which we'll pass from buildContext)
// For now, sort files by estimated tokens (smaller files first for better efficiency)
const sortedFiles = [...files].sort((a, b) => {
const aTokens = this.countTokens(a.contents.toString());
const bTokens = this.countTokens(b.contents.toString());
return aTokens - bTokens;
});
const processedFiles: string[] = [];
for (const smartfile of sortedFiles) {
// Calculate original token count
const originalContent = smartfile.contents.toString();
@@ -215,6 +244,154 @@ ${processedContent}
return context;
}
/**
* Convert files to context with smart analysis and prioritization
* @param metadata - File metadata to analyze
* @param taskType - Task type for context-aware prioritization
* @param mode - Context mode to use
* @returns Context string
*/
public async convertFilesToContextWithAnalysis(
metadata: IFileMetadata[],
taskType: TaskType,
mode: ContextMode = this.contextMode
): Promise<string> {
// Reset context result
this.contextResult = {
context: '',
tokenCount: 0,
includedFiles: [],
trimmedFiles: [],
excludedFiles: [],
tokenSavings: 0
};
// Analyze files for smart prioritization
const analysis = await this.analyzer.analyze(metadata, taskType, []);
// Sort files by importance score (highest first)
const sortedAnalysis = [...analysis.files].sort(
(a, b) => b.importanceScore - a.importanceScore
);
// Filter out excluded tier
const relevantFiles = sortedAnalysis.filter(f => f.tier !== 'excluded');
let totalTokenCount = 0;
let totalOriginalTokens = 0;
const processedFiles: string[] = [];
// Load files with cache support
for (const fileAnalysis of relevantFiles) {
try {
// Check cache first
let contents: string;
let originalTokenCount: number;
const cached = await this.cache.get(fileAnalysis.path);
if (cached) {
contents = cached.contents;
originalTokenCount = cached.tokenCount;
} else {
// Load file
const fileData = await plugins.smartfile.fs.toStringSync(fileAnalysis.path);
contents = fileData;
originalTokenCount = this.countTokens(contents);
// Cache it
await this.cache.set({
path: fileAnalysis.path,
contents,
tokenCount: originalTokenCount,
mtime: Date.now(),
cachedAt: Date.now()
});
}
totalOriginalTokens += originalTokenCount;
// Apply tier-based trimming
let processedContent = contents;
let trimLevel: 'none' | 'light' | 'aggressive' = 'light';
if (fileAnalysis.tier === 'essential') {
trimLevel = 'none';
} else if (fileAnalysis.tier === 'important') {
trimLevel = 'light';
} else if (fileAnalysis.tier === 'optional') {
trimLevel = 'aggressive';
}
// Apply trimming based on mode and tier
if (mode !== 'full' && trimLevel !== 'none') {
const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
processedContent = this.trimmer.trimFileWithLevel(
relativePath,
contents,
trimLevel
);
}
// Calculate token count
const processedTokenCount = this.countTokens(processedContent);
// Check token budget
if (totalTokenCount + processedTokenCount > this.tokenBudget) {
// We don't have budget for this file
const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
this.contextResult.excludedFiles.push({
path: fileAnalysis.path,
contents,
relativePath,
tokenCount: originalTokenCount,
importanceScore: fileAnalysis.importanceScore
});
continue;
}
// Format the file for context
const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
const formattedContent = `
====== START OF FILE ${relativePath} ======
${processedContent}
====== END OF FILE ${relativePath} ======
`;
processedFiles.push(formattedContent);
totalTokenCount += processedTokenCount;
// Track file in appropriate list
const fileInfo: IFileInfo = {
path: fileAnalysis.path,
contents: processedContent,
relativePath,
tokenCount: processedTokenCount,
importanceScore: fileAnalysis.importanceScore
};
if (trimLevel === 'none' || processedContent === contents) {
this.contextResult.includedFiles.push(fileInfo);
} else {
this.contextResult.trimmedFiles.push(fileInfo);
this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
}
} catch (error) {
console.warn(`Failed to process file ${fileAnalysis.path}:`, error.message);
}
}
// Join all processed files
const context = processedFiles.join('\n');
// Update context result
this.contextResult.context = context;
this.contextResult.tokenCount = totalTokenCount;
return context;
}
/**
* Build context for the project
* @param taskType Optional task type for task-specific context
@@ -233,42 +410,71 @@ ${processedContent}
}
}
// Gather files
const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
const files = await this.gatherFiles(
taskConfig?.includePaths,
taskConfig?.excludePaths
);
// Convert files to context
// Create an array of all files to process
const allFiles: plugins.smartfile.SmartFile[] = [];
// Add individual files
if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
// Add arrays of files
if (files.smartfilesMod) {
if (Array.isArray(files.smartfilesMod)) {
allFiles.push(...files.smartfilesMod);
} else {
allFiles.push(files.smartfilesMod);
// Check if analyzer is enabled in config
const analyzerConfig = this.configManager.getAnalyzerConfig();
const useAnalyzer = analyzerConfig.enabled && taskType;
if (useAnalyzer) {
// Use new smart context building with lazy loading and analysis
const taskConfig = this.configManager.getTaskConfig(taskType!);
// Build globs for scanning
const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [
'ts/**/*.ts',
'ts*/**/*.ts'
];
// Add config files
const configGlobs = [
'package.json',
'readme.md',
'readme.hints.md',
'npmextra.json'
];
// Scan files for metadata (fast, doesn't load contents)
const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]);
// Use analyzer to build context with smart prioritization
await this.convertFilesToContextWithAnalysis(metadata, taskType!, this.contextMode);
} else {
// Fall back to old method for backward compatibility
const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
const files = await this.gatherFiles(
taskConfig?.includePaths,
taskConfig?.excludePaths
);
// Convert files to context
// Create an array of all files to process
const allFiles: plugins.smartfile.SmartFile[] = [];
// Add individual files
if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
// Add arrays of files
if (files.smartfilesMod) {
if (Array.isArray(files.smartfilesMod)) {
allFiles.push(...files.smartfilesMod);
} else {
allFiles.push(files.smartfilesMod);
}
}
}
if (files.smartfilesTest) {
if (Array.isArray(files.smartfilesTest)) {
allFiles.push(...files.smartfilesTest);
} else {
allFiles.push(files.smartfilesTest);
if (files.smartfilesTest) {
if (Array.isArray(files.smartfilesTest)) {
allFiles.push(...files.smartfilesTest);
} else {
allFiles.push(files.smartfilesTest);
}
}
await this.convertFilesToContext(allFiles);
}
const context = await this.convertFilesToContext(allFiles);
return this.contextResult;
}