feat(docs): Update project metadata and documentation to reflect comprehensive AI-enhanced features and improved installation and usage instructions
This commit is contained in:
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@git.zone/tsdoc',
|
||||
version: '1.4.5',
|
||||
description: 'An advanced TypeScript documentation tool using AI to generate and enhance documentation for TypeScript projects.'
|
||||
version: '1.5.0',
|
||||
description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
|
||||
}
|
||||
|
||||
@ -31,15 +31,27 @@ export class Commit {
|
||||
'pnpm-lock.yaml',
|
||||
'package-lock.json',
|
||||
]);
|
||||
const projectContext = new ProjectContext(this.projectDir);
|
||||
let contextString = await projectContext.update();
|
||||
contextString = `
|
||||
${contextString}
|
||||
|
||||
Below is the diff of the uncommitted changes. If nothing is changed, there are no changes:
|
||||
|
||||
${diffStringArray[0] ? diffStringArray.join('\n\n') : 'No changes.'}
|
||||
`;
|
||||
// Use the new TaskContextFactory for optimized context
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
|
||||
await taskContextFactory.initialize();
|
||||
|
||||
// Generate context specifically for commit task
|
||||
const contextResult = await taskContextFactory.createContextForCommit(
|
||||
diffStringArray[0] ? diffStringArray.join('\n\n') : 'No changes.'
|
||||
);
|
||||
|
||||
// Get the optimized context string
|
||||
let contextString = contextResult.context;
|
||||
|
||||
// Log token usage statistics
|
||||
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
|
||||
|
||||
// Check for token overflow against model limits
|
||||
const MODEL_TOKEN_LIMIT = 200000; // o4-mini
|
||||
if (contextResult.tokenCount > MODEL_TOKEN_LIMIT * 0.9) {
|
||||
console.log(`⚠️ Warning: Context size (${contextResult.tokenCount} tokens) is close to or exceeds model limit (${MODEL_TOKEN_LIMIT} tokens).`);
|
||||
console.log(`The model may not be able to process all information effectively.`);
|
||||
}
|
||||
|
||||
let result = await this.aiDocsRef.openaiInstance.chat({
|
||||
systemMessage: `
|
||||
|
||||
@ -18,9 +18,16 @@ export class Description {
|
||||
}
|
||||
|
||||
public async build() {
|
||||
// we can now assemble the directory structure.
|
||||
const projectContext = new ProjectContext(this.projectDir);
|
||||
const contextString = await projectContext.update();
|
||||
// Use the new TaskContextFactory for optimized context
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
|
||||
await taskContextFactory.initialize();
|
||||
|
||||
// Generate context specifically for description task
|
||||
const contextResult = await taskContextFactory.createContextForDescription();
|
||||
const contextString = contextResult.context;
|
||||
|
||||
// Log token usage statistics
|
||||
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
|
||||
|
||||
let result = await this.aiDocsRef.openaiInstance.chat({
|
||||
systemMessage: `
|
||||
@ -48,7 +55,11 @@ Don't wrap the JSON in three ticks json!!!
|
||||
result.message.replace('```json', '').replace('```', ''),
|
||||
);
|
||||
|
||||
const npmextraJson = (await projectContext.gatherFiles()).smartfilesNpmextraJSON;
|
||||
// Create a standard ProjectContext instance for file operations
|
||||
const projectContext = new ProjectContext(this.projectDir);
|
||||
const files = await projectContext.gatherFiles();
|
||||
|
||||
const npmextraJson = files.smartfilesNpmextraJSON;
|
||||
const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString());
|
||||
|
||||
npmextraJsonContent.gitzone.module.description = resultObject.description;
|
||||
@ -58,7 +69,7 @@ Don't wrap the JSON in three ticks json!!!
|
||||
await npmextraJson.write();
|
||||
|
||||
// do the same with packageJson
|
||||
const packageJson = (await projectContext.gatherFiles()).smartfilePackageJSON;
|
||||
const packageJson = files.smartfilePackageJSON;
|
||||
const packageJsonContent = JSON.parse(packageJson.contents.toString());
|
||||
packageJsonContent.description = resultObject.description;
|
||||
packageJsonContent.keywords = resultObject.keywords;
|
||||
|
||||
@ -5,6 +5,8 @@ export class ProjectContext {
|
||||
|
||||
// INSTANCE
|
||||
public projectDir: string;
|
||||
private tokenCount: number = 0;
|
||||
private contextString: string = '';
|
||||
|
||||
constructor(projectDirArg: string) {
|
||||
this.projectDir = projectDirArg;
|
||||
@ -63,6 +65,24 @@ ${smartfile.contents.toString()}
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the token count for a string using the GPT tokenizer
|
||||
* @param text The text to count tokens for
|
||||
* @param model The model to use for token counting (default: gpt-3.5-turbo)
|
||||
* @returns The number of tokens in the text
|
||||
*/
|
||||
public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
|
||||
try {
|
||||
// Use the gpt-tokenizer library to count tokens
|
||||
const tokens = plugins.gptTokenizer.encode(text);
|
||||
return tokens.length;
|
||||
} catch (error) {
|
||||
console.error('Error counting tokens:', error);
|
||||
// Provide a rough estimate (4 chars per token) if tokenization fails
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
}
|
||||
|
||||
private async buildContext(dirArg: string) {
|
||||
const files = await this.gatherFiles();
|
||||
let context = await this.convertFilesToContext([
|
||||
@ -73,10 +93,33 @@ ${smartfile.contents.toString()}
|
||||
...files.smartfilesMod,
|
||||
...files.smartfilesTest,
|
||||
]);
|
||||
// Count tokens in the context
|
||||
this.contextString = context;
|
||||
this.tokenCount = this.countTokens(context);
|
||||
|
||||
// console.log(context);
|
||||
return context;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the token count for the current context
|
||||
* @returns The number of tokens in the context
|
||||
*/
|
||||
public getTokenCount(): number {
|
||||
return this.tokenCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get both the context string and its token count
|
||||
* @returns An object containing the context string and token count
|
||||
*/
|
||||
public getContextWithTokenCount(): { context: string; tokenCount: number } {
|
||||
return {
|
||||
context: this.contextString,
|
||||
tokenCount: this.tokenCount
|
||||
};
|
||||
}
|
||||
|
||||
public async update() {
|
||||
const result = await this.buildContext(this.projectDir);
|
||||
return result;
|
||||
|
||||
@ -17,11 +17,19 @@ export class Readme {
|
||||
public async build() {
|
||||
let finalReadmeString = ``;
|
||||
|
||||
// we can now assemble the directory structure.
|
||||
const projectContext = new ProjectContext(this.projectDir);
|
||||
const contextString = await projectContext.update();
|
||||
// Use the new TaskContextFactory for optimized context
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
|
||||
await taskContextFactory.initialize();
|
||||
|
||||
// Generate context specifically for readme task
|
||||
const contextResult = await taskContextFactory.createContextForReadme();
|
||||
const contextString = contextResult.context;
|
||||
|
||||
// Log token usage statistics
|
||||
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
|
||||
|
||||
// lets first check legal before introducung any cost
|
||||
const projectContext = new ProjectContext(this.projectDir);
|
||||
const npmExtraJson = JSON.parse(
|
||||
(await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString()
|
||||
);
|
||||
|
||||
@ -94,4 +94,37 @@ export class AiDoc {
|
||||
const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
|
||||
return await projectContextInstance.gatherFiles();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the context with token count information
|
||||
* @param projectDirArg The path to the project directory
|
||||
* @returns An object containing the context string and its token count
|
||||
*/
|
||||
public async getProjectContextWithTokenCount(projectDirArg: string) {
|
||||
const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
|
||||
await projectContextInstance.update();
|
||||
return projectContextInstance.getContextWithTokenCount();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get just the token count for a project's context
|
||||
* @param projectDirArg The path to the project directory
|
||||
* @returns The number of tokens in the project context
|
||||
*/
|
||||
public async getProjectContextTokenCount(projectDirArg: string) {
|
||||
const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
|
||||
await projectContextInstance.update();
|
||||
return projectContextInstance.getTokenCount();
|
||||
}
|
||||
|
||||
/**
|
||||
* Count tokens in a text string using GPT tokenizer
|
||||
* @param text The text to count tokens for
|
||||
* @param model The model to use for tokenization (default: gpt-3.5-turbo)
|
||||
* @returns The number of tokens in the text
|
||||
*/
|
||||
public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
|
||||
const projectContextInstance = new aiDocsClasses.ProjectContext('');
|
||||
return projectContextInstance.countTokens(text, model);
|
||||
}
|
||||
}
|
||||
|
||||
128
ts/cli.ts
128
ts/cli.ts
@ -4,6 +4,7 @@ import { logger } from './logging.js';
|
||||
|
||||
import { TypeDoc } from './classes.typedoc.js';
|
||||
import { AiDoc } from './classes.aidoc.js';
|
||||
import * as context from './context/index.js';
|
||||
|
||||
export const run = async () => {
|
||||
const tsdocCli = new plugins.smartcli.Smartcli();
|
||||
@ -30,6 +31,18 @@ export const run = async () => {
|
||||
tsdocCli.addCommand('aidoc').subscribe(async (argvArg) => {
|
||||
const aidocInstance = new AiDoc();
|
||||
await aidocInstance.start();
|
||||
|
||||
// Get context token count if requested
|
||||
if (argvArg.tokens || argvArg.showTokens) {
|
||||
logger.log('info', `Calculating context token count...`);
|
||||
const tokenCount = await aidocInstance.getProjectContextTokenCount(paths.cwd);
|
||||
logger.log('ok', `Total context token count: ${tokenCount}`);
|
||||
|
||||
if (argvArg.tokensOnly) {
|
||||
return; // Exit early if we only want token count
|
||||
}
|
||||
}
|
||||
|
||||
logger.log('info', `Generating new readme...`);
|
||||
logger.log('info', `This may take some time...`);
|
||||
await aidocInstance.buildReadme(paths.cwd);
|
||||
@ -38,6 +51,121 @@ export const run = async () => {
|
||||
await aidocInstance.buildDescription(paths.cwd);
|
||||
});
|
||||
|
||||
tsdocCli.addCommand('tokens').subscribe(async (argvArg) => {
|
||||
const aidocInstance = new AiDoc();
|
||||
await aidocInstance.start();
|
||||
|
||||
logger.log('info', `Calculating context token count...`);
|
||||
|
||||
// Determine context mode based on args
|
||||
let contextMode: context.ContextMode = 'full';
|
||||
if (argvArg.trim || argvArg.trimmed) {
|
||||
contextMode = 'trimmed';
|
||||
} else if (argvArg.summarize || argvArg.summarized) {
|
||||
contextMode = 'summarized';
|
||||
}
|
||||
|
||||
// Get task type if specified
|
||||
let taskType: context.TaskType | undefined = undefined;
|
||||
if (argvArg.task) {
|
||||
if (['readme', 'commit', 'description'].includes(argvArg.task)) {
|
||||
taskType = argvArg.task as context.TaskType;
|
||||
} else {
|
||||
logger.log('warn', `Unknown task type: ${argvArg.task}. Using default context.`);
|
||||
}
|
||||
}
|
||||
|
||||
// Use enhanced context
|
||||
const taskFactory = new context.TaskContextFactory(paths.cwd);
|
||||
await taskFactory.initialize();
|
||||
|
||||
let contextResult: context.IContextResult;
|
||||
|
||||
if (argvArg.all) {
|
||||
// Show stats for all task types
|
||||
const stats = await taskFactory.getTokenStats();
|
||||
|
||||
logger.log('ok', 'Token statistics by task:');
|
||||
for (const [task, data] of Object.entries(stats)) {
|
||||
logger.log('info', `\n${task.toUpperCase()}:`);
|
||||
logger.log('info', ` Tokens: ${data.tokenCount}`);
|
||||
logger.log('info', ` Token savings: ${data.savings}`);
|
||||
logger.log('info', ` Files: ${data.includedFiles} included, ${data.trimmedFiles} trimmed, ${data.excludedFiles} excluded`);
|
||||
|
||||
// Calculate percentage of model context
|
||||
const o4MiniPercentage = (data.tokenCount / 200000 * 100).toFixed(2);
|
||||
logger.log('info', ` Context usage: ${o4MiniPercentage}% of o4-mini (200K tokens)`);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (taskType) {
|
||||
// Get context for specific task
|
||||
contextResult = await taskFactory.createContextForTask(taskType);
|
||||
} else {
|
||||
// Get generic context with specified mode
|
||||
const enhancedContext = new context.EnhancedContext(paths.cwd);
|
||||
await enhancedContext.initialize();
|
||||
enhancedContext.setContextMode(contextMode);
|
||||
|
||||
if (argvArg.maxTokens) {
|
||||
enhancedContext.setTokenBudget(parseInt(argvArg.maxTokens, 10));
|
||||
}
|
||||
|
||||
contextResult = await enhancedContext.buildContext();
|
||||
}
|
||||
|
||||
// Display results
|
||||
logger.log('ok', `Total context token count: ${contextResult.tokenCount}`);
|
||||
logger.log('info', `Files included: ${contextResult.includedFiles.length}`);
|
||||
logger.log('info', `Files trimmed: ${contextResult.trimmedFiles.length}`);
|
||||
logger.log('info', `Files excluded: ${contextResult.excludedFiles.length}`);
|
||||
logger.log('info', `Token savings: ${contextResult.tokenSavings}`);
|
||||
|
||||
if (argvArg.detailed) {
|
||||
// Show more detailed info about the context and token usage
|
||||
const o4MiniPercentage = (contextResult.tokenCount / 200000 * 100).toFixed(2);
|
||||
logger.log('info', `Token usage: ${o4MiniPercentage}% of o4-mini 200K token context window`);
|
||||
|
||||
if (argvArg.model) {
|
||||
// Show percentages for different models
|
||||
if (argvArg.model === 'gpt4') {
|
||||
const gpt4Percentage = (contextResult.tokenCount / 8192 * 100).toFixed(2);
|
||||
logger.log('info', `Token usage (GPT-4): ${gpt4Percentage}% of 8192 token context window`);
|
||||
} else if (argvArg.model === 'gpt35') {
|
||||
const gpt35Percentage = (contextResult.tokenCount / 4096 * 100).toFixed(2);
|
||||
logger.log('info', `Token usage (GPT-3.5): ${gpt35Percentage}% of 4096 token context window`);
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate cost (approximate values)
|
||||
const o4MiniInputCost = 0.00005; // per 1K tokens for o4-mini
|
||||
const estimatedCost = (contextResult.tokenCount / 1000 * o4MiniInputCost).toFixed(6);
|
||||
logger.log('info', `Estimated input cost: $${estimatedCost} (o4-mini)`);
|
||||
|
||||
if (argvArg.listFiles) {
|
||||
// List files included in context
|
||||
logger.log('info', '\nIncluded files:');
|
||||
contextResult.includedFiles.forEach(file => {
|
||||
logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`);
|
||||
});
|
||||
|
||||
logger.log('info', '\nTrimmed files:');
|
||||
contextResult.trimmedFiles.forEach(file => {
|
||||
logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`);
|
||||
});
|
||||
|
||||
if (contextResult.excludedFiles.length > 0) {
|
||||
logger.log('info', '\nExcluded files:');
|
||||
contextResult.excludedFiles.forEach(file => {
|
||||
logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
tsdocCli.addCommand('test').subscribe((argvArg) => {
|
||||
tsdocCli.triggerCommand('typedoc', argvArg);
|
||||
process.on('exit', async () => {
|
||||
|
||||
209
ts/context/config-manager.ts
Normal file
209
ts/context/config-manager.ts
Normal file
@ -0,0 +1,209 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IContextConfig, ITrimConfig, ITaskConfig, TaskType, ContextMode } from './types.js';
|
||||
|
||||
/**
|
||||
* Manages configuration for context building
|
||||
*/
|
||||
export class ConfigManager {
|
||||
private static instance: ConfigManager;
|
||||
private config: IContextConfig;
|
||||
private projectDir: string = '';
|
||||
|
||||
/**
|
||||
* Get the singleton instance of ConfigManager
|
||||
*/
|
||||
public static getInstance(): ConfigManager {
|
||||
if (!ConfigManager.instance) {
|
||||
ConfigManager.instance = new ConfigManager();
|
||||
}
|
||||
return ConfigManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Private constructor for singleton pattern
|
||||
*/
|
||||
private constructor() {
|
||||
this.config = this.getDefaultConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the config manager with a project directory
|
||||
* @param projectDir The project directory
|
||||
*/
|
||||
public async initialize(projectDir: string): Promise<void> {
|
||||
this.projectDir = projectDir;
|
||||
await this.loadConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the default configuration
|
||||
*/
|
||||
private getDefaultConfig(): IContextConfig {
|
||||
return {
|
||||
maxTokens: 190000, // Default for o4-mini with some buffer
|
||||
defaultMode: 'trimmed',
|
||||
taskSpecificSettings: {
|
||||
readme: {
|
||||
mode: 'trimmed',
|
||||
includePaths: ['ts/', 'src/'],
|
||||
excludePaths: ['test/', 'node_modules/']
|
||||
},
|
||||
commit: {
|
||||
mode: 'trimmed',
|
||||
focusOnChangedFiles: true
|
||||
},
|
||||
description: {
|
||||
mode: 'trimmed',
|
||||
includePackageInfo: true
|
||||
}
|
||||
},
|
||||
trimming: {
|
||||
removeImplementations: true,
|
||||
preserveInterfaces: true,
|
||||
preserveTypeDefs: true,
|
||||
preserveJSDoc: true,
|
||||
maxFunctionLines: 5,
|
||||
removeComments: true,
|
||||
removeBlankLines: true
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Load configuration from npmextra.json
|
||||
*/
|
||||
private async loadConfig(): Promise<void> {
|
||||
try {
|
||||
if (!this.projectDir) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Create KeyValueStore for this project
|
||||
// We'll just use smartfile directly instead of KeyValueStore
|
||||
|
||||
// Read the npmextra.json file
|
||||
const npmextraJsonFile = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'npmextra.json')
|
||||
);
|
||||
const npmextraContent = JSON.parse(npmextraJsonFile.contents.toString());
|
||||
|
||||
// Check for tsdoc context configuration
|
||||
if (npmextraContent?.tsdoc?.context) {
|
||||
// Merge with default config
|
||||
this.config = this.mergeConfigs(this.config, npmextraContent.tsdoc.context);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error loading context configuration:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge configurations, with userConfig taking precedence
|
||||
* @param defaultConfig The default configuration
|
||||
* @param userConfig The user configuration
|
||||
*/
|
||||
private mergeConfigs(defaultConfig: IContextConfig, userConfig: Partial<IContextConfig>): IContextConfig {
|
||||
const result: IContextConfig = { ...defaultConfig };
|
||||
|
||||
// Merge top-level properties
|
||||
if (userConfig.maxTokens !== undefined) result.maxTokens = userConfig.maxTokens;
|
||||
if (userConfig.defaultMode !== undefined) result.defaultMode = userConfig.defaultMode;
|
||||
|
||||
// Merge task-specific settings
|
||||
if (userConfig.taskSpecificSettings) {
|
||||
result.taskSpecificSettings = result.taskSpecificSettings || {};
|
||||
|
||||
// For each task type, merge settings
|
||||
(['readme', 'commit', 'description'] as TaskType[]).forEach(taskType => {
|
||||
if (userConfig.taskSpecificSettings?.[taskType]) {
|
||||
result.taskSpecificSettings![taskType] = {
|
||||
...result.taskSpecificSettings![taskType],
|
||||
...userConfig.taskSpecificSettings[taskType]
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Merge trimming configuration
|
||||
if (userConfig.trimming) {
|
||||
result.trimming = {
|
||||
...result.trimming,
|
||||
...userConfig.trimming
|
||||
};
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the complete configuration
|
||||
*/
|
||||
public getConfig(): IContextConfig {
|
||||
return this.config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the trimming configuration
|
||||
*/
|
||||
public getTrimConfig(): ITrimConfig {
|
||||
return this.config.trimming || {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get configuration for a specific task
|
||||
* @param taskType The type of task
|
||||
*/
|
||||
public getTaskConfig(taskType: TaskType): ITaskConfig {
|
||||
// Get task-specific config or empty object
|
||||
const taskConfig = this.config.taskSpecificSettings?.[taskType] || {};
|
||||
|
||||
// If mode is not specified, use default mode
|
||||
if (!taskConfig.mode) {
|
||||
taskConfig.mode = this.config.defaultMode;
|
||||
}
|
||||
|
||||
return taskConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the maximum tokens allowed for context
|
||||
*/
|
||||
public getMaxTokens(): number {
|
||||
return this.config.maxTokens || 190000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the configuration
|
||||
* @param config The new configuration
|
||||
*/
|
||||
public async updateConfig(config: Partial<IContextConfig>): Promise<void> {
|
||||
// Merge with existing config
|
||||
this.config = this.mergeConfigs(this.config, config);
|
||||
|
||||
try {
|
||||
if (!this.projectDir) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Read the existing npmextra.json file
|
||||
const npmextraJsonPath = plugins.path.join(this.projectDir, 'npmextra.json');
|
||||
let npmextraContent = {};
|
||||
|
||||
if (await plugins.smartfile.fs.fileExists(npmextraJsonPath)) {
|
||||
const npmextraJsonFile = await plugins.smartfile.SmartFile.fromFilePath(npmextraJsonPath);
|
||||
npmextraContent = JSON.parse(npmextraJsonFile.contents.toString()) || {};
|
||||
}
|
||||
|
||||
// Update the tsdoc context configuration
|
||||
const typedContent = npmextraContent as any;
|
||||
if (!typedContent.tsdoc) typedContent.tsdoc = {};
|
||||
typedContent.tsdoc.context = this.config;
|
||||
|
||||
// Write back to npmextra.json
|
||||
const updatedContent = JSON.stringify(npmextraContent, null, 2);
|
||||
await plugins.smartfile.memory.toFs(updatedContent, npmextraJsonPath);
|
||||
} catch (error) {
|
||||
console.error('Error updating context configuration:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
246
ts/context/context-trimmer.ts
Normal file
246
ts/context/context-trimmer.ts
Normal file
@ -0,0 +1,246 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { ITrimConfig, ContextMode } from './types.js';
|
||||
|
||||
/**
|
||||
* Class responsible for trimming file contents to reduce token usage
|
||||
* while preserving important information for context
|
||||
*/
|
||||
export class ContextTrimmer {
|
||||
private config: ITrimConfig;
|
||||
|
||||
/**
|
||||
* Create a new ContextTrimmer with the given configuration
|
||||
* @param config The trimming configuration
|
||||
*/
|
||||
constructor(config?: ITrimConfig) {
|
||||
this.config = {
|
||||
removeImplementations: true,
|
||||
preserveInterfaces: true,
|
||||
preserveTypeDefs: true,
|
||||
preserveJSDoc: true,
|
||||
maxFunctionLines: 5,
|
||||
removeComments: true,
|
||||
removeBlankLines: true,
|
||||
...config
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim a file's contents based on the configuration
|
||||
* @param filePath The path to the file
|
||||
* @param content The file's contents
|
||||
* @param mode The context mode to use
|
||||
* @returns The trimmed file contents
|
||||
*/
|
||||
public trimFile(filePath: string, content: string, mode: ContextMode = 'trimmed'): string {
|
||||
// If mode is 'full', return the original content
|
||||
if (mode === 'full') {
|
||||
return content;
|
||||
}
|
||||
|
||||
// Process based on file type
|
||||
if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) {
|
||||
return this.trimTypeScriptFile(content);
|
||||
} else if (filePath.endsWith('.md')) {
|
||||
return this.trimMarkdownFile(content);
|
||||
} else if (filePath.endsWith('.json')) {
|
||||
return this.trimJsonFile(content);
|
||||
}
|
||||
|
||||
// Default to returning the original content for unknown file types
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim a TypeScript file to reduce token usage
|
||||
* @param content The TypeScript file contents
|
||||
* @returns The trimmed file contents
|
||||
*/
|
||||
private trimTypeScriptFile(content: string): string {
|
||||
let result = content;
|
||||
|
||||
// Step 1: Preserve JSDoc comments if configured
|
||||
const jsDocComments: string[] = [];
|
||||
if (this.config.preserveJSDoc) {
|
||||
const jsDocRegex = /\/\*\*[\s\S]*?\*\//g;
|
||||
const matches = result.match(jsDocRegex) || [];
|
||||
jsDocComments.push(...matches);
|
||||
}
|
||||
|
||||
// Step 2: Remove comments if configured
|
||||
if (this.config.removeComments) {
|
||||
// Remove single-line comments
|
||||
result = result.replace(/\/\/.*$/gm, '');
|
||||
// Remove multi-line comments (except JSDoc if preserveJSDoc is true)
|
||||
if (!this.config.preserveJSDoc) {
|
||||
result = result.replace(/\/\*[\s\S]*?\*\//g, '');
|
||||
} else {
|
||||
// Only remove non-JSDoc comments
|
||||
result = result.replace(/\/\*(?!\*)[\s\S]*?\*\//g, '');
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Remove function implementations if configured
|
||||
if (this.config.removeImplementations) {
|
||||
// Match function and method bodies
|
||||
result = result.replace(
|
||||
/(\b(function|constructor|async function)\s+[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
|
||||
(match, start, funcType, body, end) => {
|
||||
// Keep function signature and opening brace, replace body with comment
|
||||
return `${start} /* implementation removed */ ${end}`;
|
||||
}
|
||||
);
|
||||
|
||||
// Match arrow function bodies
|
||||
result = result.replace(
|
||||
/(\([^)]*\)\s*=>\s*{)([\s\S]*?)(})/g,
|
||||
(match, start, body, end) => {
|
||||
return `${start} /* implementation removed */ ${end}`;
|
||||
}
|
||||
);
|
||||
|
||||
// Match method declarations
|
||||
result = result.replace(
|
||||
/(^\s*[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/gm,
|
||||
(match, start, body, end) => {
|
||||
return `${start} /* implementation removed */ ${end}`;
|
||||
}
|
||||
);
|
||||
|
||||
// Match class methods
|
||||
result = result.replace(
|
||||
/(\b(public|private|protected|static|async)?\s+[\w$]+\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
|
||||
(match, start, modifier, body, end) => {
|
||||
return `${start} /* implementation removed */ ${end}`;
|
||||
}
|
||||
);
|
||||
} else if (this.config.maxFunctionLines && this.config.maxFunctionLines > 0) {
|
||||
// If not removing implementations completely, limit the number of lines
|
||||
// Match function and method bodies
|
||||
result = result.replace(
|
||||
/(\b(function|constructor|async function)\s+[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
|
||||
(match, start, funcType, body, end) => {
|
||||
return this.limitFunctionBody(start, body, end);
|
||||
}
|
||||
);
|
||||
|
||||
// Match arrow function bodies
|
||||
result = result.replace(
|
||||
/(\([^)]*\)\s*=>\s*{)([\s\S]*?)(})/g,
|
||||
(match, start, body, end) => {
|
||||
return this.limitFunctionBody(start, body, end);
|
||||
}
|
||||
);
|
||||
|
||||
// Match method declarations
|
||||
result = result.replace(
|
||||
/(^\s*[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/gm,
|
||||
(match, start, body, end) => {
|
||||
return this.limitFunctionBody(start, body, end);
|
||||
}
|
||||
);
|
||||
|
||||
// Match class methods
|
||||
result = result.replace(
|
||||
/(\b(public|private|protected|static|async)?\s+[\w$]+\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
|
||||
(match, start, modifier, body, end) => {
|
||||
return this.limitFunctionBody(start, body, end);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Step 4: Remove blank lines if configured
|
||||
if (this.config.removeBlankLines) {
|
||||
result = result.replace(/^\s*[\r\n]/gm, '');
|
||||
}
|
||||
|
||||
// Step 5: Restore preserved JSDoc comments
|
||||
if (this.config.preserveJSDoc && jsDocComments.length > 0) {
|
||||
// This is a placeholder; we already preserved JSDoc comments in the regex steps
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Limit a function body to a maximum number of lines
|
||||
* @param start The function signature and opening brace
|
||||
* @param body The function body
|
||||
* @param end The closing brace
|
||||
* @returns The limited function body
|
||||
*/
|
||||
private limitFunctionBody(start: string, body: string, end: string): string {
|
||||
const lines = body.split('\n');
|
||||
if (lines.length > this.config.maxFunctionLines!) {
|
||||
const limitedBody = lines.slice(0, this.config.maxFunctionLines!).join('\n');
|
||||
return `${start}${limitedBody}\n // ... (${lines.length - this.config.maxFunctionLines!} lines trimmed)\n${end}`;
|
||||
}
|
||||
return `${start}${body}${end}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim a Markdown file to reduce token usage
|
||||
* @param content The Markdown file contents
|
||||
* @returns The trimmed file contents
|
||||
*/
|
||||
private trimMarkdownFile(content: string): string {
|
||||
// For markdown files, we generally want to keep most content
|
||||
// but we can remove lengthy code blocks if needed
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim a JSON file to reduce token usage
|
||||
* @param content The JSON file contents
|
||||
* @returns The trimmed file contents
|
||||
*/
|
||||
private trimJsonFile(content: string): string {
|
||||
try {
|
||||
// Parse the JSON
|
||||
const json = JSON.parse(content);
|
||||
|
||||
// For package.json, keep only essential information
|
||||
if ('name' in json && 'version' in json && 'dependencies' in json) {
|
||||
const essentialKeys = [
|
||||
'name', 'version', 'description', 'author', 'license',
|
||||
'main', 'types', 'exports', 'type'
|
||||
];
|
||||
|
||||
const trimmedJson: any = {};
|
||||
essentialKeys.forEach(key => {
|
||||
if (key in json) {
|
||||
trimmedJson[key] = json[key];
|
||||
}
|
||||
});
|
||||
|
||||
// Add dependency information without versions
|
||||
if ('dependencies' in json) {
|
||||
trimmedJson.dependencies = Object.keys(json.dependencies).reduce((acc, dep) => {
|
||||
acc[dep] = '*'; // Replace version with wildcard
|
||||
return acc;
|
||||
}, {} as Record<string, string>);
|
||||
}
|
||||
|
||||
// Return the trimmed JSON
|
||||
return JSON.stringify(trimmedJson, null, 2);
|
||||
}
|
||||
|
||||
// For other JSON files, leave as is
|
||||
return content;
|
||||
} catch (error) {
|
||||
// If there's an error parsing the JSON, return the original content
|
||||
return content;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the trimmer configuration
|
||||
* @param config The new configuration to apply
|
||||
*/
|
||||
public updateConfig(config: ITrimConfig): void {
|
||||
this.config = {
|
||||
...this.config,
|
||||
...config
|
||||
};
|
||||
}
|
||||
}
|
||||
343
ts/context/enhanced-context.ts
Normal file
343
ts/context/enhanced-context.ts
Normal file
@ -0,0 +1,343 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { ContextMode, IContextResult, IFileInfo, TaskType } from './types.js';
|
||||
import { ContextTrimmer } from './context-trimmer.js';
|
||||
import { ConfigManager } from './config-manager.js';
|
||||
|
||||
/**
|
||||
* Enhanced ProjectContext that supports context optimization strategies
|
||||
*/
|
||||
export class EnhancedContext {
|
||||
private projectDir: string;
|
||||
private trimmer: ContextTrimmer;
|
||||
private configManager: ConfigManager;
|
||||
private contextMode: ContextMode = 'trimmed';
|
||||
private tokenBudget: number = 190000; // Default for o4-mini
|
||||
private contextResult: IContextResult = {
|
||||
context: '',
|
||||
tokenCount: 0,
|
||||
includedFiles: [],
|
||||
trimmedFiles: [],
|
||||
excludedFiles: [],
|
||||
tokenSavings: 0
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a new EnhancedContext
|
||||
* @param projectDirArg The project directory
|
||||
*/
|
||||
constructor(projectDirArg: string) {
|
||||
this.projectDir = projectDirArg;
|
||||
this.configManager = ConfigManager.getInstance();
|
||||
this.trimmer = new ContextTrimmer(this.configManager.getTrimConfig());
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the context builder
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
await this.configManager.initialize(this.projectDir);
|
||||
this.tokenBudget = this.configManager.getMaxTokens();
|
||||
this.trimmer.updateConfig(this.configManager.getTrimConfig());
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the context mode
|
||||
* @param mode The context mode to use
|
||||
*/
|
||||
public setContextMode(mode: ContextMode): void {
|
||||
this.contextMode = mode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the token budget
|
||||
* @param maxTokens The maximum tokens to use
|
||||
*/
|
||||
public setTokenBudget(maxTokens: number): void {
|
||||
this.tokenBudget = maxTokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gather files from the project
|
||||
* @param includePaths Optional paths to include
|
||||
* @param excludePaths Optional paths to exclude
|
||||
*/
|
||||
public async gatherFiles(includePaths?: string[], excludePaths?: string[]): Promise<Record<string, plugins.smartfile.SmartFile | plugins.smartfile.SmartFile[]>> {
|
||||
const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'package.json'),
|
||||
this.projectDir,
|
||||
);
|
||||
|
||||
const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'readme.md'),
|
||||
this.projectDir,
|
||||
);
|
||||
|
||||
const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'readme.hints.md'),
|
||||
this.projectDir,
|
||||
);
|
||||
|
||||
const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'npmextra.json'),
|
||||
this.projectDir,
|
||||
);
|
||||
|
||||
// Use provided include paths or default to all TypeScript files
|
||||
const includeGlobs = includePaths?.map(path => `${path}/**/*.ts`) || ['ts*/**/*.ts'];
|
||||
|
||||
// Get TypeScript files
|
||||
const smartfilesModPromises = includeGlobs.map(glob =>
|
||||
plugins.smartfile.fs.fileTreeToObject(this.projectDir, glob)
|
||||
);
|
||||
|
||||
const smartfilesModArrays = await Promise.all(smartfilesModPromises);
|
||||
|
||||
// Flatten the arrays
|
||||
const smartfilesMod: plugins.smartfile.SmartFile[] = [];
|
||||
smartfilesModArrays.forEach(array => {
|
||||
smartfilesMod.push(...array);
|
||||
});
|
||||
|
||||
// Get test files if not excluded
|
||||
let smartfilesTest: plugins.smartfile.SmartFile[] = [];
|
||||
if (!excludePaths?.includes('test/')) {
|
||||
smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(
|
||||
this.projectDir,
|
||||
'test/**/*.ts',
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
smartfilePackageJSON,
|
||||
smartfilesReadme,
|
||||
smartfilesReadmeHints,
|
||||
smartfilesNpmextraJSON,
|
||||
smartfilesMod,
|
||||
smartfilesTest,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert files to context string
|
||||
* @param files The files to convert
|
||||
* @param mode The context mode to use
|
||||
*/
|
||||
public async convertFilesToContext(
|
||||
files: plugins.smartfile.SmartFile[],
|
||||
mode: ContextMode = this.contextMode
|
||||
): Promise<string> {
|
||||
// Reset context result
|
||||
this.contextResult = {
|
||||
context: '',
|
||||
tokenCount: 0,
|
||||
includedFiles: [],
|
||||
trimmedFiles: [],
|
||||
excludedFiles: [],
|
||||
tokenSavings: 0
|
||||
};
|
||||
|
||||
let totalTokenCount = 0;
|
||||
let totalOriginalTokens = 0;
|
||||
|
||||
// Sort files by importance (for now just a simple alphabetical sort)
|
||||
// Later this could be enhanced with more sophisticated prioritization
|
||||
const sortedFiles = [...files].sort((a, b) => a.relative.localeCompare(b.relative));
|
||||
|
||||
const processedFiles: string[] = [];
|
||||
|
||||
for (const smartfile of sortedFiles) {
|
||||
// Calculate original token count
|
||||
const originalContent = smartfile.contents.toString();
|
||||
const originalTokenCount = this.countTokens(originalContent);
|
||||
totalOriginalTokens += originalTokenCount;
|
||||
|
||||
// Apply trimming based on mode
|
||||
let processedContent = originalContent;
|
||||
|
||||
if (mode !== 'full') {
|
||||
processedContent = this.trimmer.trimFile(
|
||||
smartfile.relative,
|
||||
originalContent,
|
||||
mode
|
||||
);
|
||||
}
|
||||
|
||||
// Calculate new token count
|
||||
const processedTokenCount = this.countTokens(processedContent);
|
||||
|
||||
// Check if we have budget for this file
|
||||
if (totalTokenCount + processedTokenCount > this.tokenBudget) {
|
||||
// We don't have budget for this file
|
||||
this.contextResult.excludedFiles.push({
|
||||
path: smartfile.path,
|
||||
contents: originalContent,
|
||||
relativePath: smartfile.relative,
|
||||
tokenCount: originalTokenCount
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Format the file for context
|
||||
const formattedContent = `
|
||||
====== START OF FILE ${smartfile.relative} ======
|
||||
|
||||
${processedContent}
|
||||
|
||||
====== END OF FILE ${smartfile.relative} ======
|
||||
`;
|
||||
|
||||
processedFiles.push(formattedContent);
|
||||
totalTokenCount += processedTokenCount;
|
||||
|
||||
// Track file in appropriate list
|
||||
const fileInfo: IFileInfo = {
|
||||
path: smartfile.path,
|
||||
contents: processedContent,
|
||||
relativePath: smartfile.relative,
|
||||
tokenCount: processedTokenCount
|
||||
};
|
||||
|
||||
if (mode === 'full' || processedContent === originalContent) {
|
||||
this.contextResult.includedFiles.push(fileInfo);
|
||||
} else {
|
||||
this.contextResult.trimmedFiles.push(fileInfo);
|
||||
this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
|
||||
}
|
||||
}
|
||||
|
||||
// Join all processed files
|
||||
const context = processedFiles.join('\n');
|
||||
|
||||
// Update context result
|
||||
this.contextResult.context = context;
|
||||
this.contextResult.tokenCount = totalTokenCount;
|
||||
|
||||
return context;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build context for the project
|
||||
* @param taskType Optional task type for task-specific context
|
||||
*/
|
||||
public async buildContext(taskType?: TaskType): Promise<IContextResult> {
|
||||
// Initialize if needed
|
||||
if (this.tokenBudget === 0) {
|
||||
await this.initialize();
|
||||
}
|
||||
|
||||
// Get task-specific configuration if a task type is provided
|
||||
if (taskType) {
|
||||
const taskConfig = this.configManager.getTaskConfig(taskType);
|
||||
if (taskConfig.mode) {
|
||||
this.setContextMode(taskConfig.mode);
|
||||
}
|
||||
}
|
||||
|
||||
// Gather files
|
||||
const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
|
||||
const files = await this.gatherFiles(
|
||||
taskConfig?.includePaths,
|
||||
taskConfig?.excludePaths
|
||||
);
|
||||
|
||||
// Convert files to context
|
||||
// Create an array of all files to process
|
||||
const allFiles: plugins.smartfile.SmartFile[] = [];
|
||||
|
||||
// Add individual files
|
||||
if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
|
||||
if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
|
||||
if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
|
||||
if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
|
||||
|
||||
// Add arrays of files
|
||||
if (files.smartfilesMod) {
|
||||
if (Array.isArray(files.smartfilesMod)) {
|
||||
allFiles.push(...files.smartfilesMod);
|
||||
} else {
|
||||
allFiles.push(files.smartfilesMod);
|
||||
}
|
||||
}
|
||||
|
||||
if (files.smartfilesTest) {
|
||||
if (Array.isArray(files.smartfilesTest)) {
|
||||
allFiles.push(...files.smartfilesTest);
|
||||
} else {
|
||||
allFiles.push(files.smartfilesTest);
|
||||
}
|
||||
}
|
||||
|
||||
const context = await this.convertFilesToContext(allFiles);
|
||||
|
||||
return this.contextResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the context with git diff information for commit tasks
|
||||
* @param gitDiff The git diff to include
|
||||
*/
|
||||
public updateWithGitDiff(gitDiff: string): IContextResult {
|
||||
// If we don't have a context yet, return empty result
|
||||
if (!this.contextResult.context) {
|
||||
return this.contextResult;
|
||||
}
|
||||
|
||||
// Add git diff to context
|
||||
const diffSection = `
|
||||
====== GIT DIFF ======
|
||||
|
||||
${gitDiff}
|
||||
|
||||
====== END GIT DIFF ======
|
||||
`;
|
||||
|
||||
const diffTokenCount = this.countTokens(diffSection);
|
||||
|
||||
// Update context and token count
|
||||
this.contextResult.context += diffSection;
|
||||
this.contextResult.tokenCount += diffTokenCount;
|
||||
|
||||
return this.contextResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count tokens in a string
|
||||
* @param text The text to count tokens for
|
||||
* @param model The model to use for token counting
|
||||
*/
|
||||
public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
|
||||
try {
|
||||
// Use the gpt-tokenizer library to count tokens
|
||||
const tokens = plugins.gptTokenizer.encode(text);
|
||||
return tokens.length;
|
||||
} catch (error) {
|
||||
console.error('Error counting tokens:', error);
|
||||
// Provide a rough estimate if tokenization fails
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the context result
|
||||
*/
|
||||
public getContextResult(): IContextResult {
|
||||
return this.contextResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the token count for the current context
|
||||
*/
|
||||
public getTokenCount(): number {
|
||||
return this.contextResult.tokenCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get both the context string and its token count
|
||||
*/
|
||||
public getContextWithTokenCount(): { context: string; tokenCount: number } {
|
||||
return {
|
||||
context: this.contextResult.context,
|
||||
tokenCount: this.contextResult.tokenCount
|
||||
};
|
||||
}
|
||||
}
|
||||
32
ts/context/index.ts
Normal file
32
ts/context/index.ts
Normal file
@ -0,0 +1,32 @@
|
||||
import { EnhancedContext } from './enhanced-context.js';
|
||||
import { TaskContextFactory } from './task-context-factory.js';
|
||||
import { ConfigManager } from './config-manager.js';
|
||||
import { ContextTrimmer } from './context-trimmer.js';
|
||||
import type {
|
||||
ContextMode,
|
||||
IContextConfig,
|
||||
IContextResult,
|
||||
IFileInfo,
|
||||
ITrimConfig,
|
||||
ITaskConfig,
|
||||
TaskType
|
||||
} from './types.js';
|
||||
|
||||
export {
|
||||
// Classes
|
||||
EnhancedContext,
|
||||
TaskContextFactory,
|
||||
ConfigManager,
|
||||
ContextTrimmer,
|
||||
};
|
||||
|
||||
// Types
|
||||
export type {
|
||||
ContextMode,
|
||||
IContextConfig,
|
||||
IContextResult,
|
||||
IFileInfo,
|
||||
ITrimConfig,
|
||||
ITaskConfig,
|
||||
TaskType
|
||||
};
|
||||
138
ts/context/task-context-factory.ts
Normal file
138
ts/context/task-context-factory.ts
Normal file
@ -0,0 +1,138 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { EnhancedContext } from './enhanced-context.js';
|
||||
import { ConfigManager } from './config-manager.js';
|
||||
import type { IContextResult, TaskType } from './types.js';
|
||||
|
||||
/**
|
||||
* Factory class for creating task-specific context
|
||||
*/
|
||||
export class TaskContextFactory {
|
||||
private projectDir: string;
|
||||
private configManager: ConfigManager;
|
||||
|
||||
/**
|
||||
* Create a new TaskContextFactory
|
||||
* @param projectDirArg The project directory
|
||||
*/
|
||||
constructor(projectDirArg: string) {
|
||||
this.projectDir = projectDirArg;
|
||||
this.configManager = ConfigManager.getInstance();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the factory
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
await this.configManager.initialize(this.projectDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create context for README generation
|
||||
*/
|
||||
public async createContextForReadme(): Promise<IContextResult> {
|
||||
const contextBuilder = new EnhancedContext(this.projectDir);
|
||||
await contextBuilder.initialize();
|
||||
|
||||
// Get README-specific configuration
|
||||
const taskConfig = this.configManager.getTaskConfig('readme');
|
||||
if (taskConfig.mode) {
|
||||
contextBuilder.setContextMode(taskConfig.mode);
|
||||
}
|
||||
|
||||
// Build the context for README task
|
||||
return await contextBuilder.buildContext('readme');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create context for description generation
|
||||
*/
|
||||
public async createContextForDescription(): Promise<IContextResult> {
|
||||
const contextBuilder = new EnhancedContext(this.projectDir);
|
||||
await contextBuilder.initialize();
|
||||
|
||||
// Get description-specific configuration
|
||||
const taskConfig = this.configManager.getTaskConfig('description');
|
||||
if (taskConfig.mode) {
|
||||
contextBuilder.setContextMode(taskConfig.mode);
|
||||
}
|
||||
|
||||
// Build the context for description task
|
||||
return await contextBuilder.buildContext('description');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create context for commit message generation
|
||||
* @param gitDiff Optional git diff to include
|
||||
*/
|
||||
public async createContextForCommit(gitDiff?: string): Promise<IContextResult> {
|
||||
const contextBuilder = new EnhancedContext(this.projectDir);
|
||||
await contextBuilder.initialize();
|
||||
|
||||
// Get commit-specific configuration
|
||||
const taskConfig = this.configManager.getTaskConfig('commit');
|
||||
if (taskConfig.mode) {
|
||||
contextBuilder.setContextMode(taskConfig.mode);
|
||||
}
|
||||
|
||||
// Build the context for commit task
|
||||
const contextResult = await contextBuilder.buildContext('commit');
|
||||
|
||||
// If git diff is provided, add it to the context
|
||||
if (gitDiff) {
|
||||
contextBuilder.updateWithGitDiff(gitDiff);
|
||||
}
|
||||
|
||||
return contextBuilder.getContextResult();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create context for any task type
|
||||
* @param taskType The task type to create context for
|
||||
* @param additionalContent Optional additional content to include
|
||||
*/
|
||||
public async createContextForTask(
|
||||
taskType: TaskType,
|
||||
additionalContent?: string
|
||||
): Promise<IContextResult> {
|
||||
switch (taskType) {
|
||||
case 'readme':
|
||||
return this.createContextForReadme();
|
||||
case 'description':
|
||||
return this.createContextForDescription();
|
||||
case 'commit':
|
||||
return this.createContextForCommit(additionalContent);
|
||||
default:
|
||||
// Generic context for unknown task types
|
||||
const contextBuilder = new EnhancedContext(this.projectDir);
|
||||
await contextBuilder.initialize();
|
||||
return await contextBuilder.buildContext();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get token stats for all task types
|
||||
*/
|
||||
public async getTokenStats(): Promise<Record<TaskType, {
|
||||
tokenCount: number;
|
||||
savings: number;
|
||||
includedFiles: number;
|
||||
trimmedFiles: number;
|
||||
excludedFiles: number;
|
||||
}>> {
|
||||
const taskTypes: TaskType[] = ['readme', 'description', 'commit'];
|
||||
const stats: Record<TaskType, any> = {} as any;
|
||||
|
||||
for (const taskType of taskTypes) {
|
||||
const result = await this.createContextForTask(taskType);
|
||||
stats[taskType] = {
|
||||
tokenCount: result.tokenCount,
|
||||
savings: result.tokenSavings,
|
||||
includedFiles: result.includedFiles.length,
|
||||
trimmedFiles: result.trimmedFiles.length,
|
||||
excludedFiles: result.excludedFiles.length
|
||||
};
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
95
ts/context/types.ts
Normal file
95
ts/context/types.ts
Normal file
@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Context processing mode to control how context is built
|
||||
*/
|
||||
export type ContextMode = 'full' | 'trimmed' | 'summarized';
|
||||
|
||||
/**
|
||||
* Configuration for context trimming
|
||||
*/
|
||||
export interface ITrimConfig {
|
||||
/** Whether to remove function implementations */
|
||||
removeImplementations?: boolean;
|
||||
/** Whether to preserve interface definitions */
|
||||
preserveInterfaces?: boolean;
|
||||
/** Whether to preserve type definitions */
|
||||
preserveTypeDefs?: boolean;
|
||||
/** Whether to preserve JSDoc comments */
|
||||
preserveJSDoc?: boolean;
|
||||
/** Maximum lines to keep for function bodies (if not removing completely) */
|
||||
maxFunctionLines?: number;
|
||||
/** Whether to remove normal comments (non-JSDoc) */
|
||||
removeComments?: boolean;
|
||||
/** Whether to remove blank lines */
|
||||
removeBlankLines?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Task types that require different context optimization
|
||||
*/
|
||||
export type TaskType = 'readme' | 'commit' | 'description';
|
||||
|
||||
/**
|
||||
* Configuration for different tasks
|
||||
*/
|
||||
export interface ITaskConfig {
|
||||
/** The context mode to use for this task */
|
||||
mode?: ContextMode;
|
||||
/** File paths to include for this task */
|
||||
includePaths?: string[];
|
||||
/** File paths to exclude for this task */
|
||||
excludePaths?: string[];
|
||||
/** For commit tasks, whether to focus on changed files */
|
||||
focusOnChangedFiles?: boolean;
|
||||
/** For description tasks, whether to include package info */
|
||||
includePackageInfo?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete context configuration
|
||||
*/
|
||||
export interface IContextConfig {
|
||||
/** Maximum tokens to use for context */
|
||||
maxTokens?: number;
|
||||
/** Default context mode */
|
||||
defaultMode?: ContextMode;
|
||||
/** Task-specific settings */
|
||||
taskSpecificSettings?: {
|
||||
[key in TaskType]?: ITaskConfig;
|
||||
};
|
||||
/** Trimming configuration */
|
||||
trimming?: ITrimConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic file information interface
|
||||
*/
|
||||
export interface IFileInfo {
|
||||
/** The file path */
|
||||
path: string;
|
||||
/** The file contents */
|
||||
contents: string;
|
||||
/** The file's relative path from the project root */
|
||||
relativePath: string;
|
||||
/** The estimated token count of the file */
|
||||
tokenCount?: number;
|
||||
/** The file's importance score (higher is more important) */
|
||||
importanceScore?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of context building
|
||||
*/
|
||||
export interface IContextResult {
|
||||
/** The generated context string */
|
||||
context: string;
|
||||
/** The total token count of the context */
|
||||
tokenCount: number;
|
||||
/** Files included in the context */
|
||||
includedFiles: IFileInfo[];
|
||||
/** Files that were trimmed */
|
||||
trimmedFiles: IFileInfo[];
|
||||
/** Files that were excluded */
|
||||
excludedFiles: IFileInfo[];
|
||||
/** Token savings from trimming */
|
||||
tokenSavings: number;
|
||||
}
|
||||
@ -41,5 +41,6 @@ export { tspublish };
|
||||
|
||||
// third party scope
|
||||
import * as typedoc from 'typedoc';
|
||||
import * as gptTokenizer from 'gpt-tokenizer';
|
||||
|
||||
export { typedoc };
|
||||
export { typedoc, gptTokenizer };
|
||||
|
||||
Reference in New Issue
Block a user