From bcded1eafac788a2dc1c1f85c49d7b555ac4d271 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Mon, 15 Dec 2025 14:34:02 +0000 Subject: [PATCH] update --- package.json | 3 +- pnpm-lock.yaml | 10 +- test/test.diffprocessor.node.ts | 2 +- test/test.iterativecontextbuilder.node.ts | 147 ------- test/test.lazyfileloader.node.ts | 243 ---------- ts/aidocs_classes/commit.ts | 62 +-- ts/aidocs_classes/description.ts | 108 +++-- ts/aidocs_classes/projectcontext.ts | 21 +- ts/aidocs_classes/readme.ts | 239 ++++++---- ts/classes.aidoc.ts | 29 +- ts/cli.ts | 120 +---- ts/context/config-manager.ts | 369 ---------------- ts/context/context-analyzer.ts | 391 ----------------- ts/context/context-cache.ts | 286 ------------ ts/context/context-trimmer.ts | 310 ------------- ts/context/enhanced-context.ts | 332 -------------- ts/context/index.ts | 70 --- ts/context/iterative-context-builder.ts | 512 ---------------------- ts/context/lazy-file-loader.ts | 207 --------- ts/context/task-context-factory.ts | 120 ----- ts/context/types.ts | 324 -------------- ts/plugins.ts | 3 +- 22 files changed, 288 insertions(+), 3620 deletions(-) delete mode 100644 test/test.iterativecontextbuilder.node.ts delete mode 100644 test/test.lazyfileloader.node.ts delete mode 100644 ts/context/config-manager.ts delete mode 100644 ts/context/context-analyzer.ts delete mode 100644 ts/context/context-cache.ts delete mode 100644 ts/context/context-trimmer.ts delete mode 100644 ts/context/enhanced-context.ts delete mode 100644 ts/context/index.ts delete mode 100644 ts/context/iterative-context-builder.ts delete mode 100644 ts/context/lazy-file-loader.ts delete mode 100644 ts/context/task-context-factory.ts delete mode 100644 ts/context/types.ts diff --git a/package.json b/package.json index dbde369..76bd195 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "@push.rocks/early": "^4.0.4", "@push.rocks/npmextra": "^5.3.3", "@push.rocks/qenv": "^6.1.3", - "@push.rocks/smartagent": "^1.1.1", + "@push.rocks/smartagent": "file:../../push.rocks/smartagent", "@push.rocks/smartai": "^0.8.0", "@push.rocks/smartcli": "^4.0.19", "@push.rocks/smartdelay": "^3.0.5", @@ -42,7 +42,6 @@ "@push.rocks/smartpath": "^6.0.0", "@push.rocks/smartshell": "^3.3.0", "@push.rocks/smarttime": "^4.1.1", - "gpt-tokenizer": "^3.4.0", "typedoc": "^0.28.15", "typescript": "^5.9.3" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0598a1f..2f4c20b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: ^6.1.3 version: 6.1.3 '@push.rocks/smartagent': - specifier: ^1.1.1 - version: 1.1.1(typescript@5.9.3)(ws@8.18.3)(zod@3.25.76) + specifier: file:../../push.rocks/smartagent + version: file:../../push.rocks/smartagent(typescript@5.9.3)(ws@8.18.3)(zod@3.25.76) '@push.rocks/smartai': specifier: ^0.8.0 version: 0.8.0(typescript@5.9.3)(ws@8.18.3)(zod@3.25.76) @@ -1158,8 +1158,8 @@ packages: '@push.rocks/qenv@6.1.3': resolution: {integrity: sha512-+z2hsAU/7CIgpYLFqvda8cn9rUBMHqLdQLjsFfRn5jPoD7dJ5rFlpkbhfM4Ws8mHMniwWaxGKo+q/YBhtzRBLg==} - '@push.rocks/smartagent@1.1.1': - resolution: {integrity: sha512-N/lY+MQX8B7NvLOiJTRmv79KmJ0PUGOcQkkn94frEHLSY7yTCPT7OFEjgmTxkKL04MlILATBmYaS1bIHVxPYAA==} + '@push.rocks/smartagent@file:../../push.rocks/smartagent': + resolution: {directory: ../../push.rocks/smartagent, type: directory} '@push.rocks/smartai@0.8.0': resolution: {integrity: sha512-guzi28meUDc3mydC8kpoA+4pzExRQqygXYFDD4qQSWPpIRHQ7qhpeNqJzrrGezT1yOH5Gb9taPEGwT56hI+nwQ==} @@ -6926,7 +6926,7 @@ snapshots: '@push.rocks/smartlog': 3.1.10 '@push.rocks/smartpath': 6.0.0 - '@push.rocks/smartagent@1.1.1(typescript@5.9.3)(ws@8.18.3)(zod@3.25.76)': + '@push.rocks/smartagent@file:../../push.rocks/smartagent(typescript@5.9.3)(ws@8.18.3)(zod@3.25.76)': dependencies: '@push.rocks/smartai': 0.8.0(typescript@5.9.3)(ws@8.18.3)(zod@3.25.76) '@push.rocks/smartbrowser': 2.0.8(typescript@5.9.3) diff --git a/test/test.diffprocessor.node.ts b/test/test.diffprocessor.node.ts index e62e16d..433bb74 100644 --- a/test/test.diffprocessor.node.ts +++ b/test/test.diffprocessor.node.ts @@ -1,5 +1,5 @@ import { tap, expect } from '@git.zone/tstest/tapbundle'; -import { DiffProcessor } from '../ts/context/diff-processor.js'; +import { DiffProcessor } from '../ts/classes.diffprocessor.js'; // Sample diff strings for testing const createSmallDiff = (filepath: string, addedLines = 5, removedLines = 3): string => { diff --git a/test/test.iterativecontextbuilder.node.ts b/test/test.iterativecontextbuilder.node.ts deleted file mode 100644 index 36f2c44..0000000 --- a/test/test.iterativecontextbuilder.node.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { tap, expect } from '@git.zone/tstest/tapbundle'; -import * as path from 'path'; -import { IterativeContextBuilder } from '../ts/context/iterative-context-builder.js'; -import type { IIterativeConfig, TaskType } from '../ts/context/types.js'; -import * as qenv from '@push.rocks/qenv'; - -// Test project directory -const testProjectRoot = path.join(process.cwd()); - -// Helper to check if OPENAI_TOKEN is available -async function hasOpenAIToken(): Promise { - try { - const qenvInstance = new qenv.Qenv(); - const token = await qenvInstance.getEnvVarOnDemand('OPENAI_TOKEN'); - return !!token; - } catch (error) { - return false; - } -} - -tap.test('IterativeContextBuilder should create instance with default config', async () => { - const builder = new IterativeContextBuilder(testProjectRoot); - expect(builder).toBeInstanceOf(IterativeContextBuilder); -}); - -tap.test('IterativeContextBuilder should create instance with custom config', async () => { - const customConfig: Partial = { - maxIterations: 3, - firstPassFileLimit: 5, - subsequentPassFileLimit: 3, - temperature: 0.5, - model: 'gpt-4', - }; - const builder = new IterativeContextBuilder(testProjectRoot, customConfig); - expect(builder).toBeInstanceOf(IterativeContextBuilder); -}); - -tap.test('IterativeContextBuilder should initialize successfully', async () => { - if (!(await hasOpenAIToken())) { - console.log('⚠️ Skipping initialization test - OPENAI_TOKEN not available'); - return; - } - - const builder = new IterativeContextBuilder(testProjectRoot); - await builder.initialize(); - // If we get here without error, initialization succeeded - expect(true).toEqual(true); -}); - -tap.test('IterativeContextBuilder should build context iteratively for readme task', async () => { - if (!(await hasOpenAIToken())) { - console.log('⚠️ Skipping iterative build test - OPENAI_TOKEN not available'); - return; - } - - const builder = new IterativeContextBuilder(testProjectRoot, { - maxIterations: 2, // Limit iterations for testing - firstPassFileLimit: 3, - subsequentPassFileLimit: 2, - }); - - await builder.initialize(); - - const result = await builder.buildContextIteratively('readme'); - - // Verify result structure - expect(result).toBeTypeOf('object'); - expect(result.context).toBeTypeOf('string'); - expect(result.context.length).toBeGreaterThan(0); - expect(result.tokenCount).toBeTypeOf('number'); - expect(result.tokenCount).toBeGreaterThan(0); - expect(result.includedFiles).toBeInstanceOf(Array); - expect(result.includedFiles.length).toBeGreaterThan(0); - expect(result.iterationCount).toBeTypeOf('number'); - expect(result.iterationCount).toBeGreaterThan(0); - expect(result.iterationCount).toBeLessThanOrEqual(2); - expect(result.iterations).toBeInstanceOf(Array); - expect(result.iterations.length).toEqual(result.iterationCount); - expect(result.apiCallCount).toBeTypeOf('number'); - expect(result.apiCallCount).toBeGreaterThan(0); - expect(result.totalDuration).toBeTypeOf('number'); - expect(result.totalDuration).toBeGreaterThan(0); - - // Verify iteration structure - for (const iteration of result.iterations) { - expect(iteration.iteration).toBeTypeOf('number'); - expect(iteration.filesLoaded).toBeInstanceOf(Array); - expect(iteration.tokensUsed).toBeTypeOf('number'); - expect(iteration.totalTokensUsed).toBeTypeOf('number'); - expect(iteration.decision).toBeTypeOf('object'); - expect(iteration.duration).toBeTypeOf('number'); - } - - console.log(`✅ Iterative context build completed:`); - console.log(` Iterations: ${result.iterationCount}`); - console.log(` Files: ${result.includedFiles.length}`); - console.log(` Tokens: ${result.tokenCount}`); - console.log(` API calls: ${result.apiCallCount}`); - console.log(` Duration: ${(result.totalDuration / 1000).toFixed(2)}s`); -}); - -tap.test('IterativeContextBuilder should respect token budget', async () => { - if (!(await hasOpenAIToken())) { - console.log('⚠️ Skipping token budget test - OPENAI_TOKEN not available'); - return; - } - - const builder = new IterativeContextBuilder(testProjectRoot, { - maxIterations: 5, - }); - - await builder.initialize(); - - const result = await builder.buildContextIteratively('description'); - - // Token count should not exceed budget significantly (allow 5% margin for safety) - const configManager = (await import('../ts/context/config-manager.js')).ConfigManager.getInstance(); - const maxTokens = configManager.getMaxTokens(); - expect(result.tokenCount).toBeLessThanOrEqual(maxTokens * 1.05); - - console.log(`✅ Token budget respected: ${result.tokenCount}/${maxTokens}`); -}); - -tap.test('IterativeContextBuilder should work with different task types', async () => { - if (!(await hasOpenAIToken())) { - console.log('⚠️ Skipping task types test - OPENAI_TOKEN not available'); - return; - } - - const taskTypes: TaskType[] = ['readme', 'description', 'commit']; - - for (const taskType of taskTypes) { - const builder = new IterativeContextBuilder(testProjectRoot, { - maxIterations: 2, - firstPassFileLimit: 2, - }); - - await builder.initialize(); - const result = await builder.buildContextIteratively(taskType); - - expect(result.includedFiles.length).toBeGreaterThan(0); - - console.log(`✅ ${taskType}: ${result.includedFiles.length} files, ${result.tokenCount} tokens`); - } -}); - -export default tap.start(); diff --git a/test/test.lazyfileloader.node.ts b/test/test.lazyfileloader.node.ts deleted file mode 100644 index 82550f9..0000000 --- a/test/test.lazyfileloader.node.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { tap, expect } from '@git.zone/tstest/tapbundle'; -import * as path from 'path'; -import { LazyFileLoader } from '../ts/context/lazy-file-loader.js'; -import type { IFileMetadata } from '../ts/context/types.js'; - -const testProjectRoot = process.cwd(); - -tap.test('LazyFileLoader should create instance with project root', async () => { - const loader = new LazyFileLoader(testProjectRoot); - expect(loader).toBeInstanceOf(LazyFileLoader); -}); - -tap.test('LazyFileLoader.getMetadata should return file metadata without loading contents', async () => { - const loader = new LazyFileLoader(testProjectRoot); - const packageJsonPath = path.join(testProjectRoot, 'package.json'); - - const metadata = await loader.getMetadata(packageJsonPath); - - expect(metadata.path).toEqual(packageJsonPath); - expect(metadata.relativePath).toEqual('package.json'); - expect(metadata.size).toBeGreaterThan(0); - expect(metadata.mtime).toBeGreaterThan(0); - expect(metadata.estimatedTokens).toBeGreaterThan(0); - // Rough estimate: size / 4 (with reasonable tolerance) - expect(metadata.estimatedTokens).toBeGreaterThan(metadata.size / 5); - expect(metadata.estimatedTokens).toBeLessThan(metadata.size / 3); -}); - -tap.test('LazyFileLoader.getMetadata should cache metadata for same file', async () => { - const loader = new LazyFileLoader(testProjectRoot); - const packageJsonPath = path.join(testProjectRoot, 'package.json'); - - const metadata1 = await loader.getMetadata(packageJsonPath); - const metadata2 = await loader.getMetadata(packageJsonPath); - - // Should return identical metadata from cache - expect(metadata1.mtime).toEqual(metadata2.mtime); - expect(metadata1.size).toEqual(metadata2.size); - expect(metadata1.estimatedTokens).toEqual(metadata2.estimatedTokens); -}); - -tap.test('LazyFileLoader.scanFiles should scan TypeScript files', async () => { - const loader = new LazyFileLoader(testProjectRoot); - - const metadata = await loader.scanFiles(['ts/context/types.ts']); - - expect(metadata.length).toBeGreaterThan(0); - const typesFile = metadata.find(m => m.relativePath.includes('types.ts')); - expect(typesFile).toBeDefined(); - expect(typesFile!.size).toBeGreaterThan(0); - expect(typesFile!.estimatedTokens).toBeGreaterThan(0); -}); - -tap.test('LazyFileLoader.scanFiles should handle multiple globs', async () => { - const loader = new LazyFileLoader(testProjectRoot); - - const metadata = await loader.scanFiles([ - 'package.json', - 'readme.md' - ]); - - expect(metadata.length).toBeGreaterThanOrEqual(2); - const hasPackageJson = metadata.some(m => m.relativePath === 'package.json'); - const hasReadme = metadata.some(m => m.relativePath.toLowerCase() === 'readme.md'); - expect(hasPackageJson).toEqual(true); - expect(hasReadme).toEqual(true); -}); - -tap.test('LazyFileLoader.loadFile should load file with actual token count', async () => { - const loader = new LazyFileLoader(testProjectRoot); - const packageJsonPath = path.join(testProjectRoot, 'package.json'); - - const tokenizer = (content: string) => Math.ceil(content.length / 4); - const fileInfo = await loader.loadFile(packageJsonPath, tokenizer); - - expect(fileInfo.path).toEqual(packageJsonPath); - expect(fileInfo.contents).toBeDefined(); - expect(fileInfo.contents.length).toBeGreaterThan(0); - expect(fileInfo.tokenCount).toBeGreaterThan(0); - expect(fileInfo.relativePath).toEqual('package.json'); -}); - -tap.test('LazyFileLoader.loadFiles should load multiple files in parallel', async () => { - const loader = new LazyFileLoader(testProjectRoot); - - const metadata: IFileMetadata[] = [ - { - path: path.join(testProjectRoot, 'package.json'), - relativePath: 'package.json', - size: 100, - mtime: Date.now(), - estimatedTokens: 25 - }, - { - path: path.join(testProjectRoot, 'readme.md'), - relativePath: 'readme.md', - size: 200, - mtime: Date.now(), - estimatedTokens: 50 - } - ]; - - const tokenizer = (content: string) => Math.ceil(content.length / 4); - const startTime = Date.now(); - const files = await loader.loadFiles(metadata, tokenizer); - const endTime = Date.now(); - - expect(files.length).toEqual(2); - expect(files[0].contents).toBeDefined(); - expect(files[1].contents).toBeDefined(); - - // Should be fast (parallel loading) - expect(endTime - startTime).toBeLessThan(5000); // 5 seconds max -}); - -tap.test('LazyFileLoader.updateImportanceScores should update cached metadata', async () => { - const loader = new LazyFileLoader(testProjectRoot); - const packageJsonPath = path.join(testProjectRoot, 'package.json'); - - // Get initial metadata - await loader.getMetadata(packageJsonPath); - - // Update importance scores - const scores = new Map(); - scores.set(packageJsonPath, 0.95); - loader.updateImportanceScores(scores); - - // Check cached metadata has updated score - const cached = loader.getCachedMetadata(); - const packageJsonMeta = cached.find(m => m.path === packageJsonPath); - - expect(packageJsonMeta).toBeDefined(); - expect(packageJsonMeta!.importanceScore).toEqual(0.95); -}); - -tap.test('LazyFileLoader.getTotalEstimatedTokens should sum all cached metadata tokens', async () => { - const loader = new LazyFileLoader(testProjectRoot); - - // Scan some files - await loader.scanFiles(['package.json', 'readme.md']); - - const totalTokens = loader.getTotalEstimatedTokens(); - - expect(totalTokens).toBeGreaterThan(0); -}); - -tap.test('LazyFileLoader.clearCache should clear metadata cache', async () => { - const loader = new LazyFileLoader(testProjectRoot); - - // Scan files to populate cache - await loader.scanFiles(['package.json']); - expect(loader.getCachedMetadata().length).toBeGreaterThan(0); - - // Clear cache - loader.clearCache(); - - expect(loader.getCachedMetadata().length).toEqual(0); -}); - -tap.test('LazyFileLoader.getCachedMetadata should return all cached entries', async () => { - const loader = new LazyFileLoader(testProjectRoot); - - // Scan files - await loader.scanFiles(['package.json', 'readme.md']); - - const cached = loader.getCachedMetadata(); - - expect(cached.length).toBeGreaterThanOrEqual(2); - expect(cached.every(m => m.path && m.size && m.estimatedTokens)).toEqual(true); -}); - -tap.test('LazyFileLoader should handle non-existent files gracefully', async () => { - const loader = new LazyFileLoader(testProjectRoot); - const nonExistentPath = path.join(testProjectRoot, 'this-file-does-not-exist.ts'); - - try { - await loader.getMetadata(nonExistentPath); - expect(false).toEqual(true); // Should not reach here - } catch (error) { - expect(error).toBeDefined(); - } -}); - -tap.test('LazyFileLoader.loadFiles should filter out failed file loads', async () => { - const loader = new LazyFileLoader(testProjectRoot); - - const metadata: IFileMetadata[] = [ - { - path: path.join(testProjectRoot, 'package.json'), - relativePath: 'package.json', - size: 100, - mtime: Date.now(), - estimatedTokens: 25 - }, - { - path: path.join(testProjectRoot, 'non-existent-file.txt'), - relativePath: 'non-existent-file.txt', - size: 100, - mtime: Date.now(), - estimatedTokens: 25 - } - ]; - - const tokenizer = (content: string) => Math.ceil(content.length / 4); - const files = await loader.loadFiles(metadata, tokenizer); - - // Should only include the successfully loaded file - expect(files.length).toEqual(1); - expect(files[0].relativePath).toEqual('package.json'); -}); - -tap.test('LazyFileLoader should handle glob patterns for TypeScript source files', async () => { - const loader = new LazyFileLoader(testProjectRoot); - - const metadata = await loader.scanFiles(['ts/context/*.ts']); - - expect(metadata.length).toBeGreaterThan(0); - - // Should find multiple context files - const hasEnhancedContext = metadata.some(m => m.relativePath.includes('enhanced-context.ts')); - const hasTypes = metadata.some(m => m.relativePath.includes('types.ts')); - - expect(hasEnhancedContext).toEqual(true); - expect(hasTypes).toEqual(true); -}); - -tap.test('LazyFileLoader should estimate tokens reasonably accurately', async () => { - const loader = new LazyFileLoader(testProjectRoot); - const packageJsonPath = path.join(testProjectRoot, 'package.json'); - - const metadata = await loader.getMetadata(packageJsonPath); - const tokenizer = (content: string) => Math.ceil(content.length / 4); - const fileInfo = await loader.loadFile(packageJsonPath, tokenizer); - - // Estimated tokens should be close to actual (within reasonable range) - const difference = Math.abs(metadata.estimatedTokens - fileInfo.tokenCount); - const percentDiff = (difference / fileInfo.tokenCount) * 100; - - // Should be within 20% accuracy (since it's just an estimate) - expect(percentDiff).toBeLessThan(20); -}); - -export default tap.start(); diff --git a/ts/aidocs_classes/commit.ts b/ts/aidocs_classes/commit.ts index e493fd0..a868595 100644 --- a/ts/aidocs_classes/commit.ts +++ b/ts/aidocs_classes/commit.ts @@ -1,7 +1,7 @@ import * as plugins from '../plugins.js'; import { AiDoc } from '../classes.aidoc.js'; import { ProjectContext } from './projectcontext.js'; -import { DiffProcessor } from '../context/diff-processor.js'; +import { DiffProcessor } from '../classes.diffprocessor.js'; export interface INextCommitObject { recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project @@ -114,32 +114,10 @@ export class Commit { processedDiffString = 'No changes.'; } - // Use the new TaskContextFactory for optimized context - const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory( - this.projectDir, - this.aiDocsRef.openaiInstance - ); - await taskContextFactory.initialize(); - - // Generate context specifically for commit task - const contextResult = await taskContextFactory.createContextForCommit(processedDiffString); - - // Get the optimized context string - let contextString = contextResult.context; - - // Log token usage statistics - console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`); - - // Check for token overflow against model limits - const MODEL_TOKEN_LIMIT = 200000; // o4-mini - if (contextResult.tokenCount > MODEL_TOKEN_LIMIT * 0.9) { - console.log(`⚠️ Warning: Context size (${contextResult.tokenCount} tokens) is close to or exceeds model limit (${MODEL_TOKEN_LIMIT} tokens).`); - console.log(`The model may not be able to process all information effectively.`); - } - - // Use DualAgentOrchestrator for commit message generation with Guardian validation + // Use DualAgentOrchestrator for commit message generation + // Note: No filesystem tool needed - the diff already contains all change information const commitOrchestrator = new plugins.smartagent.DualAgentOrchestrator({ - openaiToken: this.aiDocsRef.getOpenaiToken(), + smartAiInstance: this.aiDocsRef.smartAiInstance, defaultProvider: 'openai', guardianPolicyPrompt: ` You validate commit messages for semantic versioning compliance. @@ -154,7 +132,7 @@ APPROVE if: REJECT with specific feedback if: - Version level doesn't match the scope of changes (e.g., "feat" for a typo fix should be "fix") - Message is vague, unprofessional, or contains sensitive information -- JSON is malformed or missing required fields (recommendedNextVersionLevel, recommendedNextVersionScope, recommendedNextVersionMessage, recommendedNextVersionDetails, recommendedNextVersion) +- JSON is malformed or missing required fields `, }); @@ -162,9 +140,12 @@ REJECT with specific feedback if: const commitTaskPrompt = ` You create a commit message for a git commit. -The commit message should be based on the files in the project. -You should not include any licensing information. -You should not include any personal information. +Project directory: ${this.projectDir} + +Analyze the git diff below to understand what changed and generate a commit message. + +You should not include any licensing information or personal information. +Never mention CLAUDE code, or codex. Important: Answer only in valid JSON. @@ -173,21 +154,20 @@ Your answer should be parseable with JSON.parse() without modifying anything. Here is the structure of the JSON you should return: interface { - recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project - recommendedNextVersionScope: string; // the recommended scope name of the next version, like "core" or "cli", or specific class names. - recommendedNextVersionMessage: string; // the commit message. Don't put fix() feat() or BREAKING CHANGE in the message. Please just the message itself. + recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level + recommendedNextVersionScope: string; // scope name like "core", "cli", or specific class names + recommendedNextVersionMessage: string; // the commit message (don't include fix/feat prefix) recommendedNextVersionDetails: string[]; // detailed bullet points for the changelog - recommendedNextVersion: string; // the recommended next version of the project, x.x.x + recommendedNextVersion: string; // the recommended next version x.x.x } -For the recommendedNextVersionDetails, please only add a detail entries to the array if it has an obvious value to the reader. +For recommendedNextVersionDetails, only add entries that have obvious value to the reader. -You are being given the files of the project. You should use them to create the commit message. -Also you are given a diff. -Never mention CLAUDE code, or codex. +Here is the git diff showing what changed: -Project context and diff: -${contextString} +${processedDiffString} + +Generate the commit message based on these changes. `; const commitResult = await commitOrchestrator.run(commitTaskPrompt); @@ -214,7 +194,7 @@ ${contextString} // Use DualAgentOrchestrator for changelog generation with Guardian validation const changelogOrchestrator = new plugins.smartagent.DualAgentOrchestrator({ - openaiToken: this.aiDocsRef.getOpenaiToken(), + smartAiInstance: this.aiDocsRef.smartAiInstance, defaultProvider: 'openai', guardianPolicyPrompt: ` You validate changelog generation. diff --git a/ts/aidocs_classes/description.ts b/ts/aidocs_classes/description.ts index 46f61c9..ffaaaa3 100644 --- a/ts/aidocs_classes/description.ts +++ b/ts/aidocs_classes/description.ts @@ -18,50 +18,72 @@ export class Description { } public async build() { - // Use the new TaskContextFactory for optimized context - const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory( - this.projectDir, - this.aiDocsRef.openaiInstance - ); - await taskContextFactory.initialize(); - - // Generate context specifically for description task - const contextResult = await taskContextFactory.createContextForDescription(); - const contextString = contextResult.context; - - // Log token usage statistics - console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`); - - let result = await this.aiDocsRef.openaiInstance.chat({ - systemMessage: ` -You create a json adhering the following interface: -{ - description: string; // a sensible short, one sentence description of the project - keywords: string[]; // an array of tags that describe the project -} - -The description should be based on what you understand from the project's files. -The keywords should be based on use cases you see from the files. -Don't be cheap about the way you think. - -Important: Answer only in valid JSON. -You answer should be parseable with JSON.parse() without modifying anything. - -Don't wrap the JSON in three ticks json!!! - `, - messageHistory: [], - userMessage: contextString, - }); - - console.log(result.message); - const resultObject: IDescriptionInterface = JSON.parse( - result.message.replace('```json', '').replace('```', ''), - ); - - // Create a standard ProjectContext instance for file operations + // Gather project context upfront to avoid token explosion from filesystem tool const projectContext = new ProjectContext(this.projectDir); const files = await projectContext.gatherFiles(); - + const contextString = await projectContext.convertFilesToContext([ + files.smartfilePackageJSON, + files.smartfilesNpmextraJSON, + ...files.smartfilesMod.slice(0, 10), // Limit to first 10 source files for description + ]); + + // Use DualAgentOrchestrator for description generation + const descriptionOrchestrator = new plugins.smartagent.DualAgentOrchestrator({ + smartAiInstance: this.aiDocsRef.smartAiInstance, + defaultProvider: 'openai', + guardianPolicyPrompt: ` +You validate description generation. + +APPROVE if: +- JSON is valid and parseable +- Description is a clear, concise one-sentence summary +- Keywords are relevant to the project's use cases +- Both description and keywords fields are present + +REJECT if: +- JSON is malformed or wrapped in markdown code blocks +- Description is too long or vague +- Keywords are irrelevant or generic +`, + }); + + await descriptionOrchestrator.start(); + + const descriptionTaskPrompt = ` +You create a project description and keywords for an npm package. + +Analyze the project files provided below to understand the codebase, then generate a description and keywords. + +Your response must be valid JSON adhering to this interface: +{ + description: string; // a sensible short, one sentence description of the project + keywords: string[]; // an array of tags that describe the project based on use cases +} + +Important: Answer only in valid JSON. +Your answer should be parseable with JSON.parse() without modifying anything. +Don't wrap the JSON in \`\`\`json\`\`\` - just return the raw JSON object. + +Here are the project files: + +${contextString} + +Generate the description based on these files. +`; + + const descriptionResult = await descriptionOrchestrator.run(descriptionTaskPrompt); + await descriptionOrchestrator.stop(); + + if (!descriptionResult.success) { + throw new Error(`Description generation failed: ${descriptionResult.status}`); + } + + console.log(descriptionResult.result); + const resultObject: IDescriptionInterface = JSON.parse( + descriptionResult.result.replace('```json', '').replace('```', ''), + ); + + // Use the already gathered files for updates const npmextraJson = files.smartfilesNpmextraJSON; const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString()); @@ -82,6 +104,6 @@ Don't wrap the JSON in three ticks json!!! console.log(`\n======================\n`); console.log(JSON.stringify(resultObject, null, 2)); console.log(`\n======================\n`); - return result.message; + return descriptionResult.result; } } diff --git a/ts/aidocs_classes/projectcontext.ts b/ts/aidocs_classes/projectcontext.ts index 4a1b20a..f6f369f 100644 --- a/ts/aidocs_classes/projectcontext.ts +++ b/ts/aidocs_classes/projectcontext.ts @@ -64,21 +64,14 @@ ${smartfile.contents.toString()} } /** - * Calculate the token count for a string using the GPT tokenizer - * @param text The text to count tokens for - * @param model The model to use for token counting (default: gpt-3.5-turbo) - * @returns The number of tokens in the text + * Estimate token count for a string + * Uses a rough estimate of 4 characters per token + * @param text The text to estimate tokens for + * @returns Estimated number of tokens */ - public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number { - try { - // Use the gpt-tokenizer library to count tokens - const tokens = plugins.gptTokenizer.encode(text); - return tokens.length; - } catch (error) { - console.error('Error counting tokens:', error); - // Provide a rough estimate (4 chars per token) if tokenization fails - return Math.ceil(text.length / 4); - } + public countTokens(text: string): number { + // Rough estimate: ~4 characters per token for English text + return Math.ceil(text.length / 4); } private async buildContext(dirArg: string) { diff --git a/ts/aidocs_classes/readme.ts b/ts/aidocs_classes/readme.ts index a005fb5..c825454 100644 --- a/ts/aidocs_classes/readme.ts +++ b/ts/aidocs_classes/readme.ts @@ -17,20 +17,6 @@ export class Readme { public async build() { let finalReadmeString = ``; - // Use the new TaskContextFactory for optimized context - const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory( - this.projectDir, - this.aiDocsRef.openaiInstance - ); - await taskContextFactory.initialize(); - - // Generate context specifically for readme task - const contextResult = await taskContextFactory.createContextForReadme(); - const contextString = contextResult.context; - - // Log token usage statistics - console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`); - // lets first check legal before introducung any cost const projectContext = new ProjectContext(this.projectDir); const npmExtraJson = JSON.parse( @@ -42,50 +28,88 @@ export class Readme { console.log(error); } - let result = await this.aiDocsRef.openaiInstance.chat({ - systemMessage: ` -You create markdown readmes for npm projects. You only output the markdown readme. + // Gather project context upfront to avoid token explosion from filesystem tool + const contextString = await projectContext.convertFilesToContext([ + (await projectContext.gatherFiles()).smartfilePackageJSON, + (await projectContext.gatherFiles()).smartfilesReadme, + (await projectContext.gatherFiles()).smartfilesReadmeHints, + (await projectContext.gatherFiles()).smartfilesNpmextraJSON, + ...(await projectContext.gatherFiles()).smartfilesMod, + ]); -The Readme should follow the following template: + // Use DualAgentOrchestrator for readme generation + const readmeOrchestrator = new plugins.smartagent.DualAgentOrchestrator({ + smartAiInstance: this.aiDocsRef.smartAiInstance, + defaultProvider: 'openai', + guardianPolicyPrompt: ` +You validate README generation. + +APPROVE if: +- README follows proper markdown format +- Contains Install and Usage sections +- Code examples are correct TypeScript/ESM syntax +- Documentation is comprehensive and helpful + +REJECT if: +- README is incomplete or poorly formatted +- Contains licensing information (added separately) +- Uses CommonJS syntax instead of ESM +- Contains "in conclusion" or similar filler +`, + }); + + await readmeOrchestrator.start(); + + const readmeTaskPrompt = ` +You create markdown READMEs for npm projects. You only output the markdown readme. + +Analyze the project files provided below to understand the codebase, then generate a comprehensive README. + +The README should follow this template: # Project Name -[ - The name is the module name of package.json - The description is in the description field of package.json -] +[The name from package.json and description] ## Install -[ - Write a short text on how to install the project -] +[Short text on how to install the project] ## Usage -[ +[ Give code examples here. Construct sensible scenarios for the user. Make sure to show a complete set of features of the module. Don't omit use cases. - It does not matter how much time you need. ALWAYS USE ESM SYNTAX AND TYPESCRIPT. - DON'T CHICKEN OUT. Write at least 4000 words. More if necessary. - If there is already a readme, take the Usage section as base. Remove outdated content, and expand and improve upon the valid parts. - Super important: Check for completenes. - Don't include any licensing information. This will be added in a later step. - Avoid "in conclusions". - - Good to know: - * npmextra.json contains overall module information. - * readme.hints.md provides valuable hints about module ideas. + Write at least 4000 words. More if necessary. + If there is already a readme, take the Usage section as base. Remove outdated content, expand and improve. + Check for completeness. + Don't include any licensing information. This will be added later. + Avoid "in conclusion" statements. ] - `, - messageHistory: [], - userMessage: contextString, - }); - finalReadmeString += result.message + '\n' + legalInfo; +Here are the project files: + +${contextString} + +Generate the README based on these files. +`; + + const readmeResult = await readmeOrchestrator.run(readmeTaskPrompt); + await readmeOrchestrator.stop(); + + if (!readmeResult.success) { + throw new Error(`README generation failed: ${readmeResult.status}`); + } + + // Clean up markdown formatting if wrapped in code blocks + let resultMessage = readmeResult.result + .replace(/^```markdown\n?/i, '') + .replace(/\n?```$/i, ''); + + finalReadmeString += resultMessage + '\n' + legalInfo; console.log(`\n======================\n`); - console.log(result.message); + console.log(resultMessage); console.log(`\n======================\n`); const readme = (await projectContext.gatherFiles()).smartfilesReadme; @@ -96,60 +120,95 @@ The Readme should follow the following template: const tsPublishInstance = new plugins.tspublish.TsPublish(); const subModules = await tsPublishInstance.getModuleSubDirs(paths.cwd); logger.log('info', `Found ${Object.keys(subModules).length} sub modules`); + for (const subModule of Object.keys(subModules)) { logger.log('info', `Building readme for ${subModule}`); - const subModuleContextString = await projectContext.update(); - let result = await this.aiDocsRef.openaiInstance.chat({ - systemMessage: ` - You create markdown readmes for npm projects. You only output the markdown readme. - IMPORTANT: YOU ARE NOW CREATING THE README FOR THE FOLLOWING SUB MODULE: ${subModule} !!!!!!!!!!! - The Sub Module will be published with the following data: - ${JSON.stringify(await plugins.fsInstance.file(plugins.path.join(paths.cwd, subModule, 'tspublish.json')).encoding('utf8').read(), null, 2)} + const tspublishData = await plugins.fsInstance + .file(plugins.path.join(paths.cwd, subModule, 'tspublish.json')) + .encoding('utf8') + .read(); - - The Readme should follow the following template: - - # Project Name - [ - The name is the module name of package.json - The description is in the description field of package.json - ] - - ## Install - [ - Write a short text on how to install the project - ] - - ## Usage - [ - Give code examples here. - Construct sensible scenarios for the user. - Make sure to show a complete set of features of the module. - Don't omit use cases. - It does not matter how much time you need. - ALWAYS USE ESM SYNTAX AND TYPESCRIPT. - DON'T CHICKEN OUT. Write at least 4000 words. More if necessary. - If there is already a readme, take the Usage section as base. Remove outdated content, and expand and improve upon the valid parts. - Super important: Check for completenes. - Don't include any licensing information. This will be added in a later step. - Avoid "in conclusions". - - Good to know: - * npmextra.json contains overall module information. - * readme.hints.md provides valuable hints about module ideas. - * Your output lands directly in the readme.md file. - * Don't use \`\`\` at the beginning or the end. It'll cause problems. Only use it for codeblocks. You are directly writing markdown. No need to introduce it weirdly. - ] - `, - messageHistory: [], - userMessage: subModuleContextString, + // Gather submodule context + const subModuleContext = new ProjectContext(plugins.path.join(paths.cwd, subModule)); + let subModuleContextString = ''; + try { + const subModuleFiles = await subModuleContext.gatherFiles(); + subModuleContextString = await subModuleContext.convertFilesToContext([ + subModuleFiles.smartfilePackageJSON, + subModuleFiles.smartfilesNpmextraJSON, + ...subModuleFiles.smartfilesMod, + ]); + } catch (e) { + // Submodule may not have all files, continue with what we have + logger.log('warn', `Could not gather full context for ${subModule}`); + } + + // Create a new orchestrator for each submodule + const subModuleOrchestrator = new plugins.smartagent.DualAgentOrchestrator({ + smartAiInstance: this.aiDocsRef.smartAiInstance, + defaultProvider: 'openai', + guardianPolicyPrompt: ` +You validate README generation for submodules. + +APPROVE comprehensive, well-formatted markdown with ESM TypeScript examples. +REJECT incomplete READMEs or those with licensing info. +`, }); - const subModuleReadmeString = result.message + '\n' + legalInfo; - await plugins.fsInstance.file(plugins.path.join(paths.cwd, subModule, 'readme.md')).encoding('utf8').write(subModuleReadmeString); - logger.log('success', `Built readme for ${subModule}`); + await subModuleOrchestrator.start(); + + const subModulePrompt = ` +You create markdown READMEs for npm projects. You only output the markdown readme. +SUB MODULE: ${subModule} + +IMPORTANT: YOU ARE CREATING THE README FOR THIS SUB MODULE: ${subModule} +The Sub Module will be published with: +${JSON.stringify(tspublishData, null, 2)} + +Generate a README following the template: + +# Project Name +[name and description from package.json] + +## Install +[installation instructions] + +## Usage +[ + Code examples with complete features. + ESM TypeScript syntax only. + Write at least 4000 words. + No licensing information. + No "in conclusion". +] + +Don't use \`\`\` at the beginning or end. Only for code blocks. + +Here are the submodule files: + +${subModuleContextString} + +Generate the README based on these files. +`; + + const subModuleResult = await subModuleOrchestrator.run(subModulePrompt); + await subModuleOrchestrator.stop(); + + if (subModuleResult.success) { + const subModuleReadmeString = subModuleResult.result + .replace(/^```markdown\n?/i, '') + .replace(/\n?```$/i, '') + '\n' + legalInfo; + await plugins.fsInstance + .file(plugins.path.join(paths.cwd, subModule, 'readme.md')) + .encoding('utf8') + .write(subModuleReadmeString); + logger.log('success', `Built readme for ${subModule}`); + } else { + logger.log('error', `Failed to build readme for ${subModule}: ${subModuleResult.status}`); + } } - return result.message; + + return resultMessage; } } diff --git a/ts/classes.aidoc.ts b/ts/classes.aidoc.ts index 46d054d..f69a684 100644 --- a/ts/classes.aidoc.ts +++ b/ts/classes.aidoc.ts @@ -8,7 +8,7 @@ export class AiDoc { public npmextraKV: plugins.npmextra.KeyValueStore; public qenvInstance: plugins.qenv.Qenv; public aidocInteract: plugins.smartinteract.SmartInteract; - public openaiInstance: plugins.smartai.OpenAiProvider; + public smartAiInstance: plugins.smartai.SmartAi; argvArg: any; @@ -85,20 +85,28 @@ export class AiDoc { } // lets assume we have an OPENAI_Token now - this.openaiInstance = new plugins.smartai.OpenAiProvider({ + this.smartAiInstance = new plugins.smartai.SmartAi({ openaiToken: this.openaiToken, }); - await this.openaiInstance.start(); + await this.smartAiInstance.start(); } public async stop() { - if (this.openaiInstance) { - await this.openaiInstance.stop(); + if (this.smartAiInstance) { + await this.smartAiInstance.stop(); } // No explicit cleanup needed for npmextraKV or aidocInteract // They don't keep event loop alive } + /** + * Get the OpenAI provider for direct chat calls + * This is a convenience getter to access the provider from SmartAi + */ + public get openaiProvider(): plugins.smartai.OpenAiProvider { + return this.smartAiInstance.openaiProvider; + } + public getOpenaiToken(): string { return this.openaiToken; } @@ -146,13 +154,12 @@ export class AiDoc { } /** - * Count tokens in a text string using GPT tokenizer - * @param text The text to count tokens for - * @param model The model to use for tokenization (default: gpt-3.5-turbo) - * @returns The number of tokens in the text + * Estimate token count in a text string + * @param text The text to estimate tokens for + * @returns Estimated number of tokens */ - public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number { + public countTokens(text: string): number { const projectContextInstance = new aiDocsClasses.ProjectContext(''); - return projectContextInstance.countTokens(text, model); + return projectContextInstance.countTokens(text); } } diff --git a/ts/cli.ts b/ts/cli.ts index 67234ec..19327c4 100644 --- a/ts/cli.ts +++ b/ts/cli.ts @@ -4,7 +4,6 @@ import { logger } from './logging.js'; import { TypeDoc } from './classes.typedoc.js'; import { AiDoc } from './classes.aidoc.js'; -import * as context from './context/index.js'; export const run = async () => { const tsdocCli = new plugins.smartcli.Smartcli(); @@ -32,17 +31,6 @@ export const run = async () => { const aidocInstance = new AiDoc(); await aidocInstance.start(); - // Get context token count if requested - if (argvArg.tokens || argvArg.showTokens) { - logger.log('info', `Calculating context token count...`); - const tokenCount = await aidocInstance.getProjectContextTokenCount(paths.cwd); - logger.log('ok', `Total context token count: ${tokenCount}`); - - if (argvArg.tokensOnly) { - return; // Exit early if we only want token count - } - } - logger.log('info', `Generating new readme...`); logger.log('info', `This may take some time...`); await aidocInstance.buildReadme(paths.cwd); @@ -51,102 +39,34 @@ export const run = async () => { await aidocInstance.buildDescription(paths.cwd); }); - tsdocCli.addCommand('tokens').subscribe(async (argvArg) => { + tsdocCli.addCommand('readme').subscribe(async (argvArg) => { const aidocInstance = new AiDoc(); await aidocInstance.start(); - logger.log('info', `Calculating context token count...`); + logger.log('info', `Generating new readme...`); + logger.log('info', `This may take some time...`); + await aidocInstance.buildReadme(paths.cwd); + }); - // Get task type if specified - let taskType: context.TaskType | undefined = undefined; - if (argvArg.task) { - if (['readme', 'commit', 'description'].includes(argvArg.task)) { - taskType = argvArg.task as context.TaskType; - } else { - logger.log('warn', `Unknown task type: ${argvArg.task}. Using default (readme).`); - taskType = 'readme'; - } - } else { - // Default to readme if no task specified - taskType = 'readme'; - } + tsdocCli.addCommand('description').subscribe(async (argvArg) => { + const aidocInstance = new AiDoc(); + await aidocInstance.start(); - // Use iterative context building - const taskFactory = new context.TaskContextFactory(paths.cwd); - await taskFactory.initialize(); + logger.log('info', `Generating new description and keywords...`); + logger.log('info', `This may take some time...`); + await aidocInstance.buildDescription(paths.cwd); + }); - let contextResult: context.IIterativeContextResult; + tsdocCli.addCommand('commit').subscribe(async (argvArg) => { + const aidocInstance = new AiDoc(); + await aidocInstance.start(); - if (argvArg.all) { - // Show stats for all task types - const stats = await taskFactory.getTokenStats(); + logger.log('info', `Generating commit message...`); + logger.log('info', `This may take some time...`); + const commitObject = await aidocInstance.buildNextCommitObject(paths.cwd); - logger.log('ok', 'Token statistics by task:'); - for (const [task, data] of Object.entries(stats)) { - logger.log('info', `\n${task.toUpperCase()}:`); - logger.log('info', ` Tokens: ${data.tokenCount}`); - logger.log('info', ` Token savings: ${data.savings}`); - logger.log('info', ` Files: ${data.includedFiles} included, ${data.trimmedFiles} trimmed, ${data.excludedFiles} excluded`); - - // Calculate percentage of model context - const o4MiniPercentage = (data.tokenCount / 200000 * 100).toFixed(2); - logger.log('info', ` Context usage: ${o4MiniPercentage}% of o4-mini (200K tokens)`); - } - - return; - } - - // Get context for specific task - contextResult = await taskFactory.createContextForTask(taskType); - - // Display results - logger.log('ok', `Total context token count: ${contextResult.tokenCount}`); - logger.log('info', `Files included: ${contextResult.includedFiles.length}`); - logger.log('info', `Files trimmed: ${contextResult.trimmedFiles.length}`); - logger.log('info', `Files excluded: ${contextResult.excludedFiles.length}`); - logger.log('info', `Token savings: ${contextResult.tokenSavings}`); - - if (argvArg.detailed) { - // Show more detailed info about the context and token usage - const o4MiniPercentage = (contextResult.tokenCount / 200000 * 100).toFixed(2); - logger.log('info', `Token usage: ${o4MiniPercentage}% of o4-mini 200K token context window`); - - if (argvArg.model) { - // Show percentages for different models - if (argvArg.model === 'gpt4') { - const gpt4Percentage = (contextResult.tokenCount / 8192 * 100).toFixed(2); - logger.log('info', `Token usage (GPT-4): ${gpt4Percentage}% of 8192 token context window`); - } else if (argvArg.model === 'gpt35') { - const gpt35Percentage = (contextResult.tokenCount / 4096 * 100).toFixed(2); - logger.log('info', `Token usage (GPT-3.5): ${gpt35Percentage}% of 4096 token context window`); - } - } - - // Estimate cost (approximate values) - const o4MiniInputCost = 0.00005; // per 1K tokens for o4-mini - const estimatedCost = (contextResult.tokenCount / 1000 * o4MiniInputCost).toFixed(6); - logger.log('info', `Estimated input cost: $${estimatedCost} (o4-mini)`); - - if (argvArg.listFiles) { - // List files included in context - logger.log('info', '\nIncluded files:'); - contextResult.includedFiles.forEach(file => { - logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`); - }); - - logger.log('info', '\nTrimmed files:'); - contextResult.trimmedFiles.forEach(file => { - logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`); - }); - - if (contextResult.excludedFiles.length > 0) { - logger.log('info', '\nExcluded files:'); - contextResult.excludedFiles.forEach(file => { - logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`); - }); - } - } - } + logger.log('ok', `Commit message generated:`); + console.log(JSON.stringify(commitObject, null, 2)); }); tsdocCli.addCommand('test').subscribe((argvArg) => { diff --git a/ts/context/config-manager.ts b/ts/context/config-manager.ts deleted file mode 100644 index 86ca59d..0000000 --- a/ts/context/config-manager.ts +++ /dev/null @@ -1,369 +0,0 @@ -import * as plugins from '../plugins.js'; -import * as fs from 'fs'; -import type { - IContextConfig, - ITrimConfig, - ITaskConfig, - TaskType, - ContextMode, - ICacheConfig, - IAnalyzerConfig, - IPrioritizationWeights, - ITierConfig, - IIterativeConfig -} from './types.js'; - -/** - * Manages configuration for context building - */ -export class ConfigManager { - private static instance: ConfigManager; - private config: IContextConfig; - private projectDir: string = ''; - private configCache: { mtime: number; config: IContextConfig } | null = null; - - /** - * Get the singleton instance of ConfigManager - */ - public static getInstance(): ConfigManager { - if (!ConfigManager.instance) { - ConfigManager.instance = new ConfigManager(); - } - return ConfigManager.instance; - } - - /** - * Private constructor for singleton pattern - */ - private constructor() { - this.config = this.getDefaultConfig(); - } - - /** - * Initialize the config manager with a project directory - * @param projectDir The project directory - */ - public async initialize(projectDir: string): Promise { - this.projectDir = projectDir; - await this.loadConfig(); - } - - /** - * Get the default configuration - */ - private getDefaultConfig(): IContextConfig { - return { - maxTokens: 190000, // Default for o4-mini with some buffer - defaultMode: 'trimmed', - taskSpecificSettings: { - readme: { - mode: 'trimmed', - includePaths: ['ts/', 'src/'], - excludePaths: ['test/', 'node_modules/'] - }, - commit: { - mode: 'trimmed', - focusOnChangedFiles: true - }, - description: { - mode: 'trimmed', - includePackageInfo: true - } - }, - trimming: { - removeImplementations: true, - preserveInterfaces: true, - preserveTypeDefs: true, - preserveJSDoc: true, - maxFunctionLines: 5, - removeComments: true, - removeBlankLines: true - }, - cache: { - enabled: true, - ttl: 3600, // 1 hour - maxSize: 100, // 100MB - directory: undefined // Will be set to .nogit/context-cache by ContextCache - }, - analyzer: { - useAIRefinement: false, // Disabled by default for now - aiModel: 'haiku' - }, - prioritization: { - dependencyWeight: 0.3, - relevanceWeight: 0.4, - efficiencyWeight: 0.2, - recencyWeight: 0.1 - }, - tiers: { - essential: { minScore: 0.8, trimLevel: 'none' }, - important: { minScore: 0.5, trimLevel: 'light' }, - optional: { minScore: 0.2, trimLevel: 'aggressive' } - }, - iterative: { - maxIterations: 5, - firstPassFileLimit: 10, - subsequentPassFileLimit: 5, - temperature: 0.3, - model: 'gpt-4-turbo-preview' - } - }; - } - - /** - * Load configuration from npmextra.json - */ - private async loadConfig(): Promise { - try { - if (!this.projectDir) { - return; - } - - const npmextraJsonPath = plugins.path.join(this.projectDir, 'npmextra.json'); - - // Check if file exists - const fileExists = await plugins.fsInstance.file(npmextraJsonPath).exists(); - if (!fileExists) { - return; - } - - // Check cache - const stats = await fs.promises.stat(npmextraJsonPath); - const currentMtime = Math.floor(stats.mtimeMs); - - if (this.configCache && this.configCache.mtime === currentMtime) { - // Use cached config - this.config = this.configCache.config; - return; - } - - // Read the npmextra.json file - const npmextraJsonFile = await plugins.smartfileFactory.fromFilePath(npmextraJsonPath); - const npmextraContent = JSON.parse(npmextraJsonFile.contents.toString()); - - // Check for tsdoc context configuration - if (npmextraContent?.['@git.zone/tsdoc']?.context) { - // Merge with default config - this.config = this.mergeConfigs(this.config, npmextraContent['@git.zone/tsdoc'].context); - } - - // Cache the config - this.configCache = { - mtime: currentMtime, - config: { ...this.config } - }; - } catch (error) { - console.error('Error loading context configuration:', error); - } - } - - /** - * Merge configurations, with userConfig taking precedence - * @param defaultConfig The default configuration - * @param userConfig The user configuration - */ - private mergeConfigs(defaultConfig: IContextConfig, userConfig: Partial): IContextConfig { - const result: IContextConfig = { ...defaultConfig }; - - // Merge top-level properties - if (userConfig.maxTokens !== undefined) result.maxTokens = userConfig.maxTokens; - if (userConfig.defaultMode !== undefined) result.defaultMode = userConfig.defaultMode; - - // Merge task-specific settings - if (userConfig.taskSpecificSettings) { - result.taskSpecificSettings = result.taskSpecificSettings || {}; - - // For each task type, merge settings - (['readme', 'commit', 'description'] as TaskType[]).forEach(taskType => { - if (userConfig.taskSpecificSettings?.[taskType]) { - result.taskSpecificSettings![taskType] = { - ...result.taskSpecificSettings![taskType], - ...userConfig.taskSpecificSettings[taskType] - }; - } - }); - } - - // Merge trimming configuration - if (userConfig.trimming) { - result.trimming = { - ...result.trimming, - ...userConfig.trimming - }; - } - - // Merge cache configuration - if (userConfig.cache) { - result.cache = { - ...result.cache, - ...userConfig.cache - }; - } - - // Merge analyzer configuration - if (userConfig.analyzer) { - result.analyzer = { - ...result.analyzer, - ...userConfig.analyzer - }; - } - - // Merge prioritization weights - if (userConfig.prioritization) { - result.prioritization = { - ...result.prioritization, - ...userConfig.prioritization - }; - } - - // Merge tier configuration - if (userConfig.tiers) { - result.tiers = { - ...result.tiers, - ...userConfig.tiers - }; - } - - // Merge iterative configuration - if (userConfig.iterative) { - result.iterative = { - ...result.iterative, - ...userConfig.iterative - }; - } - - return result; - } - - /** - * Get the complete configuration - */ - public getConfig(): IContextConfig { - return this.config; - } - - /** - * Get the trimming configuration - */ - public getTrimConfig(): ITrimConfig { - return this.config.trimming || {}; - } - - /** - * Get configuration for a specific task - * @param taskType The type of task - */ - public getTaskConfig(taskType: TaskType): ITaskConfig { - // Get task-specific config or empty object - const taskConfig = this.config.taskSpecificSettings?.[taskType] || {}; - - // If mode is not specified, use default mode - if (!taskConfig.mode) { - taskConfig.mode = this.config.defaultMode; - } - - return taskConfig; - } - - /** - * Get the maximum tokens allowed for context - */ - public getMaxTokens(): number { - return this.config.maxTokens || 190000; - } - - /** - * Update the configuration - * @param config The new configuration - */ - public async updateConfig(config: Partial): Promise { - // Merge with existing config - this.config = this.mergeConfigs(this.config, config); - - // Invalidate cache - this.configCache = null; - - try { - if (!this.projectDir) { - return; - } - - // Read the existing npmextra.json file - const npmextraJsonPath = plugins.path.join(this.projectDir, 'npmextra.json'); - let npmextraContent = {}; - - if (await plugins.fsInstance.file(npmextraJsonPath).exists()) { - const npmextraJsonFile = await plugins.smartfileFactory.fromFilePath(npmextraJsonPath); - npmextraContent = JSON.parse(npmextraJsonFile.contents.toString()) || {}; - } - - // Update the tsdoc context configuration - const typedContent = npmextraContent as any; - if (!typedContent.tsdoc) typedContent.tsdoc = {}; - typedContent.tsdoc.context = this.config; - - // Write back to npmextra.json - const updatedContent = JSON.stringify(npmextraContent, null, 2); - await plugins.fsInstance.file(npmextraJsonPath).encoding('utf8').write(updatedContent); - } catch (error) { - console.error('Error updating context configuration:', error); - } - } - - /** - * Get cache configuration - */ - public getCacheConfig(): ICacheConfig { - return this.config.cache || { enabled: true, ttl: 3600, maxSize: 100 }; - } - - /** - * Get analyzer configuration - */ - public getAnalyzerConfig(): IAnalyzerConfig { - return this.config.analyzer || { useAIRefinement: false, aiModel: 'haiku' }; - } - - /** - * Get prioritization weights - */ - public getPrioritizationWeights(): IPrioritizationWeights { - return this.config.prioritization || { - dependencyWeight: 0.3, - relevanceWeight: 0.4, - efficiencyWeight: 0.2, - recencyWeight: 0.1 - }; - } - - /** - * Get tier configuration - */ - public getTierConfig(): ITierConfig { - return this.config.tiers || { - essential: { minScore: 0.8, trimLevel: 'none' }, - important: { minScore: 0.5, trimLevel: 'light' }, - optional: { minScore: 0.2, trimLevel: 'aggressive' } - }; - } - - /** - * Get iterative configuration - */ - public getIterativeConfig(): IIterativeConfig { - return this.config.iterative || { - maxIterations: 5, - firstPassFileLimit: 10, - subsequentPassFileLimit: 5, - temperature: 0.3, - model: 'gpt-4-turbo-preview' - }; - } - - /** - * Clear the config cache (force reload on next access) - */ - public clearCache(): void { - this.configCache = null; - } -} \ No newline at end of file diff --git a/ts/context/context-analyzer.ts b/ts/context/context-analyzer.ts deleted file mode 100644 index d0e3498..0000000 --- a/ts/context/context-analyzer.ts +++ /dev/null @@ -1,391 +0,0 @@ -import * as plugins from '../plugins.js'; -import type { - IFileMetadata, - IFileDependencies, - IFileAnalysis, - IAnalysisResult, - TaskType, - IPrioritizationWeights, - ITierConfig, -} from './types.js'; - -/** - * ContextAnalyzer provides intelligent file selection and prioritization - * based on dependency analysis, task relevance, and configurable weights - */ -export class ContextAnalyzer { - private projectRoot: string; - private weights: Required; - private tiers: Required; - - /** - * Creates a new ContextAnalyzer - * @param projectRoot - Root directory of the project - * @param weights - Prioritization weights - * @param tiers - Tier configuration - */ - constructor( - projectRoot: string, - weights: Partial = {}, - tiers: Partial = {} - ) { - this.projectRoot = projectRoot; - - // Default weights - this.weights = { - dependencyWeight: weights.dependencyWeight ?? 0.3, - relevanceWeight: weights.relevanceWeight ?? 0.4, - efficiencyWeight: weights.efficiencyWeight ?? 0.2, - recencyWeight: weights.recencyWeight ?? 0.1, - }; - - // Default tiers - this.tiers = { - essential: tiers.essential ?? { minScore: 0.8, trimLevel: 'none' }, - important: tiers.important ?? { minScore: 0.5, trimLevel: 'light' }, - optional: tiers.optional ?? { minScore: 0.2, trimLevel: 'aggressive' }, - }; - } - - /** - * Analyzes files for a specific task type - * @param metadata - Array of file metadata to analyze - * @param taskType - Type of task being performed - * @param changedFiles - Optional list of recently changed files (for commits) - * @returns Analysis result with scored files - */ - public async analyze( - metadata: IFileMetadata[], - taskType: TaskType, - changedFiles: string[] = [] - ): Promise { - const startTime = Date.now(); - - // Build dependency graph - const dependencyGraph = await this.buildDependencyGraph(metadata); - - // Calculate centrality scores - this.calculateCentrality(dependencyGraph); - - // Analyze each file - const files: IFileAnalysis[] = []; - for (const meta of metadata) { - const analysis = await this.analyzeFile( - meta, - taskType, - dependencyGraph, - changedFiles - ); - files.push(analysis); - } - - // Sort by importance score (highest first) - files.sort((a, b) => b.importanceScore - a.importanceScore); - - const analysisDuration = Date.now() - startTime; - - return { - taskType, - files, - dependencyGraph, - totalFiles: metadata.length, - analysisDuration, - }; - } - - /** - * Builds a dependency graph from file metadata - * @param metadata - Array of file metadata - * @returns Dependency graph as a map - */ - private async buildDependencyGraph( - metadata: IFileMetadata[] - ): Promise> { - const graph = new Map(); - - // Initialize graph entries - for (const meta of metadata) { - graph.set(meta.path, { - path: meta.path, - imports: [], - importedBy: [], - centrality: 0, - }); - } - - // Parse imports from each file - for (const meta of metadata) { - try { - const contents = await plugins.fsInstance.file(meta.path).encoding('utf8').read() as string; - const imports = this.extractImports(contents, meta.path); - - const deps = graph.get(meta.path)!; - deps.imports = imports; - - // Update importedBy for imported files - for (const importPath of imports) { - const importedDeps = graph.get(importPath); - if (importedDeps) { - importedDeps.importedBy.push(meta.path); - } - } - } catch (error) { - console.warn(`Failed to parse imports from ${meta.path}:`, error.message); - } - } - - return graph; - } - - /** - * Extracts import statements from file contents - * @param contents - File contents - * @param filePath - Path of the file being analyzed - * @returns Array of absolute paths to imported files - */ - private extractImports(contents: string, filePath: string): string[] { - const imports: string[] = []; - const fileDir = plugins.path.dirname(filePath); - - // Match various import patterns - const importRegex = /(?:import|export).*?from\s+['"](.+?)['"]/g; - let match; - - while ((match = importRegex.exec(contents)) !== null) { - const importPath = match[1]; - - // Skip external modules - if (!importPath.startsWith('.')) { - continue; - } - - // Resolve relative import to absolute path - let resolvedPath = plugins.path.resolve(fileDir, importPath); - - // Handle various file extensions - const extensions = ['.ts', '.js', '.tsx', '.jsx', '/index.ts', '/index.js']; - let found = false; - - for (const ext of extensions) { - const testPath = resolvedPath.endsWith(ext) ? resolvedPath : resolvedPath + ext; - try { - // Use synchronous file check to avoid async in this context - const fs = require('fs'); - const exists = fs.existsSync(testPath); - if (exists) { - imports.push(testPath); - found = true; - break; - } - } catch (error) { - // Continue trying other extensions - } - } - - if (!found && !resolvedPath.includes('.')) { - // Try with .ts extension as default - imports.push(resolvedPath + '.ts'); - } - } - - return imports; - } - - /** - * Calculates centrality scores for all nodes in the dependency graph - * Uses a simplified PageRank-like algorithm - * @param graph - Dependency graph - */ - private calculateCentrality(graph: Map): void { - const damping = 0.85; - const iterations = 10; - const nodeCount = graph.size; - - // Initialize scores - const scores = new Map(); - for (const path of graph.keys()) { - scores.set(path, 1.0 / nodeCount); - } - - // Iterative calculation - for (let i = 0; i < iterations; i++) { - const newScores = new Map(); - - for (const [path, deps] of graph.entries()) { - let score = (1 - damping) / nodeCount; - - // Add contributions from nodes that import this file - for (const importerPath of deps.importedBy) { - const importerDeps = graph.get(importerPath); - if (importerDeps) { - const importerScore = scores.get(importerPath) ?? 0; - const outgoingCount = importerDeps.imports.length || 1; - score += damping * (importerScore / outgoingCount); - } - } - - newScores.set(path, score); - } - - // Update scores - for (const [path, score] of newScores) { - scores.set(path, score); - } - } - - // Normalize scores to 0-1 range - const maxScore = Math.max(...scores.values()); - if (maxScore > 0) { - for (const deps of graph.values()) { - const score = scores.get(deps.path) ?? 0; - deps.centrality = score / maxScore; - } - } - } - - /** - * Analyzes a single file - * @param meta - File metadata - * @param taskType - Task being performed - * @param graph - Dependency graph - * @param changedFiles - Recently changed files - * @returns File analysis - */ - private async analyzeFile( - meta: IFileMetadata, - taskType: TaskType, - graph: Map, - changedFiles: string[] - ): Promise { - const deps = graph.get(meta.path); - const centralityScore = deps?.centrality ?? 0; - - // Calculate task-specific relevance - const relevanceScore = this.calculateRelevance(meta, taskType); - - // Calculate efficiency (information per token) - const efficiencyScore = this.calculateEfficiency(meta); - - // Calculate recency (for commit tasks) - const recencyScore = this.calculateRecency(meta, changedFiles); - - // Calculate combined importance score - const importanceScore = - relevanceScore * this.weights.relevanceWeight + - centralityScore * this.weights.dependencyWeight + - efficiencyScore * this.weights.efficiencyWeight + - recencyScore * this.weights.recencyWeight; - - // Assign tier - const tier = this.assignTier(importanceScore); - - return { - path: meta.path, - relevanceScore, - centralityScore, - efficiencyScore, - recencyScore, - importanceScore, - tier, - reason: this.generateReason(meta, taskType, importanceScore, tier), - }; - } - - /** - * Calculates task-specific relevance score - */ - private calculateRelevance(meta: IFileMetadata, taskType: TaskType): number { - const relativePath = meta.relativePath.toLowerCase(); - let score = 0.5; // Base score - - // README generation - prioritize public APIs and main exports - if (taskType === 'readme') { - if (relativePath.includes('index.ts')) score += 0.3; - if (relativePath.match(/^ts\/[^\/]+\.ts$/)) score += 0.2; // Root level exports - if (relativePath.includes('test/')) score -= 0.3; - if (relativePath.includes('classes/')) score += 0.1; - if (relativePath.includes('interfaces/')) score += 0.1; - } - - // Commit messages - prioritize changed files and their dependencies - if (taskType === 'commit') { - if (relativePath.includes('test/')) score -= 0.2; - // Recency will handle changed files - } - - // Description generation - prioritize main exports and core interfaces - if (taskType === 'description') { - if (relativePath.includes('index.ts')) score += 0.4; - if (relativePath.match(/^ts\/[^\/]+\.ts$/)) score += 0.3; - if (relativePath.includes('test/')) score -= 0.4; - if (relativePath.includes('interfaces/')) score += 0.2; - } - - return Math.max(0, Math.min(1, score)); - } - - /** - * Calculates efficiency score (information density) - */ - private calculateEfficiency(meta: IFileMetadata): number { - // Prefer files that are not too large (good signal-to-noise ratio) - const optimalSize = 5000; // ~1250 tokens - const distance = Math.abs(meta.estimatedTokens - optimalSize); - const normalized = Math.max(0, 1 - distance / optimalSize); - - return normalized; - } - - /** - * Calculates recency score for changed files - */ - private calculateRecency(meta: IFileMetadata, changedFiles: string[]): number { - if (changedFiles.length === 0) { - return 0; - } - - // Check if this file was changed - const isChanged = changedFiles.some((changed) => changed === meta.path); - - return isChanged ? 1.0 : 0.0; - } - - /** - * Assigns a tier based on importance score - */ - private assignTier(score: number): 'essential' | 'important' | 'optional' | 'excluded' { - if (score >= this.tiers.essential.minScore) return 'essential'; - if (score >= this.tiers.important.minScore) return 'important'; - if (score >= this.tiers.optional.minScore) return 'optional'; - return 'excluded'; - } - - /** - * Generates a human-readable reason for the score - */ - private generateReason( - meta: IFileMetadata, - taskType: TaskType, - score: number, - tier: string - ): string { - const reasons: string[] = []; - - if (meta.relativePath.includes('index.ts')) { - reasons.push('main export file'); - } - - if (meta.relativePath.includes('test/')) { - reasons.push('test file (lower priority)'); - } - - if (taskType === 'readme' && meta.relativePath.match(/^ts\/[^\/]+\.ts$/)) { - reasons.push('root-level module'); - } - - reasons.push(`score: ${score.toFixed(2)}`); - reasons.push(`tier: ${tier}`); - - return reasons.join(', '); - } -} diff --git a/ts/context/context-cache.ts b/ts/context/context-cache.ts deleted file mode 100644 index 89c4070..0000000 --- a/ts/context/context-cache.ts +++ /dev/null @@ -1,286 +0,0 @@ -import * as plugins from '../plugins.js'; -import * as fs from 'fs'; -import type { ICacheEntry, ICacheConfig } from './types.js'; -import { logger } from '../logging.js'; - -/** - * ContextCache provides persistent caching of file contents and token counts - * with automatic invalidation on file changes - */ -export class ContextCache { - private cacheDir: string; - private cache: Map = new Map(); - private config: Required; - private cacheIndexPath: string; - - /** - * Creates a new ContextCache - * @param projectRoot - Root directory of the project - * @param config - Cache configuration - */ - constructor(projectRoot: string, config: Partial = {}) { - this.config = { - enabled: config.enabled ?? true, - ttl: config.ttl ?? 3600, // 1 hour default - maxSize: config.maxSize ?? 100, // 100MB default - directory: config.directory ?? plugins.path.join(projectRoot, '.nogit', 'context-cache'), - }; - - this.cacheDir = this.config.directory; - this.cacheIndexPath = plugins.path.join(this.cacheDir, 'index.json'); - } - - /** - * Initializes the cache by loading from disk - */ - public async init(): Promise { - if (!this.config.enabled) { - return; - } - - // Ensure cache directory exists - await plugins.fsInstance.directory(this.cacheDir).recursive().create(); - - // Load cache index if it exists - try { - const indexExists = await plugins.fsInstance.file(this.cacheIndexPath).exists(); - if (indexExists) { - const indexContent = await plugins.fsInstance.file(this.cacheIndexPath).encoding('utf8').read() as string; - const indexData = JSON.parse(indexContent) as ICacheEntry[]; - if (Array.isArray(indexData)) { - for (const entry of indexData) { - this.cache.set(entry.path, entry); - } - } - } - } catch (error) { - console.warn('Failed to load cache index:', error.message); - // Start with empty cache if loading fails - } - - // Clean up expired and invalid entries - await this.cleanup(); - } - - /** - * Gets a cached entry if it's still valid - * @param filePath - Absolute path to the file - * @returns Cache entry if valid, null otherwise - */ - public async get(filePath: string): Promise { - if (!this.config.enabled) { - return null; - } - - const entry = this.cache.get(filePath); - if (!entry) { - return null; - } - - // Check if entry is expired - const now = Date.now(); - if (now - entry.cachedAt > this.config.ttl * 1000) { - this.cache.delete(filePath); - return null; - } - - // Check if file has been modified - try { - const stats = await fs.promises.stat(filePath); - const currentMtime = Math.floor(stats.mtimeMs); - - if (currentMtime !== entry.mtime) { - // File has changed, invalidate cache - this.cache.delete(filePath); - return null; - } - - return entry; - } catch (error) { - // File doesn't exist anymore - this.cache.delete(filePath); - return null; - } - } - - /** - * Stores a cache entry - * @param entry - Cache entry to store - */ - public async set(entry: ICacheEntry): Promise { - if (!this.config.enabled) { - return; - } - - this.cache.set(entry.path, entry); - - // Check cache size and evict old entries if needed - await this.enforceMaxSize(); - - // Persist to disk (async, don't await) - this.persist().catch((error) => { - console.warn('Failed to persist cache:', error.message); - }); - } - - /** - * Stores multiple cache entries - * @param entries - Array of cache entries - */ - public async setMany(entries: ICacheEntry[]): Promise { - if (!this.config.enabled) { - return; - } - - for (const entry of entries) { - this.cache.set(entry.path, entry); - } - - await this.enforceMaxSize(); - await this.persist(); - } - - /** - * Checks if a file is cached and valid - * @param filePath - Absolute path to the file - * @returns True if cached and valid - */ - public async has(filePath: string): Promise { - const entry = await this.get(filePath); - return entry !== null; - } - - /** - * Gets cache statistics - */ - public getStats(): { - entries: number; - totalSize: number; - oldestEntry: number | null; - newestEntry: number | null; - } { - let totalSize = 0; - let oldestEntry: number | null = null; - let newestEntry: number | null = null; - - for (const entry of this.cache.values()) { - totalSize += entry.contents.length; - - if (oldestEntry === null || entry.cachedAt < oldestEntry) { - oldestEntry = entry.cachedAt; - } - - if (newestEntry === null || entry.cachedAt > newestEntry) { - newestEntry = entry.cachedAt; - } - } - - return { - entries: this.cache.size, - totalSize, - oldestEntry, - newestEntry, - }; - } - - /** - * Clears all cache entries - */ - public async clear(): Promise { - this.cache.clear(); - await this.persist(); - } - - /** - * Clears specific cache entries - * @param filePaths - Array of file paths to clear - */ - public async clearPaths(filePaths: string[]): Promise { - for (const path of filePaths) { - this.cache.delete(path); - } - await this.persist(); - } - - /** - * Cleans up expired and invalid cache entries - */ - private async cleanup(): Promise { - const now = Date.now(); - const toDelete: string[] = []; - - for (const [path, entry] of this.cache.entries()) { - // Check expiration - if (now - entry.cachedAt > this.config.ttl * 1000) { - toDelete.push(path); - continue; - } - - // Check if file still exists and hasn't changed - try { - const stats = await fs.promises.stat(path); - const currentMtime = Math.floor(stats.mtimeMs); - - if (currentMtime !== entry.mtime) { - toDelete.push(path); - } - } catch (error) { - // File doesn't exist - toDelete.push(path); - } - } - - for (const path of toDelete) { - this.cache.delete(path); - } - - if (toDelete.length > 0) { - await this.persist(); - } - } - - /** - * Enforces maximum cache size by evicting oldest entries - */ - private async enforceMaxSize(): Promise { - const stats = this.getStats(); - const maxSizeBytes = this.config.maxSize * 1024 * 1024; // Convert MB to bytes - - if (stats.totalSize <= maxSizeBytes) { - return; - } - - // Sort entries by age (oldest first) - const entries = Array.from(this.cache.entries()).sort( - (a, b) => a[1].cachedAt - b[1].cachedAt - ); - - // Remove oldest entries until we're under the limit - let currentSize = stats.totalSize; - for (const [path, entry] of entries) { - if (currentSize <= maxSizeBytes) { - break; - } - - currentSize -= entry.contents.length; - this.cache.delete(path); - } - } - - /** - * Persists cache index to disk - */ - private async persist(): Promise { - if (!this.config.enabled) { - return; - } - - try { - const entries = Array.from(this.cache.values()); - const content = JSON.stringify(entries, null, 2); - await plugins.fsInstance.file(this.cacheIndexPath).encoding('utf8').write(content); - } catch (error) { - console.warn('Failed to persist cache index:', error.message); - } - } -} diff --git a/ts/context/context-trimmer.ts b/ts/context/context-trimmer.ts deleted file mode 100644 index d22b320..0000000 --- a/ts/context/context-trimmer.ts +++ /dev/null @@ -1,310 +0,0 @@ -import * as plugins from '../plugins.js'; -import type { ITrimConfig, ContextMode } from './types.js'; - -/** - * Class responsible for trimming file contents to reduce token usage - * while preserving important information for context - */ -export class ContextTrimmer { - private config: ITrimConfig; - - /** - * Create a new ContextTrimmer with the given configuration - * @param config The trimming configuration - */ - constructor(config?: ITrimConfig) { - this.config = { - removeImplementations: true, - preserveInterfaces: true, - preserveTypeDefs: true, - preserveJSDoc: true, - maxFunctionLines: 5, - removeComments: true, - removeBlankLines: true, - ...config - }; - } - - /** - * Trim a file's contents based on the configuration - * @param filePath The path to the file - * @param content The file's contents - * @param mode The context mode to use - * @returns The trimmed file contents - */ - public trimFile(filePath: string, content: string, mode: ContextMode = 'trimmed'): string { - // If mode is 'full', return the original content - if (mode === 'full') { - return content; - } - - // Process based on file type - if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) { - return this.trimTypeScriptFile(content); - } else if (filePath.endsWith('.md')) { - return this.trimMarkdownFile(content); - } else if (filePath.endsWith('.json')) { - return this.trimJsonFile(content); - } - - // Default to returning the original content for unknown file types - return content; - } - - /** - * Trim a TypeScript file to reduce token usage - * @param content The TypeScript file contents - * @returns The trimmed file contents - */ - private trimTypeScriptFile(content: string): string { - let result = content; - - // Step 1: Preserve JSDoc comments if configured - const jsDocComments: string[] = []; - if (this.config.preserveJSDoc) { - const jsDocRegex = /\/\*\*[\s\S]*?\*\//g; - const matches = result.match(jsDocRegex) || []; - jsDocComments.push(...matches); - } - - // Step 2: Remove comments if configured - if (this.config.removeComments) { - // Remove single-line comments - result = result.replace(/\/\/.*$/gm, ''); - // Remove multi-line comments (except JSDoc if preserveJSDoc is true) - if (!this.config.preserveJSDoc) { - result = result.replace(/\/\*[\s\S]*?\*\//g, ''); - } else { - // Only remove non-JSDoc comments - result = result.replace(/\/\*(?!\*)[\s\S]*?\*\//g, ''); - } - } - - // Step 3: Remove function implementations if configured - if (this.config.removeImplementations) { - // Match function and method bodies - result = result.replace( - /(\b(function|constructor|async function)\s+[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/g, - (match, start, funcType, body, end) => { - // Keep function signature and opening brace, replace body with comment - return `${start} /* implementation removed */ ${end}`; - } - ); - - // Match arrow function bodies - result = result.replace( - /(\([^)]*\)\s*=>\s*{)([\s\S]*?)(})/g, - (match, start, body, end) => { - return `${start} /* implementation removed */ ${end}`; - } - ); - - // Match method declarations - result = result.replace( - /(^\s*[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/gm, - (match, start, body, end) => { - return `${start} /* implementation removed */ ${end}`; - } - ); - - // Match class methods - result = result.replace( - /(\b(public|private|protected|static|async)?\s+[\w$]+\s*\([^)]*\)\s*{)([\s\S]*?)(})/g, - (match, start, modifier, body, end) => { - return `${start} /* implementation removed */ ${end}`; - } - ); - } else if (this.config.maxFunctionLines && this.config.maxFunctionLines > 0) { - // If not removing implementations completely, limit the number of lines - // Match function and method bodies - result = result.replace( - /(\b(function|constructor|async function)\s+[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/g, - (match, start, funcType, body, end) => { - return this.limitFunctionBody(start, body, end); - } - ); - - // Match arrow function bodies - result = result.replace( - /(\([^)]*\)\s*=>\s*{)([\s\S]*?)(})/g, - (match, start, body, end) => { - return this.limitFunctionBody(start, body, end); - } - ); - - // Match method declarations - result = result.replace( - /(^\s*[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/gm, - (match, start, body, end) => { - return this.limitFunctionBody(start, body, end); - } - ); - - // Match class methods - result = result.replace( - /(\b(public|private|protected|static|async)?\s+[\w$]+\s*\([^)]*\)\s*{)([\s\S]*?)(})/g, - (match, start, modifier, body, end) => { - return this.limitFunctionBody(start, body, end); - } - ); - } - - // Step 4: Remove blank lines if configured - if (this.config.removeBlankLines) { - result = result.replace(/^\s*[\r\n]/gm, ''); - } - - // Step 5: Restore preserved JSDoc comments - if (this.config.preserveJSDoc && jsDocComments.length > 0) { - // This is a placeholder; we already preserved JSDoc comments in the regex steps - } - - return result; - } - - /** - * Limit a function body to a maximum number of lines - * @param start The function signature and opening brace - * @param body The function body - * @param end The closing brace - * @returns The limited function body - */ - private limitFunctionBody(start: string, body: string, end: string): string { - const lines = body.split('\n'); - if (lines.length > this.config.maxFunctionLines!) { - const limitedBody = lines.slice(0, this.config.maxFunctionLines!).join('\n'); - return `${start}${limitedBody}\n // ... (${lines.length - this.config.maxFunctionLines!} lines trimmed)\n${end}`; - } - return `${start}${body}${end}`; - } - - /** - * Trim a Markdown file to reduce token usage - * @param content The Markdown file contents - * @returns The trimmed file contents - */ - private trimMarkdownFile(content: string): string { - // For markdown files, we generally want to keep most content - // but we can remove lengthy code blocks if needed - return content; - } - - /** - * Trim a JSON file to reduce token usage - * @param content The JSON file contents - * @returns The trimmed file contents - */ - private trimJsonFile(content: string): string { - try { - // Parse the JSON - const json = JSON.parse(content); - - // For package.json, keep only essential information - if ('name' in json && 'version' in json && 'dependencies' in json) { - const essentialKeys = [ - 'name', 'version', 'description', 'author', 'license', - 'main', 'types', 'exports', 'type' - ]; - - const trimmedJson: any = {}; - essentialKeys.forEach(key => { - if (key in json) { - trimmedJson[key] = json[key]; - } - }); - - // Add dependency information without versions - if ('dependencies' in json) { - trimmedJson.dependencies = Object.keys(json.dependencies).reduce((acc, dep) => { - acc[dep] = '*'; // Replace version with wildcard - return acc; - }, {} as Record); - } - - // Return the trimmed JSON - return JSON.stringify(trimmedJson, null, 2); - } - - // For other JSON files, leave as is - return content; - } catch (error) { - // If there's an error parsing the JSON, return the original content - return content; - } - } - - /** - * Update the trimmer configuration - * @param config The new configuration to apply - */ - public updateConfig(config: ITrimConfig): void { - this.config = { - ...this.config, - ...config - }; - } - - /** - * Trim a file based on its importance tier - * @param filePath The path to the file - * @param content The file's contents - * @param level The trimming level to apply ('none', 'light', 'aggressive') - * @returns The trimmed file contents - */ - public trimFileWithLevel( - filePath: string, - content: string, - level: 'none' | 'light' | 'aggressive' - ): string { - // No trimming for essential files - if (level === 'none') { - return content; - } - - // Create a temporary config based on level - const originalConfig = { ...this.config }; - - try { - if (level === 'light') { - // Light trimming: preserve signatures, remove only complex implementations - this.config = { - ...this.config, - removeImplementations: false, - preserveInterfaces: true, - preserveTypeDefs: true, - preserveJSDoc: true, - maxFunctionLines: 10, - removeComments: false, - removeBlankLines: true - }; - } else if (level === 'aggressive') { - // Aggressive trimming: remove all implementations, keep only signatures - this.config = { - ...this.config, - removeImplementations: true, - preserveInterfaces: true, - preserveTypeDefs: true, - preserveJSDoc: true, - maxFunctionLines: 3, - removeComments: true, - removeBlankLines: true - }; - } - - // Process based on file type - let result = content; - if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) { - result = this.trimTypeScriptFile(content); - } else if (filePath.endsWith('.md')) { - result = this.trimMarkdownFile(content); - } else if (filePath.endsWith('.json')) { - result = this.trimJsonFile(content); - } - - return result; - } finally { - // Restore original config - this.config = originalConfig; - } - } -} \ No newline at end of file diff --git a/ts/context/enhanced-context.ts b/ts/context/enhanced-context.ts deleted file mode 100644 index 7a40629..0000000 --- a/ts/context/enhanced-context.ts +++ /dev/null @@ -1,332 +0,0 @@ -import * as plugins from '../plugins.js'; -import type { ContextMode, IContextResult, IFileInfo, TaskType, IFileMetadata } from './types.js'; -import { ContextTrimmer } from './context-trimmer.js'; -import { ConfigManager } from './config-manager.js'; -import { LazyFileLoader } from './lazy-file-loader.js'; -import { ContextCache } from './context-cache.js'; -import { ContextAnalyzer } from './context-analyzer.js'; - -/** - * Enhanced ProjectContext that supports context optimization strategies - */ -export class EnhancedContext { - private projectDir: string; - private trimmer: ContextTrimmer; - private configManager: ConfigManager; - private lazyLoader: LazyFileLoader; - private cache: ContextCache; - private analyzer: ContextAnalyzer; - private contextMode: ContextMode = 'trimmed'; - private tokenBudget: number = 190000; // Default for o4-mini - private contextResult: IContextResult = { - context: '', - tokenCount: 0, - includedFiles: [], - trimmedFiles: [], - excludedFiles: [], - tokenSavings: 0 - }; - - /** - * Create a new EnhancedContext - * @param projectDirArg The project directory - */ - constructor(projectDirArg: string) { - this.projectDir = projectDirArg; - this.configManager = ConfigManager.getInstance(); - this.trimmer = new ContextTrimmer(this.configManager.getTrimConfig()); - this.lazyLoader = new LazyFileLoader(projectDirArg); - this.cache = new ContextCache(projectDirArg, this.configManager.getCacheConfig()); - this.analyzer = new ContextAnalyzer( - projectDirArg, - this.configManager.getPrioritizationWeights(), - this.configManager.getTierConfig() - ); - } - - /** - * Initialize the context builder - */ - public async initialize(): Promise { - await this.configManager.initialize(this.projectDir); - this.tokenBudget = this.configManager.getMaxTokens(); - this.trimmer.updateConfig(this.configManager.getTrimConfig()); - await this.cache.init(); - } - - /** - * Set the context mode - * @param mode The context mode to use - */ - public setContextMode(mode: ContextMode): void { - this.contextMode = mode; - } - - /** - * Set the token budget - * @param maxTokens The maximum tokens to use - */ - public setTokenBudget(maxTokens: number): void { - this.tokenBudget = maxTokens; - } - - /** - * Convert files to context with smart analysis and prioritization - * @param metadata - File metadata to analyze - * @param taskType - Task type for context-aware prioritization - * @param mode - Context mode to use - * @returns Context string - */ - public async convertFilesToContextWithAnalysis( - metadata: IFileMetadata[], - taskType: TaskType, - mode: ContextMode = this.contextMode - ): Promise { - // Reset context result - this.contextResult = { - context: '', - tokenCount: 0, - includedFiles: [], - trimmedFiles: [], - excludedFiles: [], - tokenSavings: 0 - }; - - // Analyze files for smart prioritization - const analysis = await this.analyzer.analyze(metadata, taskType, []); - - // Sort files by importance score (highest first) - const sortedAnalysis = [...analysis.files].sort( - (a, b) => b.importanceScore - a.importanceScore - ); - - // Filter out excluded tier - const relevantFiles = sortedAnalysis.filter(f => f.tier !== 'excluded'); - - let totalTokenCount = 0; - let totalOriginalTokens = 0; - const processedFiles: string[] = []; - - // Load files with cache support - for (const fileAnalysis of relevantFiles) { - try { - // Check cache first - let contents: string; - let originalTokenCount: number; - - const cached = await this.cache.get(fileAnalysis.path); - if (cached) { - contents = cached.contents; - originalTokenCount = cached.tokenCount; - } else { - // Load file - const fileData = await plugins.fsInstance.file(fileAnalysis.path).encoding('utf8').read() as string; - contents = fileData; - originalTokenCount = this.countTokens(contents); - - // Cache it - await this.cache.set({ - path: fileAnalysis.path, - contents, - tokenCount: originalTokenCount, - mtime: Date.now(), - cachedAt: Date.now() - }); - } - - totalOriginalTokens += originalTokenCount; - - // Apply tier-based trimming - let processedContent = contents; - let trimLevel: 'none' | 'light' | 'aggressive' = 'light'; - - if (fileAnalysis.tier === 'essential') { - trimLevel = 'none'; - } else if (fileAnalysis.tier === 'important') { - trimLevel = 'light'; - } else if (fileAnalysis.tier === 'optional') { - trimLevel = 'aggressive'; - } - - // Apply trimming based on mode and tier - if (mode !== 'full' && trimLevel !== 'none') { - const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path); - processedContent = this.trimmer.trimFileWithLevel( - relativePath, - contents, - trimLevel - ); - } - - // Calculate token count - const processedTokenCount = this.countTokens(processedContent); - - // Check token budget - if (totalTokenCount + processedTokenCount > this.tokenBudget) { - // We don't have budget for this file - const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path); - this.contextResult.excludedFiles.push({ - path: fileAnalysis.path, - contents, - relativePath, - tokenCount: originalTokenCount, - importanceScore: fileAnalysis.importanceScore - }); - continue; - } - - // Format the file for context - const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path); - const formattedContent = ` -====== START OF FILE ${relativePath} ====== - -${processedContent} - -====== END OF FILE ${relativePath} ====== - `; - - processedFiles.push(formattedContent); - totalTokenCount += processedTokenCount; - - // Track file in appropriate list - const fileInfo: IFileInfo = { - path: fileAnalysis.path, - contents: processedContent, - relativePath, - tokenCount: processedTokenCount, - importanceScore: fileAnalysis.importanceScore - }; - - if (trimLevel === 'none' || processedContent === contents) { - this.contextResult.includedFiles.push(fileInfo); - } else { - this.contextResult.trimmedFiles.push(fileInfo); - this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount); - } - } catch (error) { - console.warn(`Failed to process file ${fileAnalysis.path}:`, error.message); - } - } - - // Join all processed files - const context = processedFiles.join('\n'); - - // Update context result - this.contextResult.context = context; - this.contextResult.tokenCount = totalTokenCount; - - return context; - } - - /** - * Build context for the project using smart analysis - * @param taskType Task type for context-aware prioritization (defaults to 'description') - */ - public async buildContext(taskType?: TaskType): Promise { - // Initialize if needed - if (this.tokenBudget === 0) { - await this.initialize(); - } - - // Smart context building always requires a task type for optimal prioritization - // Default to 'description' if not provided - const effectiveTaskType = taskType || 'description'; - - // Get task-specific configuration - const taskConfig = this.configManager.getTaskConfig(effectiveTaskType); - if (taskConfig.mode) { - this.setContextMode(taskConfig.mode); - } - - // Build globs for scanning - const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [ - 'ts/**/*.ts', - 'ts*/**/*.ts' - ]; - - // Add config files - const configGlobs = [ - 'package.json', - 'readme.md', - 'readme.hints.md', - 'npmextra.json' - ]; - - // Scan files for metadata (fast, doesn't load contents) - const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]); - - // Use smart analyzer to build context with intelligent prioritization - await this.convertFilesToContextWithAnalysis(metadata, effectiveTaskType, this.contextMode); - - return this.contextResult; - } - - /** - * Update the context with git diff information for commit tasks - * @param gitDiff The git diff to include - */ - public updateWithGitDiff(gitDiff: string): IContextResult { - // If we don't have a context yet, return empty result - if (!this.contextResult.context) { - return this.contextResult; - } - - // Add git diff to context - const diffSection = ` -====== GIT DIFF ====== - -${gitDiff} - -====== END GIT DIFF ====== - `; - - const diffTokenCount = this.countTokens(diffSection); - - // Update context and token count - this.contextResult.context += diffSection; - this.contextResult.tokenCount += diffTokenCount; - - return this.contextResult; - } - - /** - * Count tokens in a string - * @param text The text to count tokens for - * @param model The model to use for token counting - */ - public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number { - try { - // Use the gpt-tokenizer library to count tokens - const tokens = plugins.gptTokenizer.encode(text); - return tokens.length; - } catch (error) { - console.error('Error counting tokens:', error); - // Provide a rough estimate if tokenization fails - return Math.ceil(text.length / 4); - } - } - - /** - * Get the context result - */ - public getContextResult(): IContextResult { - return this.contextResult; - } - - /** - * Get the token count for the current context - */ - public getTokenCount(): number { - return this.contextResult.tokenCount; - } - - /** - * Get both the context string and its token count - */ - public getContextWithTokenCount(): { context: string; tokenCount: number } { - return { - context: this.contextResult.context, - tokenCount: this.contextResult.tokenCount - }; - } -} \ No newline at end of file diff --git a/ts/context/index.ts b/ts/context/index.ts deleted file mode 100644 index c4b4eef..0000000 --- a/ts/context/index.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { EnhancedContext } from './enhanced-context.js'; -import { TaskContextFactory } from './task-context-factory.js'; -import { ConfigManager } from './config-manager.js'; -import { ContextTrimmer } from './context-trimmer.js'; -import { LazyFileLoader } from './lazy-file-loader.js'; -import { ContextCache } from './context-cache.js'; -import { ContextAnalyzer } from './context-analyzer.js'; -import { DiffProcessor } from './diff-processor.js'; -import type { - ContextMode, - IContextConfig, - IContextResult, - IFileInfo, - ITrimConfig, - ITaskConfig, - TaskType, - ICacheConfig, - IAnalyzerConfig, - IPrioritizationWeights, - ITierConfig, - ITierSettings, - IFileMetadata, - ICacheEntry, - IFileDependencies, - IFileAnalysis, - IAnalysisResult, - IIterativeConfig, - IIterativeContextResult, - IDiffFileInfo, - IProcessedDiff, - IDiffProcessorOptions -} from './types.js'; - -export { - // Classes - EnhancedContext, - TaskContextFactory, - ConfigManager, - ContextTrimmer, - LazyFileLoader, - ContextCache, - ContextAnalyzer, - DiffProcessor, -}; - -// Types -export type { - ContextMode, - IContextConfig, - IContextResult, - IFileInfo, - ITrimConfig, - ITaskConfig, - TaskType, - ICacheConfig, - IAnalyzerConfig, - IPrioritizationWeights, - ITierConfig, - ITierSettings, - IFileMetadata, - ICacheEntry, - IFileDependencies, - IFileAnalysis, - IAnalysisResult, - IIterativeConfig, - IIterativeContextResult, - IDiffFileInfo, - IProcessedDiff, - IDiffProcessorOptions -}; \ No newline at end of file diff --git a/ts/context/iterative-context-builder.ts b/ts/context/iterative-context-builder.ts deleted file mode 100644 index e68f421..0000000 --- a/ts/context/iterative-context-builder.ts +++ /dev/null @@ -1,512 +0,0 @@ -import * as plugins from '../plugins.js'; -import * as fs from 'fs'; -import { logger } from '../logging.js'; -import type { - TaskType, - IFileMetadata, - IFileInfo, - IIterativeContextResult, - IIterationState, - IFileSelectionDecision, - IContextSufficiencyDecision, - IIterativeConfig, -} from './types.js'; -import { LazyFileLoader } from './lazy-file-loader.js'; -import { ContextCache } from './context-cache.js'; -import { ContextAnalyzer } from './context-analyzer.js'; -import { ConfigManager } from './config-manager.js'; - -/** - * Iterative context builder that uses AI to intelligently select files - * across multiple iterations until sufficient context is gathered - */ -export class IterativeContextBuilder { - private projectRoot: string; - private lazyLoader: LazyFileLoader; - private cache: ContextCache; - private analyzer: ContextAnalyzer; - private config: Required; - private tokenBudget: number = 190000; - private openaiInstance: plugins.smartai.OpenAiProvider; - private externalOpenaiInstance?: plugins.smartai.OpenAiProvider; - - /** - * Creates a new IterativeContextBuilder - * @param projectRoot - Root directory of the project - * @param config - Iterative configuration - * @param openaiInstance - Optional pre-configured OpenAI provider instance - */ - constructor( - projectRoot: string, - config?: Partial, - openaiInstance?: plugins.smartai.OpenAiProvider - ) { - this.projectRoot = projectRoot; - this.lazyLoader = new LazyFileLoader(projectRoot); - this.cache = new ContextCache(projectRoot); - this.analyzer = new ContextAnalyzer(projectRoot); - this.externalOpenaiInstance = openaiInstance; - - // Default configuration - this.config = { - maxIterations: config?.maxIterations ?? 5, - firstPassFileLimit: config?.firstPassFileLimit ?? 10, - subsequentPassFileLimit: config?.subsequentPassFileLimit ?? 5, - temperature: config?.temperature ?? 0.3, - model: config?.model ?? 'gpt-4-turbo-preview', - }; - - } - - /** - * Initialize the builder - */ - public async initialize(): Promise { - await this.cache.init(); - const configManager = ConfigManager.getInstance(); - await configManager.initialize(this.projectRoot); - this.tokenBudget = configManager.getMaxTokens(); - - // Use external OpenAI instance if provided, otherwise create a new one - if (this.externalOpenaiInstance) { - this.openaiInstance = this.externalOpenaiInstance; - } else { - // Initialize OpenAI instance from environment - const qenvInstance = new plugins.qenv.Qenv(); - const openaiToken = await qenvInstance.getEnvVarOnDemand('OPENAI_TOKEN'); - if (!openaiToken) { - throw new Error('OPENAI_TOKEN environment variable is required for iterative context building'); - } - this.openaiInstance = new plugins.smartai.OpenAiProvider({ - openaiToken, - }); - await this.openaiInstance.start(); - } - } - - /** - * Build context iteratively using AI decision making - * @param taskType - Type of task being performed - * @param additionalContext - Optional additional context (e.g., git diff for commit tasks) - * @returns Complete iterative context result - */ - public async buildContextIteratively(taskType: TaskType, additionalContext?: string): Promise { - const startTime = Date.now(); - logger.log('info', '🤖 Starting iterative context building...'); - logger.log('info', ` Task: ${taskType}, Budget: ${this.tokenBudget} tokens, Max iterations: ${this.config.maxIterations}`); - - // Phase 1: Scan project files for metadata - logger.log('info', '📋 Scanning project files...'); - const metadata = await this.scanProjectFiles(taskType); - const totalEstimatedTokens = metadata.reduce((sum, m) => sum + m.estimatedTokens, 0); - logger.log('info', ` Found ${metadata.length} files (~${totalEstimatedTokens} estimated tokens)`); - - // Phase 2: Analyze files for initial prioritization - logger.log('info', '🔍 Analyzing file dependencies and importance...'); - const analysis = await this.analyzer.analyze(metadata, taskType, []); - logger.log('info', ` Analysis complete in ${analysis.analysisDuration}ms`); - - // Track state across iterations - const iterations: IIterationState[] = []; - let totalTokensUsed = 0; - let apiCallCount = 0; - let loadedContent = ''; - const includedFiles: IFileInfo[] = []; - - // If additional context (e.g., git diff) is provided, prepend it - if (additionalContext) { - // NOTE: additionalContext is expected to be pre-processed by DiffProcessor - // which intelligently samples large diffs to stay within token budget (100k default) - const MAX_DIFF_TOKENS = 200000; // Safety net for edge cases (DiffProcessor uses 100k budget) - - const diffSection = ` -====== GIT DIFF ====== - -${additionalContext} - -====== END OF GIT DIFF ====== -`; - - // Validate token count (should already be under budget from DiffProcessor) - const diffTokens = this.countTokens(diffSection); - - if (diffTokens > MAX_DIFF_TOKENS) { - logger.log('error', `❌ Pre-processed git diff exceeds safety limit (${diffTokens.toLocaleString()} tokens > ${MAX_DIFF_TOKENS.toLocaleString()} limit)`); - logger.log('error', ` This should not happen - DiffProcessor should have limited to ~100k tokens.`); - logger.log('error', ` Please check DiffProcessor configuration and output.`); - throw new Error( - `Pre-processed git diff size (${diffTokens.toLocaleString()} tokens) exceeds safety limit (${MAX_DIFF_TOKENS.toLocaleString()} tokens). ` + - `This indicates a bug in DiffProcessor or misconfiguration.` - ); - } - - loadedContent = diffSection; - totalTokensUsed += diffTokens; - logger.log('info', `📝 Added pre-processed git diff to context (${diffTokens.toLocaleString()} tokens)`); - } - - // Phase 3: Iterative file selection and loading - for (let iteration = 1; iteration <= this.config.maxIterations; iteration++) { - const iterationStart = Date.now(); - logger.log('info', `\n🤔 Iteration ${iteration}/${this.config.maxIterations}: Asking AI which files to examine...`); - - const remainingBudget = this.tokenBudget - totalTokensUsed; - logger.log('info', ` Token budget remaining: ${remainingBudget}/${this.tokenBudget} (${Math.round((remainingBudget / this.tokenBudget) * 100)}%)`); - - // Get AI decision on which files to load - const decision = await this.getFileSelectionDecision( - metadata, - analysis.files.slice(0, 30), // Top 30 files by importance - taskType, - iteration, - totalTokensUsed, - remainingBudget, - loadedContent - ); - apiCallCount++; - - logger.log('info', ` AI reasoning: ${decision.reasoning}`); - logger.log('info', ` AI requested ${decision.filesToLoad.length} files`); - - // Load requested files - const iterationFiles: IFileInfo[] = []; - let iterationTokens = 0; - - if (decision.filesToLoad.length > 0) { - logger.log('info', '📥 Loading requested files...'); - - for (const filePath of decision.filesToLoad) { - try { - const fileInfo = await this.loadFile(filePath); - if (totalTokensUsed + fileInfo.tokenCount! <= this.tokenBudget) { - const formattedFile = this.formatFileForContext(fileInfo); - loadedContent += formattedFile; - includedFiles.push(fileInfo); - iterationFiles.push(fileInfo); - iterationTokens += fileInfo.tokenCount!; - totalTokensUsed += fileInfo.tokenCount!; - - logger.log('info', ` ✓ ${fileInfo.relativePath} (${fileInfo.tokenCount} tokens)`); - } else { - logger.log('warn', ` ✗ ${fileInfo.relativePath} - would exceed budget, skipping`); - } - } catch (error) { - logger.log('warn', ` ✗ Failed to load ${filePath}: ${error.message}`); - } - } - } - - // Record iteration state - const iterationDuration = Date.now() - iterationStart; - iterations.push({ - iteration, - filesLoaded: iterationFiles, - tokensUsed: iterationTokens, - totalTokensUsed, - decision, - duration: iterationDuration, - }); - - logger.log('info', ` Iteration ${iteration} complete: ${iterationFiles.length} files loaded, ${iterationTokens} tokens used`); - - // Check if we should continue - if (totalTokensUsed >= this.tokenBudget * 0.95) { - logger.log('warn', '⚠️ Approaching token budget limit, stopping iterations'); - break; - } - - // Ask AI if context is sufficient - if (iteration < this.config.maxIterations) { - logger.log('info', '🤔 Asking AI if context is sufficient...'); - const sufficiencyDecision = await this.evaluateContextSufficiency( - loadedContent, - taskType, - iteration, - totalTokensUsed, - remainingBudget - iterationTokens - ); - apiCallCount++; - - logger.log('info', ` AI decision: ${sufficiencyDecision.sufficient ? '✅ SUFFICIENT' : '⏭️ NEEDS MORE'}`); - logger.log('info', ` Reasoning: ${sufficiencyDecision.reasoning}`); - - if (sufficiencyDecision.sufficient) { - logger.log('ok', '✅ Context building complete - AI determined context is sufficient'); - break; - } - } - } - - const totalDuration = Date.now() - startTime; - logger.log('ok', `\n✅ Iterative context building complete!`); - logger.log('info', ` Files included: ${includedFiles.length}`); - logger.log('info', ` Token usage: ${totalTokensUsed}/${this.tokenBudget} (${Math.round((totalTokensUsed / this.tokenBudget) * 100)}%)`); - logger.log('info', ` Iterations: ${iterations.length}, API calls: ${apiCallCount}`); - logger.log('info', ` Total duration: ${(totalDuration / 1000).toFixed(2)}s`); - - return { - context: loadedContent, - tokenCount: totalTokensUsed, - includedFiles, - trimmedFiles: [], - excludedFiles: [], - tokenSavings: 0, - iterationCount: iterations.length, - iterations, - apiCallCount, - totalDuration, - }; - } - - /** - * Scan project files based on task type - */ - private async scanProjectFiles(taskType: TaskType): Promise { - const configManager = ConfigManager.getInstance(); - const taskConfig = configManager.getTaskConfig(taskType); - - const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [ - 'ts/**/*.ts', - 'ts*/**/*.ts' - ]; - - const configGlobs = [ - 'package.json', - 'readme.md', - 'readme.hints.md', - 'npmextra.json' - ]; - - return await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]); - } - - /** - * Get AI decision on which files to load - */ - private async getFileSelectionDecision( - allMetadata: IFileMetadata[], - analyzedFiles: any[], - taskType: TaskType, - iteration: number, - tokensUsed: number, - remainingBudget: number, - loadedContent: string - ): Promise { - const isFirstIteration = iteration === 1; - const fileLimit = isFirstIteration - ? this.config.firstPassFileLimit - : this.config.subsequentPassFileLimit; - - const systemPrompt = this.buildFileSelectionPrompt( - allMetadata, - analyzedFiles, - taskType, - iteration, - tokensUsed, - remainingBudget, - loadedContent, - fileLimit - ); - - const response = await this.openaiInstance.chat({ - systemMessage: `You are an AI assistant that helps select the most relevant files for code analysis. -You must respond ONLY with valid JSON that can be parsed with JSON.parse(). -Do not wrap the JSON in markdown code blocks or add any other text.`, - userMessage: systemPrompt, - messageHistory: [], - }); - - // Parse JSON response, handling potential markdown formatting - const content = response.message.replace('```json', '').replace('```', '').trim(); - const parsed = JSON.parse(content); - - return { - reasoning: parsed.reasoning || 'No reasoning provided', - filesToLoad: parsed.files_to_load || [], - estimatedTokensNeeded: parsed.estimated_tokens_needed, - }; - } - - /** - * Build prompt for file selection - */ - private buildFileSelectionPrompt( - metadata: IFileMetadata[], - analyzedFiles: any[], - taskType: TaskType, - iteration: number, - tokensUsed: number, - remainingBudget: number, - loadedContent: string, - fileLimit: number - ): string { - const taskDescriptions = { - readme: 'generating a comprehensive README that explains the project\'s purpose, features, and API', - commit: 'analyzing code changes to generate an intelligent commit message', - description: 'generating a concise project description for package.json', - }; - - const alreadyLoadedFiles = loadedContent - ? loadedContent.split('\n======').slice(1).map(section => { - const match = section.match(/START OF FILE (.+?) ======/); - return match ? match[1] : ''; - }).filter(Boolean) - : []; - - const availableFiles = metadata - .filter(m => !alreadyLoadedFiles.includes(m.relativePath)) - .map(m => { - const analysis = analyzedFiles.find(a => a.path === m.path); - return `- ${m.relativePath} (${m.size} bytes, ~${m.estimatedTokens} tokens${analysis ? `, importance: ${analysis.importanceScore.toFixed(2)}` : ''})`; - }) - .join('\n'); - - return `You are building context for ${taskDescriptions[taskType]} in a TypeScript project. - -ITERATION: ${iteration} -TOKENS USED: ${tokensUsed}/${tokensUsed + remainingBudget} (${Math.round((tokensUsed / (tokensUsed + remainingBudget)) * 100)}%) -REMAINING BUDGET: ${remainingBudget} tokens - -${alreadyLoadedFiles.length > 0 ? `FILES ALREADY LOADED:\n${alreadyLoadedFiles.map(f => `- ${f}`).join('\n')}\n\n` : ''}AVAILABLE FILES (not yet loaded): -${availableFiles} - -Your task: Select up to ${fileLimit} files that will give you the MOST understanding for this ${taskType} task. - -${iteration === 1 ? `This is the FIRST iteration. Focus on: -- Main entry points (index.ts, main exports) -- Core classes and interfaces -- Package configuration -` : `This is iteration ${iteration}. You've already seen some files. Now focus on: -- Files that complement what you've already loaded -- Dependencies of already-loaded files -- Missing pieces for complete understanding -`} - -Consider: -1. File importance scores (if provided) -2. File paths (ts/index.ts is likely more important than ts/internal/utils.ts) -3. Token efficiency (prefer smaller files if they provide good information) -4. Remaining budget (${remainingBudget} tokens) - -Respond in JSON format: -{ - "reasoning": "Brief explanation of why you're selecting these files", - "files_to_load": ["path/to/file1.ts", "path/to/file2.ts"], - "estimated_tokens_needed": 15000 -}`; - } - - /** - * Evaluate if current context is sufficient - */ - private async evaluateContextSufficiency( - loadedContent: string, - taskType: TaskType, - iteration: number, - tokensUsed: number, - remainingBudget: number - ): Promise { - const prompt = `You have been building context for a ${taskType} task across ${iteration} iterations. - -CURRENT STATE: -- Tokens used: ${tokensUsed} -- Remaining budget: ${remainingBudget} -- Files loaded: ${loadedContent.split('\n======').length - 1} - -CONTEXT SO FAR: -${loadedContent.substring(0, 3000)}... (truncated for brevity) - -Question: Do you have SUFFICIENT context to successfully complete the ${taskType} task? - -Consider: -- For README: Do you understand the project's purpose, main features, API surface, and usage patterns? -- For commit: Do you understand what changed and why? -- For description: Do you understand the project's core value proposition? - -Respond in JSON format: -{ - "sufficient": true or false, - "reasoning": "Detailed explanation of your decision" -}`; - - const response = await this.openaiInstance.chat({ - systemMessage: `You are an AI assistant that evaluates whether gathered context is sufficient for a task. -You must respond ONLY with valid JSON that can be parsed with JSON.parse(). -Do not wrap the JSON in markdown code blocks or add any other text.`, - userMessage: prompt, - messageHistory: [], - }); - - // Parse JSON response, handling potential markdown formatting - const content = response.message.replace('```json', '').replace('```', '').trim(); - const parsed = JSON.parse(content); - - return { - sufficient: parsed.sufficient || false, - reasoning: parsed.reasoning || 'No reasoning provided', - }; - } - - /** - * Load a single file with caching - */ - private async loadFile(filePath: string): Promise { - // Try cache first - const cached = await this.cache.get(filePath); - if (cached) { - return { - path: filePath, - relativePath: plugins.path.relative(this.projectRoot, filePath), - contents: cached.contents, - tokenCount: cached.tokenCount, - }; - } - - // Load from disk - const contents = await plugins.fsInstance.file(filePath).encoding('utf8').read() as string; - const tokenCount = this.countTokens(contents); - const relativePath = plugins.path.relative(this.projectRoot, filePath); - - // Cache it - const stats = await fs.promises.stat(filePath); - await this.cache.set({ - path: filePath, - contents, - tokenCount, - mtime: Math.floor(stats.mtimeMs), - cachedAt: Date.now(), - }); - - return { - path: filePath, - relativePath, - contents, - tokenCount, - }; - } - - /** - * Format a file for inclusion in context - */ - private formatFileForContext(file: IFileInfo): string { - return ` -====== START OF FILE ${file.relativePath} ====== - -${file.contents} - -====== END OF FILE ${file.relativePath} ====== -`; - } - - /** - * Count tokens in text - */ - private countTokens(text: string): number { - try { - const tokens = plugins.gptTokenizer.encode(text); - return tokens.length; - } catch (error) { - return Math.ceil(text.length / 4); - } - } -} diff --git a/ts/context/lazy-file-loader.ts b/ts/context/lazy-file-loader.ts deleted file mode 100644 index ffb9b71..0000000 --- a/ts/context/lazy-file-loader.ts +++ /dev/null @@ -1,207 +0,0 @@ -import * as plugins from '../plugins.js'; -import * as fs from 'fs'; -import type { IFileMetadata, IFileInfo } from './types.js'; - -/** - * LazyFileLoader handles efficient file loading by: - * - Scanning files for metadata without loading contents - * - Providing fast file size and token estimates - * - Loading contents only when requested - * - Parallel loading of selected files - */ -export class LazyFileLoader { - private projectRoot: string; - private metadataCache: Map = new Map(); - - /** - * Creates a new LazyFileLoader - * @param projectRoot - Root directory of the project - */ - constructor(projectRoot: string) { - this.projectRoot = projectRoot; - } - - /** - * Scans files in given globs and creates metadata without loading contents - * @param globs - File patterns to scan (e.g., ['ts/**\/*.ts', 'test/**\/*.ts']) - * @returns Array of file metadata - */ - public async scanFiles(globs: string[]): Promise { - const metadata: IFileMetadata[] = []; - - for (const globPattern of globs) { - try { - const virtualDir = await plugins.smartfileFactory.virtualDirectoryFromPath(this.projectRoot); - // Filter files based on glob pattern using simple pattern matching - const smartFiles = virtualDir.filter(file => { - // Simple glob matching - const relativePath = file.relative; - if (globPattern.includes('**')) { - // Handle ** patterns - match any path - const pattern = globPattern.replace(/\*\*/g, '.*').replace(/\*/g, '[^/]*'); - return new RegExp(`^${pattern}$`).test(relativePath); - } else if (globPattern.includes('*')) { - // Handle single * patterns - const pattern = globPattern.replace(/\*/g, '[^/]*'); - return new RegExp(`^${pattern}$`).test(relativePath); - } else { - // Exact match - return relativePath === globPattern; - } - }).listFiles(); - - for (const smartFile of smartFiles) { - try { - const meta = await this.getMetadata(smartFile.absolutePath); - metadata.push(meta); - } catch (error) { - // Skip files that can't be read - console.warn(`Failed to get metadata for ${smartFile.absolutePath}:`, error.message); - } - } - } catch (error) { - // Skip patterns that don't match any files - console.warn(`No files found for pattern ${globPattern}`); - } - } - - return metadata; - } - - /** - * Gets metadata for a single file without loading contents - * @param filePath - Absolute path to the file - * @returns File metadata - */ - public async getMetadata(filePath: string): Promise { - // Check cache first - if (this.metadataCache.has(filePath)) { - const cached = this.metadataCache.get(filePath)!; - const currentStats = await fs.promises.stat(filePath); - - // Return cached if file hasn't changed - if (cached.mtime === Math.floor(currentStats.mtimeMs)) { - return cached; - } - } - - // Get file stats - const stats = await fs.promises.stat(filePath); - const relativePath = plugins.path.relative(this.projectRoot, filePath); - - // Estimate tokens: rough estimate of ~4 characters per token - // This is faster than reading and tokenizing the entire file - const estimatedTokens = Math.ceil(stats.size / 4); - - const metadata: IFileMetadata = { - path: filePath, - relativePath, - size: stats.size, - mtime: Math.floor(stats.mtimeMs), - estimatedTokens, - }; - - // Cache the metadata - this.metadataCache.set(filePath, metadata); - - return metadata; - } - - /** - * Loads file contents for selected files in parallel - * @param metadata - Array of file metadata to load - * @param tokenizer - Function to calculate accurate token count - * @returns Array of complete file info with contents - */ - public async loadFiles( - metadata: IFileMetadata[], - tokenizer: (content: string) => number - ): Promise { - // Load files in parallel - const loadPromises = metadata.map(async (meta) => { - try { - const contents = await plugins.fsInstance.file(meta.path).encoding('utf8').read() as string; - const tokenCount = tokenizer(contents); - - const fileInfo: IFileInfo = { - path: meta.path, - relativePath: meta.relativePath, - contents, - tokenCount, - importanceScore: meta.importanceScore, - }; - - return fileInfo; - } catch (error) { - console.warn(`Failed to load file ${meta.path}:`, error.message); - return null; - } - }); - - // Wait for all loads to complete and filter out failures - const results = await Promise.all(loadPromises); - return results.filter((r): r is IFileInfo => r !== null); - } - - /** - * Loads a single file with contents - * @param filePath - Absolute path to the file - * @param tokenizer - Function to calculate accurate token count - * @returns Complete file info with contents - */ - public async loadFile( - filePath: string, - tokenizer: (content: string) => number - ): Promise { - const meta = await this.getMetadata(filePath); - const contents = await plugins.fsInstance.file(filePath).encoding('utf8').read() as string; - const tokenCount = tokenizer(contents); - const relativePath = plugins.path.relative(this.projectRoot, filePath); - - return { - path: filePath, - relativePath, - contents, - tokenCount, - importanceScore: meta.importanceScore, - }; - } - - /** - * Updates importance scores for metadata entries - * @param scores - Map of file paths to importance scores - */ - public updateImportanceScores(scores: Map): void { - for (const [path, score] of scores) { - const meta = this.metadataCache.get(path); - if (meta) { - meta.importanceScore = score; - } - } - } - - /** - * Clears the metadata cache - */ - public clearCache(): void { - this.metadataCache.clear(); - } - - /** - * Gets total estimated tokens for all cached metadata - */ - public getTotalEstimatedTokens(): number { - let total = 0; - for (const meta of this.metadataCache.values()) { - total += meta.estimatedTokens; - } - return total; - } - - /** - * Gets cached metadata entries - */ - public getCachedMetadata(): IFileMetadata[] { - return Array.from(this.metadataCache.values()); - } -} diff --git a/ts/context/task-context-factory.ts b/ts/context/task-context-factory.ts deleted file mode 100644 index 2e1b9ae..0000000 --- a/ts/context/task-context-factory.ts +++ /dev/null @@ -1,120 +0,0 @@ -import * as plugins from '../plugins.js'; -import { IterativeContextBuilder } from './iterative-context-builder.js'; -import { ConfigManager } from './config-manager.js'; -import type { IIterativeContextResult, TaskType } from './types.js'; - -/** - * Factory class for creating task-specific context using iterative context building - */ -export class TaskContextFactory { - private projectDir: string; - private configManager: ConfigManager; - private openaiInstance?: any; // OpenAI provider instance - - /** - * Create a new TaskContextFactory - * @param projectDirArg The project directory - * @param openaiInstance Optional pre-configured OpenAI provider instance - */ - constructor(projectDirArg: string, openaiInstance?: any) { - this.projectDir = projectDirArg; - this.configManager = ConfigManager.getInstance(); - this.openaiInstance = openaiInstance; - } - - /** - * Initialize the factory - */ - public async initialize(): Promise { - await this.configManager.initialize(this.projectDir); - } - - /** - * Create context for README generation - */ - public async createContextForReadme(): Promise { - const iterativeBuilder = new IterativeContextBuilder( - this.projectDir, - this.configManager.getIterativeConfig(), - this.openaiInstance - ); - await iterativeBuilder.initialize(); - return await iterativeBuilder.buildContextIteratively('readme'); - } - - /** - * Create context for description generation - */ - public async createContextForDescription(): Promise { - const iterativeBuilder = new IterativeContextBuilder( - this.projectDir, - this.configManager.getIterativeConfig(), - this.openaiInstance - ); - await iterativeBuilder.initialize(); - return await iterativeBuilder.buildContextIteratively('description'); - } - - /** - * Create context for commit message generation - * @param gitDiff Optional git diff to include in the context - */ - public async createContextForCommit(gitDiff?: string): Promise { - const iterativeBuilder = new IterativeContextBuilder( - this.projectDir, - this.configManager.getIterativeConfig(), - this.openaiInstance - ); - await iterativeBuilder.initialize(); - return await iterativeBuilder.buildContextIteratively('commit', gitDiff); - } - - /** - * Create context for any task type - * @param taskType The task type to create context for - * @param additionalContent Optional additional content (currently not used) - */ - public async createContextForTask( - taskType: TaskType, - additionalContent?: string - ): Promise { - switch (taskType) { - case 'readme': - return this.createContextForReadme(); - case 'description': - return this.createContextForDescription(); - case 'commit': - return this.createContextForCommit(additionalContent); - default: - // Default to readme for unknown task types - return this.createContextForReadme(); - } - } - - /** - * Get token stats for all task types - */ - public async getTokenStats(): Promise> { - const taskTypes: TaskType[] = ['readme', 'description', 'commit']; - const stats: Record = {} as any; - - for (const taskType of taskTypes) { - const result = await this.createContextForTask(taskType); - stats[taskType] = { - tokenCount: result.tokenCount, - savings: result.tokenSavings, - includedFiles: result.includedFiles.length, - trimmedFiles: result.trimmedFiles.length, - excludedFiles: result.excludedFiles.length - }; - } - - return stats; - } -} diff --git a/ts/context/types.ts b/ts/context/types.ts deleted file mode 100644 index 29bc499..0000000 --- a/ts/context/types.ts +++ /dev/null @@ -1,324 +0,0 @@ -/** - * Context processing mode to control how context is built - */ -export type ContextMode = 'full' | 'trimmed' | 'summarized'; - -/** - * Configuration for context trimming - */ -export interface ITrimConfig { - /** Whether to remove function implementations */ - removeImplementations?: boolean; - /** Whether to preserve interface definitions */ - preserveInterfaces?: boolean; - /** Whether to preserve type definitions */ - preserveTypeDefs?: boolean; - /** Whether to preserve JSDoc comments */ - preserveJSDoc?: boolean; - /** Maximum lines to keep for function bodies (if not removing completely) */ - maxFunctionLines?: number; - /** Whether to remove normal comments (non-JSDoc) */ - removeComments?: boolean; - /** Whether to remove blank lines */ - removeBlankLines?: boolean; -} - -/** - * Task types that require different context optimization - */ -export type TaskType = 'readme' | 'commit' | 'description'; - -/** - * Configuration for different tasks - */ -export interface ITaskConfig { - /** The context mode to use for this task */ - mode?: ContextMode; - /** File paths to include for this task */ - includePaths?: string[]; - /** File paths to exclude for this task */ - excludePaths?: string[]; - /** For commit tasks, whether to focus on changed files */ - focusOnChangedFiles?: boolean; - /** For description tasks, whether to include package info */ - includePackageInfo?: boolean; -} - -/** - * Complete context configuration - */ -export interface IContextConfig { - /** Maximum tokens to use for context */ - maxTokens?: number; - /** Default context mode */ - defaultMode?: ContextMode; - /** Task-specific settings */ - taskSpecificSettings?: { - [key in TaskType]?: ITaskConfig; - }; - /** Trimming configuration */ - trimming?: ITrimConfig; - /** Cache configuration */ - cache?: ICacheConfig; - /** Analyzer configuration */ - analyzer?: IAnalyzerConfig; - /** Prioritization weights */ - prioritization?: IPrioritizationWeights; - /** Tier configuration for adaptive trimming */ - tiers?: ITierConfig; - /** Iterative context building configuration */ - iterative?: IIterativeConfig; -} - -/** - * Cache configuration - */ -export interface ICacheConfig { - /** Whether caching is enabled */ - enabled?: boolean; - /** Time-to-live in seconds */ - ttl?: number; - /** Maximum cache size in MB */ - maxSize?: number; - /** Cache directory path */ - directory?: string; -} - -/** - * Analyzer configuration - * Note: Smart analysis is always enabled; this config only controls advanced options - */ -export interface IAnalyzerConfig { - /** Whether to use AI refinement for selection (advanced, disabled by default) */ - useAIRefinement?: boolean; - /** AI model to use for refinement */ - aiModel?: string; -} - -/** - * Weights for file prioritization - */ -export interface IPrioritizationWeights { - /** Weight for dependency centrality */ - dependencyWeight?: number; - /** Weight for task relevance */ - relevanceWeight?: number; - /** Weight for token efficiency */ - efficiencyWeight?: number; - /** Weight for file recency */ - recencyWeight?: number; -} - -/** - * Tier configuration for adaptive trimming - */ -export interface ITierConfig { - essential?: ITierSettings; - important?: ITierSettings; - optional?: ITierSettings; -} - -/** - * Settings for a single tier - */ -export interface ITierSettings { - /** Minimum score to qualify for this tier */ - minScore: number; - /** Trimming level to apply */ - trimLevel: 'none' | 'light' | 'aggressive'; -} - -/** - * Basic file information interface - */ -export interface IFileInfo { - /** The file path */ - path: string; - /** The file contents */ - contents: string; - /** The file's relative path from the project root */ - relativePath: string; - /** The estimated token count of the file */ - tokenCount?: number; - /** The file's importance score (higher is more important) */ - importanceScore?: number; -} - -/** - * Result of context building - */ -export interface IContextResult { - /** The generated context string */ - context: string; - /** The total token count of the context */ - tokenCount: number; - /** Files included in the context */ - includedFiles: IFileInfo[]; - /** Files that were trimmed */ - trimmedFiles: IFileInfo[]; - /** Files that were excluded */ - excludedFiles: IFileInfo[]; - /** Token savings from trimming */ - tokenSavings: number; -} - -/** - * File metadata without contents (for lazy loading) - */ -export interface IFileMetadata { - /** The file path */ - path: string; - /** The file's relative path from the project root */ - relativePath: string; - /** File size in bytes */ - size: number; - /** Last modified time (Unix timestamp) */ - mtime: number; - /** Estimated token count (without loading full contents) */ - estimatedTokens: number; - /** The file's importance score */ - importanceScore?: number; -} - -/** - * Cache entry for a file - */ -export interface ICacheEntry { - /** File path */ - path: string; - /** File contents */ - contents: string; - /** Token count */ - tokenCount: number; - /** Last modified time when cached */ - mtime: number; - /** When this cache entry was created */ - cachedAt: number; -} - -/** - * Dependency information for a file - */ -export interface IFileDependencies { - /** File path */ - path: string; - /** Files this file imports */ - imports: string[]; - /** Files that import this file */ - importedBy: string[]; - /** Centrality score (0-1) - how central this file is in the dependency graph */ - centrality: number; -} - -/** - * Analysis result for a file - */ -export interface IFileAnalysis { - /** File path */ - path: string; - /** Task relevance score (0-1) */ - relevanceScore: number; - /** Dependency centrality score (0-1) */ - centralityScore: number; - /** Token efficiency score (0-1) */ - efficiencyScore: number; - /** Recency score (0-1) */ - recencyScore: number; - /** Combined importance score (0-1) */ - importanceScore: number; - /** Assigned tier */ - tier: 'essential' | 'important' | 'optional' | 'excluded'; - /** Reason for the score */ - reason?: string; -} - -/** - * Result of context analysis - */ -export interface IAnalysisResult { - /** Task type being analyzed */ - taskType: TaskType; - /** Analyzed files with scores */ - files: IFileAnalysis[]; - /** Dependency graph */ - dependencyGraph: Map; - /** Total files analyzed */ - totalFiles: number; - /** Analysis duration in ms */ - analysisDuration: number; -} - -/** - * Configuration for iterative context building - */ -export interface IIterativeConfig { - /** Maximum number of iterations allowed */ - maxIterations?: number; - /** Maximum files to request in first iteration */ - firstPassFileLimit?: number; - /** Maximum files to request in subsequent iterations */ - subsequentPassFileLimit?: number; - /** Temperature for AI decision making (0-1) */ - temperature?: number; - /** Model to use for iterative decisions */ - model?: string; -} - -/** - * AI decision for file selection - */ -export interface IFileSelectionDecision { - /** AI's reasoning for file selection */ - reasoning: string; - /** File paths to load */ - filesToLoad: string[]; - /** Estimated tokens needed */ - estimatedTokensNeeded?: number; -} - -/** - * AI decision for context sufficiency - */ -export interface IContextSufficiencyDecision { - /** Whether context is sufficient */ - sufficient: boolean; - /** AI's reasoning */ - reasoning: string; - /** Additional files needed (if not sufficient) */ - additionalFilesNeeded?: string[]; -} - -/** - * State for a single iteration - */ -export interface IIterationState { - /** Iteration number (1-based) */ - iteration: number; - /** Files loaded in this iteration */ - filesLoaded: IFileInfo[]; - /** Tokens used in this iteration */ - tokensUsed: number; - /** Total tokens used so far */ - totalTokensUsed: number; - /** AI decision made in this iteration */ - decision: IFileSelectionDecision | IContextSufficiencyDecision; - /** Duration of this iteration in ms */ - duration: number; -} - -/** - * Result of iterative context building - */ -export interface IIterativeContextResult extends IContextResult { - /** Number of iterations performed */ - iterationCount: number; - /** Details of each iteration */ - iterations: IIterationState[]; - /** Total API calls made */ - apiCallCount: number; - /** Total duration in ms */ - totalDuration: number; -} - -// Export DiffProcessor types -export type { IDiffFileInfo, IProcessedDiff, IDiffProcessorOptions } from './diff-processor.js'; \ No newline at end of file diff --git a/ts/plugins.ts b/ts/plugins.ts index 1dcbb7c..143a4f0 100644 --- a/ts/plugins.ts +++ b/ts/plugins.ts @@ -52,6 +52,5 @@ export { tspublish }; // third party scope import * as typedoc from 'typedoc'; -import * as gptTokenizer from 'gpt-tokenizer'; -export { typedoc, gptTokenizer }; +export { typedoc };