Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8fcc304ee3 | |||
| 69802b46b6 | |||
| e500455557 | |||
| 4029691ccd | |||
| 3b1c84d7e8 | |||
| f8d0895aab | |||
| d7ec2220a1 | |||
| c24ce31b1f | |||
| fec2017cc6 | |||
| 88fac91c79 | |||
| ce4da89da9 | |||
| 6524adea18 | |||
| 4bf0c02618 | |||
| f84a65217d | |||
| 3f22fc91ae | |||
| 11e65b92ec | |||
| 0a3080518f | |||
| d0a4ddbb4b | |||
| 481339d3cb | |||
| ebc3d760af |
62
changelog.md
62
changelog.md
@@ -1,5 +1,67 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-12-15 - 1.11.0 - feat(commit)
|
||||
Integrate DualAgentOrchestrator for commit message generation and improve diff/context handling
|
||||
|
||||
- Add @push.rocks/smartagent dependency and export it from plugins
|
||||
- Use DualAgentOrchestrator to generate and guardian-validate commit messages
|
||||
- Use DualAgentOrchestrator for changelog generation with guardian validation
|
||||
- Switch commit flow to TaskContextFactory and DiffProcessor for token-efficient context
|
||||
- Expose getOpenaiToken() and wire orchestrator with the project OpenAI token
|
||||
- Enhance iterative context builder and context components to better manage token budgets and sampling
|
||||
- Update npmextra.json with release config for @git.zone/cli and reference local smartagent package in package.json
|
||||
|
||||
## 2025-12-02 - 1.10.0 - feat(diff-processor)
|
||||
Improve diff sampling and file prioritization: increase inclusion thresholds, expand sampled context, and boost priority for interface/type and entry-point files
|
||||
|
||||
- Raise small/medium file thresholds used by DiffProcessor (smallFileLines 50 -> 300, mediumFileLines 200 -> 800) so more source files are included fully or summarized rather than treated as large metadata-only files
|
||||
- Increase sample window for medium files (sampleHeadLines/sampleTailLines 20 -> 75) to provide more context when summarizing diffs
|
||||
- Boost importance scoring for interfaces/type files and entry points (adds +20 for interfaces/.types and +15 for index/mod entry files) to prioritize critical API surface in diff processing
|
||||
- Keep other prioritization rules intact (source/test/config/docs/build heuristics), and align the aidoc commit DiffProcessor usage with the new defaults
|
||||
|
||||
## 2025-11-04 - 1.9.2 - fix(deps)
|
||||
Update dependencies and devDependencies to newer versions (bump multiple packages)
|
||||
|
||||
- Bumped devDependencies: @git.zone/tsbuild 2.6.8 -> 2.7.1, @git.zone/tsrun 1.2.46 -> 1.6.2, @git.zone/tstest 2.3.6 -> 2.7.0
|
||||
- Bumped runtime dependencies: @push.rocks/smartai 0.5.11 -> 0.8.0, @push.rocks/smartcli 4.0.11 -> 4.0.19, @push.rocks/smartgit 3.2.1 -> 3.3.1, @push.rocks/smartlog 3.1.9 -> 3.1.10, gpt-tokenizer 3.0.1 -> 3.2.0, typedoc 0.28.12 -> 0.28.14, typescript 5.9.2 -> 5.9.3
|
||||
- No source code changes in this commit; dependency-only updates. Run the test suite and CI to verify compatibility.
|
||||
|
||||
## 2025-11-04 - 1.9.1 - fix(iterative-context-builder)
|
||||
Rely on DiffProcessor for git diff pre-processing; remove raw char truncation, raise diff token safety, and improve logging
|
||||
|
||||
- Removed raw character-based truncation of additionalContext — diffs are expected to be pre-processed by DiffProcessor instead of blind substring truncation.
|
||||
- Now validates pre-processed diff token count only and treats DiffProcessor as the primary sampler (DiffProcessor typically uses a ~100k token budget).
|
||||
- Increased MAX_DIFF_TOKENS safety net to 200,000 to cover edge cases and avoid false positives; updated logs to reflect pre-processed diffs.
|
||||
- Improved error messaging to indicate a likely DiffProcessor misconfiguration when pre-processed diffs exceed the safety limit.
|
||||
- Updated informational logs to state that a pre-processed git diff was added to context.
|
||||
|
||||
## 2025-11-04 - 1.9.0 - feat(context)
|
||||
Add intelligent DiffProcessor to summarize and prioritize git diffs and integrate it into the commit context pipeline
|
||||
|
||||
- Add DiffProcessor (ts/context/diff-processor.ts) to intelligently process git diffs: include small files fully, summarize medium files (head/tail sampling), and mark very large files as metadata-only to stay within token budgets.
|
||||
- Integrate DiffProcessor into commit workflow (ts/aidocs_classes/commit.ts): preprocess raw diffs, emit processed diff statistics, and pass a token-efficient diff section into the TaskContextFactory for commit context generation.
|
||||
- Export DiffProcessor and its types through the context index and types (ts/context/index.ts, ts/context/types.ts) so other context components can reuse it.
|
||||
- Add comprehensive tests for the DiffProcessor behavior and integration (test/test.diffprocessor.node.ts) covering small/medium/large diffs, added/deleted files, prioritization, token budgets, and formatting for context.
|
||||
- Minor adjustments across context/task factories and builders to accept and propagate processed diff strings rather than raw diffs, reducing risk of token overflows during iterative context building.
|
||||
|
||||
## 2025-11-04 - 1.8.3 - fix(context)
|
||||
Prevent enormous git diffs and OOM during context building by adding exclusion patterns, truncation, and diagnostic logging
|
||||
|
||||
- Add comprehensive git diff exclusion globs (locks, build artifacts, maps, bundles, IDE folders, logs, caches) when collecting uncommitted diffs to avoid noisy/huge diffs
|
||||
- Pass glob patterns directly to smartgit.getUncommittedDiff for efficient server-side matching
|
||||
- Emit diagnostic statistics for diffs (files changed, total characters, estimated tokens, number of exclusion patterns) and warn on unusually large diffs
|
||||
- Introduce pre-tokenization safety checks in iterative context builder: truncate raw diff text if it exceeds MAX_DIFF_CHARS and throw a clear error if token count still exceeds MAX_DIFF_TOKENS
|
||||
- Format and log token counts using locale-aware formatting for clarity
|
||||
- Improve robustness of commit context generation to reduce risk of OOM / model-limit overruns
|
||||
|
||||
## 2025-11-03 - 1.8.0 - feat(context)
|
||||
Wire OpenAI provider through task context factory and add git-diff support to iterative context builder
|
||||
|
||||
- Pass AiDoc.openaiInstance through TaskContextFactory into IterativeContextBuilder to reuse the same OpenAI provider and avoid reinitialization.
|
||||
- IterativeContextBuilder now accepts an optional OpenAiProvider and an additionalContext string; when provided, git diffs (or other extra context) are prepended to the AI context and token counts are updated.
|
||||
- createContextForCommit now forwards the git diff into the iterative builder so commit-specific context includes the diff.
|
||||
- Updated aidocs_classes (commit, description, readme) to supply the existing openaiInstance when creating the TaskContextFactory.
|
||||
|
||||
## 2025-11-03 - 1.7.0 - feat(IterativeContextBuilder)
|
||||
Add iterative AI-driven context builder and integrate into task factory; add tests and iterative configuration
|
||||
|
||||
|
||||
@@ -31,5 +31,14 @@
|
||||
},
|
||||
"tsdoc": {
|
||||
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
|
||||
},
|
||||
"@git.zone/cli": {
|
||||
"release": {
|
||||
"registries": [
|
||||
"https://verdaccio.lossless.digital",
|
||||
"https://registry.npmjs.org"
|
||||
],
|
||||
"accessLevel": "public"
|
||||
}
|
||||
}
|
||||
}
|
||||
34
package.json
34
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@git.zone/tsdoc",
|
||||
"version": "1.7.0",
|
||||
"version": "1.11.0",
|
||||
"private": false,
|
||||
"description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
|
||||
"type": "module",
|
||||
@@ -19,30 +19,32 @@
|
||||
"buildDocs": "tsdoc"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^2.6.8",
|
||||
"@git.zone/tsrun": "^1.2.46",
|
||||
"@git.zone/tstest": "^2.3.6",
|
||||
"@types/node": "^22.15.17"
|
||||
"@git.zone/tsbuild": "^3.1.2",
|
||||
"@git.zone/tsrun": "^2.0.1",
|
||||
"@git.zone/tstest": "^3.1.3",
|
||||
"@types/node": "^25.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@git.zone/tspublish": "^1.10.3",
|
||||
"@push.rocks/early": "^4.0.3",
|
||||
"@push.rocks/early": "^4.0.4",
|
||||
"@push.rocks/npmextra": "^5.3.3",
|
||||
"@push.rocks/qenv": "^6.1.3",
|
||||
"@push.rocks/smartai": "^0.5.11",
|
||||
"@push.rocks/smartcli": "^4.0.11",
|
||||
"@push.rocks/smartagent": "file:../../push.rocks/smartagent",
|
||||
"@push.rocks/smartai": "^0.8.0",
|
||||
"@push.rocks/smartcli": "^4.0.19",
|
||||
"@push.rocks/smartdelay": "^3.0.5",
|
||||
"@push.rocks/smartfile": "^11.2.7",
|
||||
"@push.rocks/smartgit": "^3.2.1",
|
||||
"@push.rocks/smartinteract": "^2.0.15",
|
||||
"@push.rocks/smartlog": "^3.1.9",
|
||||
"@push.rocks/smartfile": "^13.1.0",
|
||||
"@push.rocks/smartfs": "^1.2.0",
|
||||
"@push.rocks/smartgit": "^3.3.1",
|
||||
"@push.rocks/smartinteract": "^2.0.16",
|
||||
"@push.rocks/smartlog": "^3.1.10",
|
||||
"@push.rocks/smartlog-destination-local": "^9.0.2",
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@push.rocks/smartshell": "^3.3.0",
|
||||
"@push.rocks/smarttime": "^4.0.6",
|
||||
"gpt-tokenizer": "^3.0.1",
|
||||
"typedoc": "^0.28.12",
|
||||
"typescript": "^5.9.2"
|
||||
"@push.rocks/smarttime": "^4.1.1",
|
||||
"gpt-tokenizer": "^3.4.0",
|
||||
"typedoc": "^0.28.15",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"files": [
|
||||
"ts/**/*",
|
||||
|
||||
5588
pnpm-lock.yaml
generated
5588
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -1,5 +0,0 @@
|
||||
onlyBuiltDependencies:
|
||||
- esbuild
|
||||
- mongodb-memory-server
|
||||
- puppeteer
|
||||
- sharp
|
||||
265
readme.md
265
readme.md
@@ -1,15 +1,18 @@
|
||||
# @git.zone/tsdoc 🚀
|
||||
**AI-Powered Documentation for TypeScript Projects**
|
||||
|
||||
> Stop writing documentation. Let AI understand your code and do it for you.
|
||||
AI-Powered Documentation for TypeScript Projects
|
||||
|
||||
## Issue Reporting and Security
|
||||
|
||||
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||
|
||||
## What is tsdoc?
|
||||
|
||||
`@git.zone/tsdoc` is a next-generation documentation tool that combines traditional TypeDoc generation with cutting-edge AI to create comprehensive, intelligent documentation for your TypeScript projects. It reads your code, understands it, and writes documentation that actually makes sense.
|
||||
`@git.zone/tsdoc` is a next-generation documentation CLI tool that combines traditional TypeDoc generation with cutting-edge AI to create comprehensive, intelligent documentation for your TypeScript projects. It reads your code, understands it, and writes documentation that actually makes sense.
|
||||
|
||||
### ✨ Key Features
|
||||
|
||||
- **🤖 AI-Enhanced Documentation** - Leverages GPT-5 and other models to generate contextual READMEs
|
||||
- **🤖 AI-Enhanced Documentation** - Leverages AI to generate contextual READMEs
|
||||
- **🧠 Smart Context Building** - Intelligent file prioritization with dependency analysis and caching
|
||||
- **📚 TypeDoc Integration** - Classic API documentation generation when you need it
|
||||
- **💬 Smart Commit Messages** - AI analyzes your changes and suggests meaningful commit messages
|
||||
@@ -22,9 +25,6 @@
|
||||
|
||||
```bash
|
||||
# Global installation (recommended)
|
||||
npm install -g @git.zone/tsdoc
|
||||
|
||||
# Or with pnpm
|
||||
pnpm add -g @git.zone/tsdoc
|
||||
|
||||
# Or use with npx
|
||||
@@ -66,7 +66,6 @@ tsdoc commit
|
||||
| `tsdoc typedoc` | Generate TypeDoc documentation |
|
||||
| `tsdoc commit` | Generate smart commit message |
|
||||
| `tsdoc tokens` | Analyze token usage for AI context |
|
||||
| `tsdoc context` | Display context information |
|
||||
|
||||
### Token Analysis
|
||||
|
||||
@@ -79,91 +78,25 @@ tsdoc tokens
|
||||
# Show detailed stats for all task types
|
||||
tsdoc tokens --all
|
||||
|
||||
# Test with trimmed context
|
||||
tsdoc tokens --trim
|
||||
# Show detailed breakdown with file listing
|
||||
tsdoc tokens --detailed --listFiles
|
||||
```
|
||||
|
||||
## Programmatic Usage
|
||||
### Command Options
|
||||
|
||||
### Generate Documentation Programmatically
|
||||
#### tsdoc aidoc
|
||||
- `--tokens` / `--showTokens` - Show token count before generating
|
||||
- `--tokensOnly` - Only show token count, don't generate
|
||||
|
||||
```typescript
|
||||
import { AiDoc } from '@git.zone/tsdoc';
|
||||
#### tsdoc typedoc
|
||||
- `--publicSubdir <dir>` - Output subdirectory within public folder
|
||||
|
||||
const generateDocs = async () => {
|
||||
const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-token' });
|
||||
await aiDoc.start();
|
||||
|
||||
// Generate README
|
||||
await aiDoc.buildReadme('./');
|
||||
|
||||
// Update package.json description
|
||||
await aiDoc.buildDescription('./');
|
||||
|
||||
// Get smart commit message
|
||||
const commit = await aiDoc.buildNextCommitObject('./');
|
||||
console.log(commit.recommendedNextVersionMessage);
|
||||
|
||||
// Don't forget to stop when done
|
||||
await aiDoc.stop();
|
||||
};
|
||||
```
|
||||
|
||||
### TypeDoc Generation
|
||||
|
||||
```typescript
|
||||
import { TypeDoc } from '@git.zone/tsdoc';
|
||||
|
||||
const typeDoc = new TypeDoc(process.cwd());
|
||||
await typeDoc.compile({ publicSubdir: 'docs' });
|
||||
```
|
||||
|
||||
### Smart Context Management
|
||||
|
||||
Control how tsdoc processes your codebase with the new intelligent context system:
|
||||
|
||||
```typescript
|
||||
import { EnhancedContext, ContextAnalyzer, LazyFileLoader, ContextCache } from '@git.zone/tsdoc';
|
||||
|
||||
const context = new EnhancedContext('./');
|
||||
await context.initialize();
|
||||
|
||||
// Set token budget
|
||||
context.setTokenBudget(100000);
|
||||
|
||||
// Choose context mode
|
||||
context.setContextMode('trimmed'); // 'full' | 'trimmed' | 'summarized'
|
||||
|
||||
// Build optimized context with smart prioritization
|
||||
const result = await context.buildContext('readme');
|
||||
console.log(`Tokens used: ${result.tokenCount}`);
|
||||
console.log(`Files included: ${result.includedFiles.length}`);
|
||||
console.log(`Token savings: ${result.tokenSavings}`);
|
||||
```
|
||||
|
||||
### Advanced: Using Individual Context Components
|
||||
|
||||
```typescript
|
||||
import { LazyFileLoader, ContextAnalyzer, ContextCache } from '@git.zone/tsdoc';
|
||||
|
||||
// Lazy file loading - scan metadata without loading contents
|
||||
const loader = new LazyFileLoader('./');
|
||||
const metadata = await loader.scanFiles(['ts/**/*.ts']);
|
||||
console.log(`Found ${metadata.length} files`);
|
||||
|
||||
// Analyze and prioritize files
|
||||
const analyzer = new ContextAnalyzer('./');
|
||||
const analysis = await analyzer.analyze(metadata, 'readme');
|
||||
|
||||
// Files are sorted by importance with dependency analysis
|
||||
for (const file of analysis.files) {
|
||||
console.log(`${file.path}: score ${file.importanceScore.toFixed(2)}, tier ${file.tier}`);
|
||||
}
|
||||
|
||||
// Context caching for performance
|
||||
const cache = new ContextCache('./', { enabled: true, ttl: 3600 });
|
||||
await cache.init();
|
||||
```
|
||||
#### tsdoc tokens
|
||||
- `--task <type>` - Specify task type: `readme`, `commit`, or `description`
|
||||
- `--all` - Show stats for all task types
|
||||
- `--detailed` - Show detailed token usage and costs
|
||||
- `--listFiles` - List all files included in context
|
||||
- `--model <name>` - Show usage for specific model (`gpt4`, `gpt35`)
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -171,7 +104,8 @@ Configure tsdoc via `npmextra.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"tsdoc": {
|
||||
"@git.zone/tsdoc": {
|
||||
"legal": "## License and Legal Information\n\n...",
|
||||
"context": {
|
||||
"maxTokens": 190000,
|
||||
"defaultMode": "trimmed",
|
||||
@@ -181,7 +115,6 @@ Configure tsdoc via `npmextra.json`:
|
||||
"maxSize": 100
|
||||
},
|
||||
"analyzer": {
|
||||
"enabled": true,
|
||||
"useAIRefinement": false
|
||||
},
|
||||
"prioritization": {
|
||||
@@ -234,11 +167,6 @@ Configure tsdoc via `npmextra.json`:
|
||||
- **maxSize** - Maximum cache size in MB (default: 100)
|
||||
- **directory** - Cache directory path (default: .nogit/context-cache)
|
||||
|
||||
#### Analyzer Configuration
|
||||
- **enabled** - Enable smart file analysis (default: true)
|
||||
- **useAIRefinement** - Use AI for additional context refinement (default: false)
|
||||
- **aiModel** - Model for AI refinement (default: 'haiku')
|
||||
|
||||
## How It Works
|
||||
|
||||
### 🚀 Smart Context Building Pipeline
|
||||
@@ -270,21 +198,14 @@ The smart context system delivers significant improvements:
|
||||
| **Relevance** | Alphabetical sorting | Smart scoring | 🎯 90%+ relevant |
|
||||
| **Cache Hits** | None | 70-80% | 🚀 Major speedup |
|
||||
|
||||
### Traditional Context Optimization
|
||||
|
||||
For projects where the analyzer is disabled, tsdoc still employs:
|
||||
|
||||
- **Intelligent Trimming** - Removes implementation details while preserving signatures
|
||||
- **JSDoc Preservation** - Keeps documentation comments
|
||||
- **Interface Prioritization** - Type definitions always included
|
||||
- **Token Budgeting** - Ensures optimal use of AI context windows
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `OPENAI_TOKEN` | Your OpenAI API key for AI features (required) |
|
||||
|
||||
The token can also be provided interactively on first run - it will be persisted in `~/.npmextra/kv/@git.zone/tsdoc.json`.
|
||||
|
||||
## Use Cases
|
||||
|
||||
### 🚀 Continuous Integration
|
||||
@@ -336,103 +257,6 @@ tsdoc commit > .git/COMMIT_EDITMSG
|
||||
}
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Multi-Module Projects
|
||||
|
||||
tsdoc automatically detects and documents multi-module projects:
|
||||
|
||||
```typescript
|
||||
const aiDoc = new AiDoc();
|
||||
await aiDoc.start();
|
||||
|
||||
// Process main project
|
||||
await aiDoc.buildReadme('./');
|
||||
|
||||
// Process submodules
|
||||
for (const module of ['packages/core', 'packages/cli']) {
|
||||
await aiDoc.buildReadme(module);
|
||||
}
|
||||
|
||||
await aiDoc.stop();
|
||||
```
|
||||
|
||||
### Custom Context Building
|
||||
|
||||
Fine-tune what gets sent to AI with task-specific contexts:
|
||||
|
||||
```typescript
|
||||
import { TaskContextFactory } from '@git.zone/tsdoc';
|
||||
|
||||
const factory = new TaskContextFactory('./');
|
||||
await factory.initialize();
|
||||
|
||||
// Get optimized context for specific tasks
|
||||
const readmeContext = await factory.createContextForReadme();
|
||||
const commitContext = await factory.createContextForCommit();
|
||||
const descContext = await factory.createContextForDescription();
|
||||
```
|
||||
|
||||
### Dependency Graph Analysis
|
||||
|
||||
Understand your codebase structure:
|
||||
|
||||
```typescript
|
||||
import { ContextAnalyzer } from '@git.zone/tsdoc';
|
||||
|
||||
const analyzer = new ContextAnalyzer('./');
|
||||
const analysis = await analyzer.analyze(metadata, 'readme');
|
||||
|
||||
// Explore dependency graph
|
||||
for (const [path, deps] of analysis.dependencyGraph) {
|
||||
console.log(`${path}:`);
|
||||
console.log(` Imports: ${deps.imports.length}`);
|
||||
console.log(` Imported by: ${deps.importedBy.length}`);
|
||||
console.log(` Centrality: ${deps.centrality.toFixed(3)}`);
|
||||
}
|
||||
```
|
||||
|
||||
## Performance & Optimization
|
||||
|
||||
### ⚡ Performance Features
|
||||
|
||||
- **Lazy Loading** - Files scanned for metadata before content loading
|
||||
- **Parallel Processing** - Multiple files loaded simultaneously
|
||||
- **Smart Caching** - Results cached with mtime-based invalidation
|
||||
- **Incremental Updates** - Only reprocess changed files
|
||||
- **Streaming** - Minimal memory footprint
|
||||
|
||||
### 💰 Cost Optimization
|
||||
|
||||
The smart context system significantly reduces AI API costs:
|
||||
|
||||
```typescript
|
||||
// Check token usage before and after optimization
|
||||
import { EnhancedContext } from '@git.zone/tsdoc';
|
||||
|
||||
const context = new EnhancedContext('./');
|
||||
await context.initialize();
|
||||
|
||||
// Build with analyzer enabled
|
||||
const result = await context.buildContext('readme');
|
||||
console.log(`Tokens: ${result.tokenCount}`);
|
||||
console.log(`Savings: ${result.tokenSavings} (${(result.tokenSavings/result.tokenCount*100).toFixed(1)}%)`);
|
||||
```
|
||||
|
||||
### 📊 Token Analysis
|
||||
|
||||
Monitor and optimize your token usage:
|
||||
|
||||
```bash
|
||||
# Analyze current token usage
|
||||
tsdoc tokens
|
||||
|
||||
# Compare modes
|
||||
tsdoc tokens --mode full # No optimization
|
||||
tsdoc tokens --mode trimmed # Standard optimization
|
||||
tsdoc tokens --analyze # With smart prioritization
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Node.js** >= 18.0.0
|
||||
@@ -446,21 +270,15 @@ tsdoc tokens --analyze # With smart prioritization
|
||||
If you hit token limits, try:
|
||||
|
||||
```bash
|
||||
# Enable smart analyzer (default)
|
||||
tsdoc aidoc
|
||||
|
||||
# Use aggressive trimming
|
||||
tsdoc aidoc --trim
|
||||
|
||||
# Check token usage details
|
||||
tsdoc tokens --all --analyze
|
||||
tsdoc tokens --all --detailed
|
||||
```
|
||||
|
||||
Or configure stricter limits:
|
||||
Or configure stricter limits in `npmextra.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"tsdoc": {
|
||||
"@git.zone/tsdoc": {
|
||||
"context": {
|
||||
"maxTokens": 100000,
|
||||
"tiers": {
|
||||
@@ -484,19 +302,16 @@ tsdoc aidoc
|
||||
|
||||
### Slow Performance
|
||||
|
||||
Enable caching and adjust settings:
|
||||
Enable caching and adjust settings in `npmextra.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"tsdoc": {
|
||||
"@git.zone/tsdoc": {
|
||||
"context": {
|
||||
"cache": {
|
||||
"enabled": true,
|
||||
"ttl": 7200,
|
||||
"maxSize": 200
|
||||
},
|
||||
"analyzer": {
|
||||
"enabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -525,9 +340,6 @@ Regenerate documentation with every change. Smart dependency analysis ensures no
|
||||
### 🎨 Beautiful Output
|
||||
Clean, professional documentation every time. AI understands your code's purpose and explains it clearly.
|
||||
|
||||
### 🛠️ Developer-Friendly
|
||||
Built by developers, for developers. Sensible defaults, powerful configuration, and extensive programmatic API.
|
||||
|
||||
### 💰 Cost-Effective
|
||||
Smart context optimization reduces AI API costs by 40-60% without sacrificing quality.
|
||||
|
||||
@@ -545,6 +357,7 @@ Smart context optimization reduces AI API costs by 40-60% without sacrificing qu
|
||||
│ ├── ContextCache # Performance caching
|
||||
│ ├── ContextAnalyzer # Intelligent file analysis
|
||||
│ ├── ContextTrimmer # Adaptive code trimming
|
||||
│ ├── DiffProcessor # Git diff optimization
|
||||
│ ├── ConfigManager # Configuration management
|
||||
│ └── TaskContextFactory # Task-specific contexts
|
||||
└── CLI # Command-line interface
|
||||
@@ -567,30 +380,28 @@ ContextTrimmer (tier-based)
|
||||
↓
|
||||
Token Budget (enforcement)
|
||||
↓
|
||||
AI Model (GPT-5)
|
||||
AI Model
|
||||
↓
|
||||
Generated Documentation
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
We appreciate your interest! However, we are not accepting external contributions at this time. If you find bugs or have feature requests, please open an issue.
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
|
||||
|
||||
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
Registered at District Court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
For any legal inquiries or further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
||||
@@ -33,7 +33,10 @@ tap.test('should build commit object', async () => {
|
||||
expect(commitObject).toHaveProperty('recommendedNextVersionLevel');
|
||||
expect(commitObject).toHaveProperty('recommendedNextVersionScope');
|
||||
expect(commitObject).toHaveProperty('recommendedNextVersionMessage');
|
||||
});
|
||||
|
||||
})
|
||||
tap.test('should stop AIdocs', async () => {
|
||||
await aidocs.stop();
|
||||
});
|
||||
|
||||
tap.start();
|
||||
|
||||
304
test/test.diffprocessor.node.ts
Normal file
304
test/test.diffprocessor.node.ts
Normal file
@@ -0,0 +1,304 @@
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import { DiffProcessor } from '../ts/context/diff-processor.js';
|
||||
|
||||
// Sample diff strings for testing
|
||||
const createSmallDiff = (filepath: string, addedLines = 5, removedLines = 3): string => {
|
||||
const lines: string[] = [];
|
||||
lines.push(`--- a/${filepath}`);
|
||||
lines.push(`+++ b/${filepath}`);
|
||||
lines.push(`@@ -1,10 +1,12 @@`);
|
||||
|
||||
for (let i = 0; i < removedLines; i++) {
|
||||
lines.push(`-removed line ${i + 1}`);
|
||||
}
|
||||
|
||||
for (let i = 0; i < addedLines; i++) {
|
||||
lines.push(`+added line ${i + 1}`);
|
||||
}
|
||||
|
||||
lines.push(' unchanged line');
|
||||
|
||||
return lines.join('\n');
|
||||
};
|
||||
|
||||
const createMediumDiff = (filepath: string): string => {
|
||||
const lines: string[] = [];
|
||||
lines.push(`--- a/${filepath}`);
|
||||
lines.push(`+++ b/${filepath}`);
|
||||
lines.push(`@@ -1,100 +1,150 @@`);
|
||||
|
||||
// 150 lines of changes
|
||||
for (let i = 0; i < 75; i++) {
|
||||
lines.push(`+added line ${i + 1}`);
|
||||
}
|
||||
|
||||
for (let i = 0; i < 75; i++) {
|
||||
lines.push(`-removed line ${i + 1}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
};
|
||||
|
||||
const createLargeDiff = (filepath: string): string => {
|
||||
const lines: string[] = [];
|
||||
lines.push(`--- a/${filepath}`);
|
||||
lines.push(`+++ b/${filepath}`);
|
||||
lines.push(`@@ -1,1000 +1,1500 @@`);
|
||||
|
||||
// 2500 lines of changes
|
||||
for (let i = 0; i < 1250; i++) {
|
||||
lines.push(`+added line ${i + 1}`);
|
||||
}
|
||||
|
||||
for (let i = 0; i < 1250; i++) {
|
||||
lines.push(`-removed line ${i + 1}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
};
|
||||
|
||||
const createDeletedFileDiff = (filepath: string): string => {
|
||||
return `--- a/${filepath}
|
||||
+++ /dev/null
|
||||
@@ -1,5 +0,0 @@
|
||||
-deleted line 1
|
||||
-deleted line 2
|
||||
-deleted line 3
|
||||
-deleted line 4
|
||||
-deleted line 5`;
|
||||
};
|
||||
|
||||
const createAddedFileDiff = (filepath: string): string => {
|
||||
return `--- /dev/null
|
||||
+++ b/${filepath}
|
||||
@@ -0,0 +1,5 @@
|
||||
+added line 1
|
||||
+added line 2
|
||||
+added line 3
|
||||
+added line 4
|
||||
+added line 5`;
|
||||
};
|
||||
|
||||
tap.test('DiffProcessor should parse small diff correctly', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const smallDiff = createSmallDiff('src/test.ts', 5, 3);
|
||||
|
||||
const result = processor.processDiffs([smallDiff]);
|
||||
|
||||
expect(result.totalFiles).toEqual(1);
|
||||
expect(result.fullDiffs.length).toEqual(1);
|
||||
expect(result.summarizedDiffs.length).toEqual(0);
|
||||
expect(result.metadataOnly.length).toEqual(0);
|
||||
expect(result.totalTokens).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should summarize medium diff', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const mediumDiff = createMediumDiff('src/medium-file.ts');
|
||||
|
||||
const result = processor.processDiffs([mediumDiff]);
|
||||
|
||||
expect(result.totalFiles).toEqual(1);
|
||||
expect(result.fullDiffs.length).toEqual(0);
|
||||
expect(result.summarizedDiffs.length).toEqual(1);
|
||||
expect(result.metadataOnly.length).toEqual(0);
|
||||
|
||||
// Verify the summarized diff contains the sample
|
||||
const formatted = processor.formatForContext(result);
|
||||
expect(formatted).toInclude('SUMMARIZED DIFFS');
|
||||
expect(formatted).toInclude('lines omitted');
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should handle large diff as metadata only', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const largeDiff = createLargeDiff('dist/bundle.js');
|
||||
|
||||
const result = processor.processDiffs([largeDiff]);
|
||||
|
||||
expect(result.totalFiles).toEqual(1);
|
||||
expect(result.fullDiffs.length).toEqual(0);
|
||||
expect(result.summarizedDiffs.length).toEqual(0);
|
||||
expect(result.metadataOnly.length).toEqual(1);
|
||||
|
||||
const formatted = processor.formatForContext(result);
|
||||
expect(formatted).toInclude('METADATA ONLY');
|
||||
expect(formatted).toInclude('dist/bundle.js');
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should prioritize source files over build artifacts', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const diffs = [
|
||||
createSmallDiff('dist/bundle.js'),
|
||||
createSmallDiff('src/important.ts'),
|
||||
createSmallDiff('build/output.js'),
|
||||
createSmallDiff('src/core.ts'),
|
||||
];
|
||||
|
||||
const result = processor.processDiffs(diffs);
|
||||
|
||||
expect(result.totalFiles).toEqual(4);
|
||||
|
||||
// Source files should be included fully first
|
||||
const formatted = processor.formatForContext(result);
|
||||
const srcImportantIndex = formatted.indexOf('src/important.ts');
|
||||
const srcCoreIndex = formatted.indexOf('src/core.ts');
|
||||
const distBundleIndex = formatted.indexOf('dist/bundle.js');
|
||||
const buildOutputIndex = formatted.indexOf('build/output.js');
|
||||
|
||||
// Source files should appear before build artifacts
|
||||
expect(srcImportantIndex).toBeLessThan(distBundleIndex);
|
||||
expect(srcCoreIndex).toBeLessThan(buildOutputIndex);
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should respect token budget', async () => {
|
||||
const processor = new DiffProcessor({
|
||||
maxDiffTokens: 500, // Very small budget to force metadata-only
|
||||
});
|
||||
|
||||
// Create multiple large diffs that will exceed budget
|
||||
const diffs = [
|
||||
createLargeDiff('src/file1.ts'),
|
||||
createLargeDiff('src/file2.ts'),
|
||||
createLargeDiff('src/file3.ts'),
|
||||
createLargeDiff('src/file4.ts'),
|
||||
];
|
||||
|
||||
const result = processor.processDiffs(diffs);
|
||||
|
||||
expect(result.totalTokens).toBeLessThanOrEqual(500);
|
||||
// With such a small budget and large files, most should be metadata only
|
||||
expect(result.metadataOnly.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should handle deleted files', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const deletedDiff = createDeletedFileDiff('src/old-file.ts');
|
||||
|
||||
const result = processor.processDiffs([deletedDiff]);
|
||||
|
||||
expect(result.totalFiles).toEqual(1);
|
||||
// Small deleted file should be included fully
|
||||
expect(result.fullDiffs.length).toEqual(1);
|
||||
|
||||
const formatted = processor.formatForContext(result);
|
||||
expect(formatted).toInclude('src/old-file.ts');
|
||||
// Verify the file appears in the output
|
||||
expect(formatted).toInclude('FULL DIFFS');
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should handle added files', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const addedDiff = createAddedFileDiff('src/new-file.ts');
|
||||
|
||||
const result = processor.processDiffs([addedDiff]);
|
||||
|
||||
expect(result.totalFiles).toEqual(1);
|
||||
// Small added file should be included fully
|
||||
expect(result.fullDiffs.length).toEqual(1);
|
||||
|
||||
const formatted = processor.formatForContext(result);
|
||||
expect(formatted).toInclude('src/new-file.ts');
|
||||
// Verify the file appears in the output
|
||||
expect(formatted).toInclude('FULL DIFFS');
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should handle mixed file sizes', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const diffs = [
|
||||
createSmallDiff('src/small.ts'),
|
||||
createMediumDiff('src/medium.ts'),
|
||||
createLargeDiff('dist/large.js'),
|
||||
];
|
||||
|
||||
const result = processor.processDiffs(diffs);
|
||||
|
||||
expect(result.totalFiles).toEqual(3);
|
||||
expect(result.fullDiffs.length).toEqual(1); // small file
|
||||
expect(result.summarizedDiffs.length).toEqual(1); // medium file
|
||||
expect(result.metadataOnly.length).toEqual(1); // large file
|
||||
|
||||
const formatted = processor.formatForContext(result);
|
||||
expect(formatted).toInclude('FULL DIFFS (1 files)');
|
||||
expect(formatted).toInclude('SUMMARIZED DIFFS (1 files)');
|
||||
expect(formatted).toInclude('METADATA ONLY (1 files)');
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should handle empty diff array', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const result = processor.processDiffs([]);
|
||||
|
||||
expect(result.totalFiles).toEqual(0);
|
||||
expect(result.fullDiffs.length).toEqual(0);
|
||||
expect(result.summarizedDiffs.length).toEqual(0);
|
||||
expect(result.metadataOnly.length).toEqual(0);
|
||||
expect(result.totalTokens).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should generate comprehensive summary', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const diffs = [
|
||||
createSmallDiff('src/file1.ts'),
|
||||
createSmallDiff('src/file2.ts'),
|
||||
createMediumDiff('src/file3.ts'),
|
||||
createLargeDiff('dist/bundle.js'),
|
||||
];
|
||||
|
||||
const result = processor.processDiffs(diffs);
|
||||
const formatted = processor.formatForContext(result);
|
||||
|
||||
expect(formatted).toInclude('GIT DIFF SUMMARY');
|
||||
expect(formatted).toInclude('Files changed: 4 total');
|
||||
expect(formatted).toInclude('included in full');
|
||||
expect(formatted).toInclude('summarized');
|
||||
expect(formatted).toInclude('metadata only');
|
||||
expect(formatted).toInclude('Estimated tokens:');
|
||||
expect(formatted).toInclude('END OF GIT DIFF');
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should handle custom options', async () => {
|
||||
const processor = new DiffProcessor({
|
||||
maxDiffTokens: 50000,
|
||||
smallFileLines: 30,
|
||||
mediumFileLines: 150,
|
||||
sampleHeadLines: 10,
|
||||
sampleTailLines: 10,
|
||||
});
|
||||
|
||||
const mediumDiff = createMediumDiff('src/file.ts'); // 150 lines
|
||||
const result = processor.processDiffs([mediumDiff]);
|
||||
|
||||
// With custom settings, this should be summarized (exactly at the mediumFileLines threshold)
|
||||
expect(result.summarizedDiffs.length).toEqual(1);
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should prioritize test files appropriately', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const diffs = [
|
||||
createSmallDiff('src/core.ts'),
|
||||
createSmallDiff('test/core.test.ts'),
|
||||
createSmallDiff('config.json'),
|
||||
];
|
||||
|
||||
const result = processor.processDiffs(diffs);
|
||||
const formatted = processor.formatForContext(result);
|
||||
|
||||
// Source files should come before test files
|
||||
const srcIndex = formatted.indexOf('src/core.ts');
|
||||
const testIndex = formatted.indexOf('test/core.test.ts');
|
||||
|
||||
expect(srcIndex).toBeLessThan(testIndex);
|
||||
});
|
||||
|
||||
tap.test('DiffProcessor should handle files with no changes gracefully', async () => {
|
||||
const processor = new DiffProcessor();
|
||||
const emptyDiff = `--- a/src/file.ts
|
||||
+++ b/src/file.ts
|
||||
@@ -1,1 +1,1 @@`;
|
||||
|
||||
const result = processor.processDiffs([emptyDiff]);
|
||||
|
||||
expect(result.totalFiles).toEqual(1);
|
||||
expect(result.fullDiffs.length).toEqual(1); // Still included as a small file
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -1,8 +0,0 @@
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import * as tsdoc from '../ts/index.js';
|
||||
|
||||
tap.test('first test', async () => {
|
||||
console.log('test');
|
||||
});
|
||||
|
||||
tap.start();
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@git.zone/tsdoc',
|
||||
version: '1.7.0',
|
||||
version: '1.11.0',
|
||||
description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { AiDoc } from '../classes.aidoc.js';
|
||||
import { ProjectContext } from './projectcontext.js';
|
||||
import { DiffProcessor } from '../context/diff-processor.js';
|
||||
|
||||
export interface INextCommitObject {
|
||||
recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project
|
||||
@@ -27,18 +28,101 @@ export class Commit {
|
||||
smartgitInstance,
|
||||
this.projectDir
|
||||
);
|
||||
const diffStringArray = await gitRepo.getUncommittedDiff([
|
||||
|
||||
// Define comprehensive exclusion patterns
|
||||
// smartgit@3.3.0+ supports glob patterns natively
|
||||
const excludePatterns = [
|
||||
// Lock files
|
||||
'pnpm-lock.yaml',
|
||||
'package-lock.json',
|
||||
]);
|
||||
'npm-shrinkwrap.json',
|
||||
'yarn.lock',
|
||||
'deno.lock',
|
||||
'bun.lockb',
|
||||
|
||||
// Build artifacts (main culprit for large diffs!)
|
||||
'dist/**',
|
||||
'dist_*/**', // dist_ts, dist_web, etc.
|
||||
'build/**',
|
||||
'.next/**',
|
||||
'out/**',
|
||||
'public/dist/**',
|
||||
|
||||
// Compiled/bundled files
|
||||
'**/*.js.map',
|
||||
'**/*.d.ts.map',
|
||||
'**/*.min.js',
|
||||
'**/*.bundle.js',
|
||||
'**/*.chunk.js',
|
||||
|
||||
// IDE/Editor directories
|
||||
'.claude/**',
|
||||
'.cursor/**',
|
||||
'.vscode/**',
|
||||
'.idea/**',
|
||||
'**/*.swp',
|
||||
'**/*.swo',
|
||||
|
||||
// Logs and caches
|
||||
'.nogit/**',
|
||||
'**/*.log',
|
||||
'.cache/**',
|
||||
'.rpt2_cache/**',
|
||||
'coverage/**',
|
||||
'.nyc_output/**',
|
||||
];
|
||||
|
||||
// Pass glob patterns directly to smartgit - it handles matching internally
|
||||
const diffStringArray = await gitRepo.getUncommittedDiff(excludePatterns);
|
||||
|
||||
// Process diffs intelligently using DiffProcessor
|
||||
let processedDiffString: string;
|
||||
|
||||
if (diffStringArray.length > 0) {
|
||||
// Diagnostic logging for raw diff statistics
|
||||
const totalChars = diffStringArray.join('\n\n').length;
|
||||
const estimatedTokens = Math.ceil(totalChars / 4);
|
||||
|
||||
console.log(`📊 Raw git diff statistics:`);
|
||||
console.log(` Files changed: ${diffStringArray.length}`);
|
||||
console.log(` Total characters: ${totalChars.toLocaleString()}`);
|
||||
console.log(` Estimated tokens: ${estimatedTokens.toLocaleString()}`);
|
||||
console.log(` Exclusion patterns: ${excludePatterns.length}`);
|
||||
|
||||
// Use DiffProcessor to intelligently handle large diffs
|
||||
const diffProcessor = new DiffProcessor({
|
||||
maxDiffTokens: 100000, // Reserve 100k tokens for diffs
|
||||
smallFileLines: 300, // Most source files are under 300 lines
|
||||
mediumFileLines: 800, // Only very large files get head/tail treatment
|
||||
sampleHeadLines: 75, // When sampling, show more context
|
||||
sampleTailLines: 75, // When sampling, show more context
|
||||
});
|
||||
|
||||
const processedDiff = diffProcessor.processDiffs(diffStringArray);
|
||||
processedDiffString = diffProcessor.formatForContext(processedDiff);
|
||||
|
||||
console.log(`📝 Processed diff statistics:`);
|
||||
console.log(` Full diffs: ${processedDiff.fullDiffs.length} files`);
|
||||
console.log(` Summarized: ${processedDiff.summarizedDiffs.length} files`);
|
||||
console.log(` Metadata only: ${processedDiff.metadataOnly.length} files`);
|
||||
console.log(` Final tokens: ${processedDiff.totalTokens.toLocaleString()}`);
|
||||
|
||||
if (estimatedTokens > 50000) {
|
||||
console.log(`✅ DiffProcessor reduced token usage: ${estimatedTokens.toLocaleString()} → ${processedDiff.totalTokens.toLocaleString()}`);
|
||||
}
|
||||
} else {
|
||||
processedDiffString = 'No changes.';
|
||||
}
|
||||
|
||||
// Use the new TaskContextFactory for optimized context
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
|
||||
await taskContextFactory.initialize();
|
||||
|
||||
// Generate context specifically for commit task
|
||||
const contextResult = await taskContextFactory.createContextForCommit(
|
||||
diffStringArray[0] ? diffStringArray.join('\n\n') : 'No changes.'
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
|
||||
this.projectDir,
|
||||
this.aiDocsRef.openaiInstance
|
||||
);
|
||||
await taskContextFactory.initialize();
|
||||
|
||||
// Generate context specifically for commit task
|
||||
const contextResult = await taskContextFactory.createContextForCommit(processedDiffString);
|
||||
|
||||
// Get the optimized context string
|
||||
let contextString = contextResult.context;
|
||||
@@ -53,8 +137,30 @@ export class Commit {
|
||||
console.log(`The model may not be able to process all information effectively.`);
|
||||
}
|
||||
|
||||
let result = await this.aiDocsRef.openaiInstance.chat({
|
||||
systemMessage: `
|
||||
// Use DualAgentOrchestrator for commit message generation with Guardian validation
|
||||
const commitOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
|
||||
openaiToken: this.aiDocsRef.getOpenaiToken(),
|
||||
defaultProvider: 'openai',
|
||||
guardianPolicyPrompt: `
|
||||
You validate commit messages for semantic versioning compliance.
|
||||
|
||||
APPROVE if:
|
||||
- Version level (fix/feat/BREAKING CHANGE) matches the scope of changes in the diff
|
||||
- Commit message is clear, professional, and follows conventional commit conventions
|
||||
- No personal information, licensing details, or AI mentions (Claude/Codex) included
|
||||
- JSON structure is valid with all required fields
|
||||
- Scope accurately reflects the changed modules/files
|
||||
|
||||
REJECT with specific feedback if:
|
||||
- Version level doesn't match the scope of changes (e.g., "feat" for a typo fix should be "fix")
|
||||
- Message is vague, unprofessional, or contains sensitive information
|
||||
- JSON is malformed or missing required fields (recommendedNextVersionLevel, recommendedNextVersionScope, recommendedNextVersionMessage, recommendedNextVersionDetails, recommendedNextVersion)
|
||||
`,
|
||||
});
|
||||
|
||||
await commitOrchestrator.start();
|
||||
|
||||
const commitTaskPrompt = `
|
||||
You create a commit message for a git commit.
|
||||
The commit message should be based on the files in the project.
|
||||
You should not include any licensing information.
|
||||
@@ -79,29 +185,57 @@ For the recommendedNextVersionDetails, please only add a detail entries to the a
|
||||
You are being given the files of the project. You should use them to create the commit message.
|
||||
Also you are given a diff.
|
||||
Never mention CLAUDE code, or codex.
|
||||
`,
|
||||
messageHistory: [],
|
||||
userMessage: contextString,
|
||||
});
|
||||
|
||||
// console.log(result.message);
|
||||
Project context and diff:
|
||||
${contextString}
|
||||
`;
|
||||
|
||||
const commitResult = await commitOrchestrator.run(commitTaskPrompt);
|
||||
await commitOrchestrator.stop();
|
||||
|
||||
if (!commitResult.success) {
|
||||
throw new Error(`Commit message generation failed: ${commitResult.status}`);
|
||||
}
|
||||
|
||||
const resultObject: INextCommitObject = JSON.parse(
|
||||
result.message.replace('```json', '').replace('```', '')
|
||||
commitResult.result.replace('```json', '').replace('```', '')
|
||||
);
|
||||
|
||||
const previousChangelogPath = plugins.path.join(this.projectDir, 'changelog.md');
|
||||
let previousChangelog: plugins.smartfile.SmartFile;
|
||||
if (await plugins.smartfile.fs.fileExists(previousChangelogPath)) {
|
||||
previousChangelog = await plugins.smartfile.SmartFile.fromFilePath(previousChangelogPath);
|
||||
if (await plugins.fsInstance.file(previousChangelogPath).exists()) {
|
||||
previousChangelog = await plugins.smartfileFactory.fromFilePath(previousChangelogPath);
|
||||
}
|
||||
|
||||
if (!previousChangelog) {
|
||||
// lets build the changelog based on that
|
||||
const commitMessages = await gitRepo.getAllCommitMessages();
|
||||
console.log(JSON.stringify(commitMessages, null, 2));
|
||||
let result2 = await this.aiDocsRef.openaiInstance.chat({
|
||||
messageHistory: [],
|
||||
systemMessage: `
|
||||
|
||||
// Use DualAgentOrchestrator for changelog generation with Guardian validation
|
||||
const changelogOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
|
||||
openaiToken: this.aiDocsRef.getOpenaiToken(),
|
||||
defaultProvider: 'openai',
|
||||
guardianPolicyPrompt: `
|
||||
You validate changelog generation.
|
||||
|
||||
APPROVE if:
|
||||
- Changelog follows proper markdown format with ## headers for each version
|
||||
- Entries are chronologically ordered (newest first)
|
||||
- Version ranges for trivial commits are properly summarized
|
||||
- No duplicate or empty entries
|
||||
- Format matches: ## yyyy-mm-dd - x.x.x - scope
|
||||
|
||||
REJECT with feedback if:
|
||||
- Markdown formatting is incorrect
|
||||
- Entries are not meaningful or helpful
|
||||
- Dates or versions are malformed
|
||||
`,
|
||||
});
|
||||
|
||||
await changelogOrchestrator.start();
|
||||
|
||||
const changelogTaskPrompt = `
|
||||
You are building a changelog.md file for the project.
|
||||
Omit commits and versions that lack relevant changes, but make sure to mention them as a range with a summarizing message instead.
|
||||
|
||||
@@ -115,17 +249,23 @@ A changelog entry should look like this:
|
||||
You are given:
|
||||
* the commit messages of the project
|
||||
|
||||
Only return the changelog file, so it can be written directly to changelog.md`,
|
||||
userMessage: `
|
||||
Only return the changelog file content, so it can be written directly to changelog.md.
|
||||
|
||||
Here are the commit messages:
|
||||
|
||||
${JSON.stringify(commitMessages, null, 2)}
|
||||
`,
|
||||
});
|
||||
`;
|
||||
|
||||
previousChangelog = await plugins.smartfile.SmartFile.fromString(
|
||||
const changelogResult = await changelogOrchestrator.run(changelogTaskPrompt);
|
||||
await changelogOrchestrator.stop();
|
||||
|
||||
if (!changelogResult.success) {
|
||||
throw new Error(`Changelog generation failed: ${changelogResult.status}`);
|
||||
}
|
||||
|
||||
previousChangelog = plugins.smartfileFactory.fromString(
|
||||
previousChangelogPath,
|
||||
result2.message.replaceAll('```markdown', '').replaceAll('```', ''),
|
||||
changelogResult.result.replaceAll('```markdown', '').replaceAll('```', ''),
|
||||
'utf8'
|
||||
);
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@ export class Description {
|
||||
|
||||
public async build() {
|
||||
// Use the new TaskContextFactory for optimized context
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
|
||||
this.projectDir,
|
||||
this.aiDocsRef.openaiInstance
|
||||
);
|
||||
await taskContextFactory.initialize();
|
||||
|
||||
// Generate context specifically for description task
|
||||
@@ -62,8 +65,8 @@ Don't wrap the JSON in three ticks json!!!
|
||||
const npmextraJson = files.smartfilesNpmextraJSON;
|
||||
const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString());
|
||||
|
||||
npmextraJsonContent.gitzone.module.description = resultObject.description;
|
||||
npmextraJsonContent.gitzone.module.keywords = resultObject.keywords;
|
||||
npmextraJsonContent['@git.zone/cli'].module.description = resultObject.description;
|
||||
npmextraJsonContent['@git.zone/cli'].module.keywords = resultObject.keywords;
|
||||
|
||||
npmextraJson.contents = Buffer.from(JSON.stringify(npmextraJsonContent, null, 2));
|
||||
await npmextraJson.write();
|
||||
|
||||
@@ -13,31 +13,29 @@ export class ProjectContext {
|
||||
}
|
||||
|
||||
public async gatherFiles() {
|
||||
const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
const smartfilePackageJSON = await plugins.smartfileFactory.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'package.json'),
|
||||
this.projectDir,
|
||||
);
|
||||
const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
const smartfilesReadme = await plugins.smartfileFactory.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'readme.md'),
|
||||
this.projectDir,
|
||||
);
|
||||
|
||||
const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
const smartfilesReadmeHints = await plugins.smartfileFactory.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'readme.hints.md'),
|
||||
this.projectDir,
|
||||
);
|
||||
const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(
|
||||
const smartfilesNpmextraJSON = await plugins.smartfileFactory.fromFilePath(
|
||||
plugins.path.join(this.projectDir, 'npmextra.json'),
|
||||
this.projectDir,
|
||||
);
|
||||
const smartfilesMod = await plugins.smartfile.fs.fileTreeToObject(
|
||||
const smartfilesMod = await plugins.smartfileFactory.virtualDirectoryFromPath(
|
||||
this.projectDir,
|
||||
'ts*/**/*.ts',
|
||||
);
|
||||
const smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(
|
||||
).then(vd => vd.filter(f => f.relative.startsWith('ts') && f.relative.endsWith('.ts')).listFiles());
|
||||
const smartfilesTest = await plugins.smartfileFactory.virtualDirectoryFromPath(
|
||||
this.projectDir,
|
||||
'test/**/*.ts',
|
||||
);
|
||||
).then(vd => vd.filter(f => f.relative.startsWith('test/') && f.relative.endsWith('.ts')).listFiles());
|
||||
return {
|
||||
smartfilePackageJSON,
|
||||
smartfilesReadme,
|
||||
|
||||
@@ -18,7 +18,10 @@ export class Readme {
|
||||
let finalReadmeString = ``;
|
||||
|
||||
// Use the new TaskContextFactory for optimized context
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
|
||||
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
|
||||
this.projectDir,
|
||||
this.aiDocsRef.openaiInstance
|
||||
);
|
||||
await taskContextFactory.initialize();
|
||||
|
||||
// Generate context specifically for readme task
|
||||
@@ -33,7 +36,7 @@ export class Readme {
|
||||
const npmExtraJson = JSON.parse(
|
||||
(await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString()
|
||||
);
|
||||
const legalInfo = npmExtraJson?.tsdoc?.legal;
|
||||
const legalInfo = npmExtraJson?.['@git.zone/tsdoc']?.legal;
|
||||
if (!legalInfo) {
|
||||
const error = new Error(`No legal information found in npmextra.json`);
|
||||
console.log(error);
|
||||
@@ -102,7 +105,7 @@ The Readme should follow the following template:
|
||||
|
||||
IMPORTANT: YOU ARE NOW CREATING THE README FOR THE FOLLOWING SUB MODULE: ${subModule} !!!!!!!!!!!
|
||||
The Sub Module will be published with the following data:
|
||||
${JSON.stringify(plugins.smartfile.fs.toStringSync(plugins.path.join(paths.cwd, subModule, 'tspublish.json')), null, 2)}
|
||||
${JSON.stringify(await plugins.fsInstance.file(plugins.path.join(paths.cwd, subModule, 'tspublish.json')).encoding('utf8').read(), null, 2)}
|
||||
|
||||
|
||||
The Readme should follow the following template:
|
||||
@@ -144,7 +147,7 @@ The Readme should follow the following template:
|
||||
});
|
||||
|
||||
const subModuleReadmeString = result.message + '\n' + legalInfo;
|
||||
await plugins.smartfile.memory.toFs(subModuleReadmeString, plugins.path.join(paths.cwd, subModule, 'readme.md'));
|
||||
await plugins.fsInstance.file(plugins.path.join(paths.cwd, subModule, 'readme.md')).encoding('utf8').write(subModuleReadmeString);
|
||||
logger.log('success', `Built readme for ${subModule}`);
|
||||
}
|
||||
return result.message;
|
||||
|
||||
@@ -36,9 +36,25 @@ export class AiDoc {
|
||||
this.aidocInteract = new plugins.smartinteract.SmartInteract();
|
||||
this.qenvInstance = new plugins.qenv.Qenv();
|
||||
if (!(await this.qenvInstance.getEnvVarOnDemand('OPENAI_TOKEN'))) {
|
||||
// Migrate old KV store path to new path if needed
|
||||
const homeDir = plugins.smartpath.get.home();
|
||||
const oldKvPath = plugins.path.join(homeDir, '.npmextra/kv/tsdoc.json');
|
||||
const newKvDir = plugins.path.join(homeDir, '.npmextra/kv/@git.zone');
|
||||
const newKvPath = plugins.path.join(newKvDir, 'tsdoc.json');
|
||||
if (
|
||||
await plugins.fsInstance.file(oldKvPath).exists() &&
|
||||
!(await plugins.fsInstance.file(newKvPath).exists())
|
||||
) {
|
||||
console.log('Migrating tsdoc KeyValueStore to @git.zone/tsdoc...');
|
||||
await plugins.fsInstance.directory(newKvDir).recursive().create();
|
||||
await plugins.fsInstance.file(oldKvPath).copy(newKvPath);
|
||||
await plugins.fsInstance.file(oldKvPath).delete();
|
||||
console.log('Migration complete: tsdoc.json -> @git.zone/tsdoc.json');
|
||||
}
|
||||
|
||||
this.npmextraKV = new plugins.npmextra.KeyValueStore({
|
||||
typeArg: 'userHomeDir',
|
||||
identityArg: 'tsdoc',
|
||||
identityArg: '@git.zone/tsdoc',
|
||||
mandatoryKeys: ['OPENAI_TOKEN'],
|
||||
});
|
||||
|
||||
@@ -64,7 +80,7 @@ export class AiDoc {
|
||||
await this.npmextraKV.writeKey('OPENAI_TOKEN', this.openaiToken);
|
||||
}
|
||||
}
|
||||
if (!this.openaiToken) {
|
||||
if (!this.openaiToken && this.npmextraKV) {
|
||||
this.openaiToken = await this.npmextraKV.readKey('OPENAI_TOKEN');
|
||||
}
|
||||
|
||||
@@ -76,7 +92,15 @@ export class AiDoc {
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
await this.openaiInstance.stop();
|
||||
if (this.openaiInstance) {
|
||||
await this.openaiInstance.stop();
|
||||
}
|
||||
// No explicit cleanup needed for npmextraKV or aidocInteract
|
||||
// They don't keep event loop alive
|
||||
}
|
||||
|
||||
public getOpenaiToken(): string {
|
||||
return this.openaiToken;
|
||||
}
|
||||
|
||||
public async buildReadme(projectDirArg: string) {
|
||||
|
||||
@@ -33,19 +33,19 @@ export class TypeDoc {
|
||||
include: [],
|
||||
};
|
||||
let startDirectory = '';
|
||||
if (plugins.smartfile.fs.isDirectory(plugins.path.join(paths.cwd, './ts'))) {
|
||||
if (await plugins.fsInstance.directory(plugins.path.join(paths.cwd, './ts')).exists()) {
|
||||
data.include.push(plugins.path.join(paths.cwd, './ts/**/*'));
|
||||
startDirectory = 'ts';
|
||||
}
|
||||
|
||||
if (plugins.smartfile.fs.isDirectory(plugins.path.join(paths.cwd, './ts_web'))) {
|
||||
if (await plugins.fsInstance.directory(plugins.path.join(paths.cwd, './ts_web')).exists()) {
|
||||
data.include.push(plugins.path.join(paths.cwd, './ts_web/**/*'));
|
||||
if (!startDirectory) {
|
||||
startDirectory = 'ts_web';
|
||||
}
|
||||
}
|
||||
|
||||
await plugins.smartfile.memory.toFs(JSON.stringify(data), paths.tsconfigFile);
|
||||
await plugins.fsInstance.file(paths.tsconfigFile).encoding('utf8').write(JSON.stringify(data));
|
||||
let targetDir = paths.publicDir;
|
||||
if (options?.publicSubdir) {
|
||||
targetDir = plugins.path.join(targetDir, options.publicSubdir);
|
||||
@@ -53,6 +53,6 @@ export class TypeDoc {
|
||||
await this.smartshellInstance.exec(
|
||||
`typedoc --tsconfig ${paths.tsconfigFile} --out ${targetDir} ${startDirectory}/index.ts`,
|
||||
);
|
||||
plugins.smartfile.fs.remove(paths.tsconfigFile);
|
||||
await plugins.fsInstance.file(paths.tsconfigFile).delete();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,7 +152,7 @@ export const run = async () => {
|
||||
tsdocCli.addCommand('test').subscribe((argvArg) => {
|
||||
tsdocCli.triggerCommand('typedoc', argvArg);
|
||||
process.on('exit', async () => {
|
||||
await plugins.smartfile.fs.remove(paths.publicDir);
|
||||
await plugins.fsInstance.directory(paths.publicDir).recursive().delete();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ export class ConfigManager {
|
||||
const npmextraJsonPath = plugins.path.join(this.projectDir, 'npmextra.json');
|
||||
|
||||
// Check if file exists
|
||||
const fileExists = await plugins.smartfile.fs.fileExists(npmextraJsonPath);
|
||||
const fileExists = await plugins.fsInstance.file(npmextraJsonPath).exists();
|
||||
if (!fileExists) {
|
||||
return;
|
||||
}
|
||||
@@ -138,13 +138,13 @@ export class ConfigManager {
|
||||
}
|
||||
|
||||
// Read the npmextra.json file
|
||||
const npmextraJsonFile = await plugins.smartfile.SmartFile.fromFilePath(npmextraJsonPath);
|
||||
const npmextraJsonFile = await plugins.smartfileFactory.fromFilePath(npmextraJsonPath);
|
||||
const npmextraContent = JSON.parse(npmextraJsonFile.contents.toString());
|
||||
|
||||
// Check for tsdoc context configuration
|
||||
if (npmextraContent?.tsdoc?.context) {
|
||||
if (npmextraContent?.['@git.zone/tsdoc']?.context) {
|
||||
// Merge with default config
|
||||
this.config = this.mergeConfigs(this.config, npmextraContent.tsdoc.context);
|
||||
this.config = this.mergeConfigs(this.config, npmextraContent['@git.zone/tsdoc'].context);
|
||||
}
|
||||
|
||||
// Cache the config
|
||||
@@ -292,8 +292,8 @@ export class ConfigManager {
|
||||
const npmextraJsonPath = plugins.path.join(this.projectDir, 'npmextra.json');
|
||||
let npmextraContent = {};
|
||||
|
||||
if (await plugins.smartfile.fs.fileExists(npmextraJsonPath)) {
|
||||
const npmextraJsonFile = await plugins.smartfile.SmartFile.fromFilePath(npmextraJsonPath);
|
||||
if (await plugins.fsInstance.file(npmextraJsonPath).exists()) {
|
||||
const npmextraJsonFile = await plugins.smartfileFactory.fromFilePath(npmextraJsonPath);
|
||||
npmextraContent = JSON.parse(npmextraJsonFile.contents.toString()) || {};
|
||||
}
|
||||
|
||||
@@ -304,7 +304,7 @@ export class ConfigManager {
|
||||
|
||||
// Write back to npmextra.json
|
||||
const updatedContent = JSON.stringify(npmextraContent, null, 2);
|
||||
await plugins.smartfile.memory.toFs(updatedContent, npmextraJsonPath);
|
||||
await plugins.fsInstance.file(npmextraJsonPath).encoding('utf8').write(updatedContent);
|
||||
} catch (error) {
|
||||
console.error('Error updating context configuration:', error);
|
||||
}
|
||||
|
||||
@@ -116,7 +116,7 @@ export class ContextAnalyzer {
|
||||
// Parse imports from each file
|
||||
for (const meta of metadata) {
|
||||
try {
|
||||
const contents = await plugins.smartfile.fs.toStringSync(meta.path);
|
||||
const contents = await plugins.fsInstance.file(meta.path).encoding('utf8').read() as string;
|
||||
const imports = this.extractImports(contents, meta.path);
|
||||
|
||||
const deps = graph.get(meta.path)!;
|
||||
|
||||
@@ -39,13 +39,13 @@ export class ContextCache {
|
||||
}
|
||||
|
||||
// Ensure cache directory exists
|
||||
await plugins.smartfile.fs.ensureDir(this.cacheDir);
|
||||
await plugins.fsInstance.directory(this.cacheDir).recursive().create();
|
||||
|
||||
// Load cache index if it exists
|
||||
try {
|
||||
const indexExists = await plugins.smartfile.fs.fileExists(this.cacheIndexPath);
|
||||
const indexExists = await plugins.fsInstance.file(this.cacheIndexPath).exists();
|
||||
if (indexExists) {
|
||||
const indexContent = await plugins.smartfile.fs.toStringSync(this.cacheIndexPath);
|
||||
const indexContent = await plugins.fsInstance.file(this.cacheIndexPath).encoding('utf8').read() as string;
|
||||
const indexData = JSON.parse(indexContent) as ICacheEntry[];
|
||||
if (Array.isArray(indexData)) {
|
||||
for (const entry of indexData) {
|
||||
@@ -278,7 +278,7 @@ export class ContextCache {
|
||||
try {
|
||||
const entries = Array.from(this.cache.values());
|
||||
const content = JSON.stringify(entries, null, 2);
|
||||
await plugins.smartfile.memory.toFs(content, this.cacheIndexPath);
|
||||
await plugins.fsInstance.file(this.cacheIndexPath).encoding('utf8').write(content);
|
||||
} catch (error) {
|
||||
console.warn('Failed to persist cache index:', error.message);
|
||||
}
|
||||
|
||||
353
ts/context/diff-processor.ts
Normal file
353
ts/context/diff-processor.ts
Normal file
@@ -0,0 +1,353 @@
|
||||
/**
|
||||
* Intelligent git diff processor that handles large diffs by sampling and prioritization
|
||||
* instead of blind truncation.
|
||||
*/
|
||||
|
||||
export interface IDiffFileInfo {
|
||||
filepath: string;
|
||||
status: 'added' | 'modified' | 'deleted';
|
||||
linesAdded: number;
|
||||
linesRemoved: number;
|
||||
totalLines: number;
|
||||
estimatedTokens: number;
|
||||
diffContent: string;
|
||||
}
|
||||
|
||||
export interface IProcessedDiff {
|
||||
summary: string; // Human-readable overview
|
||||
fullDiffs: string[]; // Small files included fully
|
||||
summarizedDiffs: string[]; // Medium files with head/tail
|
||||
metadataOnly: string[]; // Large files, just stats
|
||||
totalFiles: number;
|
||||
totalTokens: number;
|
||||
}
|
||||
|
||||
export interface IDiffProcessorOptions {
|
||||
maxDiffTokens?: number; // Maximum tokens for entire diff section (default: 100000)
|
||||
smallFileLines?: number; // Files <= this are included fully (default: 50)
|
||||
mediumFileLines?: number; // Files <= this are summarized (default: 200)
|
||||
sampleHeadLines?: number; // Lines to show at start of medium files (default: 20)
|
||||
sampleTailLines?: number; // Lines to show at end of medium files (default: 20)
|
||||
}
|
||||
|
||||
export class DiffProcessor {
|
||||
private options: Required<IDiffProcessorOptions>;
|
||||
|
||||
constructor(options: IDiffProcessorOptions = {}) {
|
||||
this.options = {
|
||||
maxDiffTokens: options.maxDiffTokens ?? 100000,
|
||||
smallFileLines: options.smallFileLines ?? 50,
|
||||
mediumFileLines: options.mediumFileLines ?? 200,
|
||||
sampleHeadLines: options.sampleHeadLines ?? 20,
|
||||
sampleTailLines: options.sampleTailLines ?? 20,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process an array of git diffs into a structured, token-efficient format
|
||||
*/
|
||||
public processDiffs(diffStringArray: string[]): IProcessedDiff {
|
||||
// Parse all diffs into file info objects
|
||||
const fileInfos: IDiffFileInfo[] = diffStringArray
|
||||
.map(diffString => this.parseDiffFile(diffString))
|
||||
.filter(info => info !== null) as IDiffFileInfo[];
|
||||
|
||||
// Prioritize files (source files first, build artifacts last)
|
||||
const prioritized = this.prioritizeFiles(fileInfos);
|
||||
|
||||
const result: IProcessedDiff = {
|
||||
summary: '',
|
||||
fullDiffs: [],
|
||||
summarizedDiffs: [],
|
||||
metadataOnly: [],
|
||||
totalFiles: prioritized.length,
|
||||
totalTokens: 0,
|
||||
};
|
||||
|
||||
let tokensUsed = 0;
|
||||
const tokenBudget = this.options.maxDiffTokens;
|
||||
|
||||
// Categorize and include files based on size and token budget
|
||||
for (const fileInfo of prioritized) {
|
||||
const remainingBudget = tokenBudget - tokensUsed;
|
||||
|
||||
if (remainingBudget <= 0) {
|
||||
// Budget exhausted - rest are metadata only
|
||||
result.metadataOnly.push(this.formatMetadataOnly(fileInfo));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fileInfo.totalLines <= this.options.smallFileLines) {
|
||||
// Small file - include fully if budget allows
|
||||
if (fileInfo.estimatedTokens <= remainingBudget) {
|
||||
const statusPrefix = this.getFileStatusPrefix(fileInfo);
|
||||
result.fullDiffs.push(`${statusPrefix}${fileInfo.diffContent}`);
|
||||
tokensUsed += fileInfo.estimatedTokens;
|
||||
} else {
|
||||
result.metadataOnly.push(this.formatMetadataOnly(fileInfo));
|
||||
}
|
||||
} else if (fileInfo.totalLines <= this.options.mediumFileLines) {
|
||||
// Medium file - try to include summary with head/tail
|
||||
const summary = this.extractDiffSample(
|
||||
fileInfo,
|
||||
this.options.sampleHeadLines,
|
||||
this.options.sampleTailLines
|
||||
);
|
||||
const summaryTokens = Math.ceil(summary.length / 4); // Rough estimate
|
||||
|
||||
if (summaryTokens <= remainingBudget) {
|
||||
result.summarizedDiffs.push(summary);
|
||||
tokensUsed += summaryTokens;
|
||||
} else {
|
||||
result.metadataOnly.push(this.formatMetadataOnly(fileInfo));
|
||||
}
|
||||
} else {
|
||||
// Large file - metadata only
|
||||
result.metadataOnly.push(this.formatMetadataOnly(fileInfo));
|
||||
}
|
||||
}
|
||||
|
||||
result.totalTokens = tokensUsed;
|
||||
result.summary = this.generateSummary(result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format the processed diff for inclusion in context
|
||||
*/
|
||||
public formatForContext(processed: IProcessedDiff): string {
|
||||
const sections: string[] = [];
|
||||
|
||||
// Summary section
|
||||
sections.push('====== GIT DIFF SUMMARY ======');
|
||||
sections.push(processed.summary);
|
||||
sections.push('');
|
||||
|
||||
// Full diffs section
|
||||
if (processed.fullDiffs.length > 0) {
|
||||
sections.push(`====== FULL DIFFS (${processed.fullDiffs.length} files) ======`);
|
||||
sections.push(processed.fullDiffs.join('\n\n'));
|
||||
sections.push('');
|
||||
}
|
||||
|
||||
// Summarized diffs section
|
||||
if (processed.summarizedDiffs.length > 0) {
|
||||
sections.push(`====== SUMMARIZED DIFFS (${processed.summarizedDiffs.length} files) ======`);
|
||||
sections.push(processed.summarizedDiffs.join('\n\n'));
|
||||
sections.push('');
|
||||
}
|
||||
|
||||
// Metadata only section
|
||||
if (processed.metadataOnly.length > 0) {
|
||||
sections.push(`====== METADATA ONLY (${processed.metadataOnly.length} files) ======`);
|
||||
sections.push(processed.metadataOnly.join('\n'));
|
||||
sections.push('');
|
||||
}
|
||||
|
||||
sections.push('====== END OF GIT DIFF ======');
|
||||
|
||||
return sections.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single git diff string into file information
|
||||
*/
|
||||
private parseDiffFile(diffString: string): IDiffFileInfo | null {
|
||||
if (!diffString || diffString.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const lines = diffString.split('\n');
|
||||
let filepath = '';
|
||||
let status: 'added' | 'modified' | 'deleted' = 'modified';
|
||||
let linesAdded = 0;
|
||||
let linesRemoved = 0;
|
||||
|
||||
// Parse diff header to extract filepath and status
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('--- a/')) {
|
||||
filepath = line.substring(6);
|
||||
} else if (line.startsWith('+++ b/')) {
|
||||
const newPath = line.substring(6);
|
||||
if (newPath === '/dev/null') {
|
||||
status = 'deleted';
|
||||
} else if (filepath === '/dev/null') {
|
||||
status = 'added';
|
||||
filepath = newPath;
|
||||
} else {
|
||||
filepath = newPath;
|
||||
}
|
||||
} else if (line.startsWith('+') && !line.startsWith('+++')) {
|
||||
linesAdded++;
|
||||
} else if (line.startsWith('-') && !line.startsWith('---')) {
|
||||
linesRemoved++;
|
||||
}
|
||||
}
|
||||
|
||||
const totalLines = linesAdded + linesRemoved;
|
||||
const estimatedTokens = Math.ceil(diffString.length / 4);
|
||||
|
||||
return {
|
||||
filepath,
|
||||
status,
|
||||
linesAdded,
|
||||
linesRemoved,
|
||||
totalLines,
|
||||
estimatedTokens,
|
||||
diffContent: diffString,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Prioritize files by importance (source files before build artifacts)
|
||||
*/
|
||||
private prioritizeFiles(files: IDiffFileInfo[]): IDiffFileInfo[] {
|
||||
return files.sort((a, b) => {
|
||||
const scoreA = this.getFileImportanceScore(a.filepath);
|
||||
const scoreB = this.getFileImportanceScore(b.filepath);
|
||||
return scoreB - scoreA; // Higher score first
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate importance score for a file path
|
||||
*/
|
||||
private getFileImportanceScore(filepath: string): number {
|
||||
// Source files - highest priority
|
||||
if (filepath.match(/^(src|lib|app|components|pages|api)\//)) {
|
||||
return 100;
|
||||
}
|
||||
|
||||
// Test files - high priority
|
||||
if (filepath.match(/\.(test|spec)\.(ts|js|tsx|jsx)$/) || filepath.startsWith('test/')) {
|
||||
return 80;
|
||||
}
|
||||
|
||||
// Configuration files - medium-high priority
|
||||
if (filepath.match(/\.(json|yaml|yml|toml|config\.(ts|js))$/)) {
|
||||
return 60;
|
||||
}
|
||||
|
||||
// Documentation - medium priority
|
||||
if (filepath.match(/\.(md|txt|rst)$/)) {
|
||||
return 40;
|
||||
}
|
||||
|
||||
// Build artifacts - low priority
|
||||
if (filepath.match(/^(dist|build|out|\.next|public\/dist)\//)) {
|
||||
return 10;
|
||||
}
|
||||
|
||||
// Start with default priority
|
||||
let score = 50;
|
||||
|
||||
// Boost interface/type files - they're usually small but critical
|
||||
if (filepath.includes('interfaces/') || filepath.includes('.types.')) {
|
||||
score += 20;
|
||||
}
|
||||
|
||||
// Boost entry points
|
||||
if (filepath.endsWith('index.ts') || filepath.endsWith('mod.ts')) {
|
||||
score += 15;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract head and tail lines from a diff, omitting the middle
|
||||
*/
|
||||
private extractDiffSample(fileInfo: IDiffFileInfo, headLines: number, tailLines: number): string {
|
||||
const lines = fileInfo.diffContent.split('\n');
|
||||
const totalLines = lines.length;
|
||||
|
||||
if (totalLines <= headLines + tailLines) {
|
||||
// File is small enough to include fully
|
||||
return fileInfo.diffContent;
|
||||
}
|
||||
|
||||
// Extract file metadata from diff header
|
||||
const headerLines: string[] = [];
|
||||
let bodyStartIndex = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].startsWith('@@')) {
|
||||
headerLines.push(...lines.slice(0, i + 1));
|
||||
bodyStartIndex = i + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const bodyLines = lines.slice(bodyStartIndex);
|
||||
const head = bodyLines.slice(0, headLines);
|
||||
const tail = bodyLines.slice(-tailLines);
|
||||
const omittedLines = bodyLines.length - headLines - tailLines;
|
||||
|
||||
const statusEmoji = fileInfo.status === 'added' ? '➕' :
|
||||
fileInfo.status === 'deleted' ? '➖' : '📝';
|
||||
|
||||
const parts: string[] = [];
|
||||
parts.push(`${statusEmoji} FILE: ${fileInfo.filepath}`);
|
||||
parts.push(`CHANGES: +${fileInfo.linesAdded} lines, -${fileInfo.linesRemoved} lines (${fileInfo.totalLines} total)`);
|
||||
parts.push('');
|
||||
parts.push(...headerLines);
|
||||
parts.push(...head);
|
||||
parts.push('');
|
||||
parts.push(`[... ${omittedLines} lines omitted - use Read tool to see full file ...]`);
|
||||
parts.push('');
|
||||
parts.push(...tail);
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file status prefix with emoji
|
||||
*/
|
||||
private getFileStatusPrefix(fileInfo: IDiffFileInfo): string {
|
||||
const statusEmoji = fileInfo.status === 'added' ? '➕' :
|
||||
fileInfo.status === 'deleted' ? '➖' : '📝';
|
||||
return `${statusEmoji} `;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract filepath from diff content
|
||||
*/
|
||||
private extractFilepathFromDiff(diffContent: string): string {
|
||||
const lines = diffContent.split('\n');
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('+++ b/')) {
|
||||
return line.substring(6);
|
||||
}
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Format file info as metadata only
|
||||
*/
|
||||
private formatMetadataOnly(fileInfo: IDiffFileInfo): string {
|
||||
const statusEmoji = fileInfo.status === 'added' ? '➕' :
|
||||
fileInfo.status === 'deleted' ? '➖' : '📝';
|
||||
return `${statusEmoji} ${fileInfo.filepath} (+${fileInfo.linesAdded}, -${fileInfo.linesRemoved})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate human-readable summary of processed diff
|
||||
*/
|
||||
private generateSummary(result: IProcessedDiff): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`Files changed: ${result.totalFiles} total`);
|
||||
parts.push(`- ${result.fullDiffs.length} included in full`);
|
||||
parts.push(`- ${result.summarizedDiffs.length} summarized (head/tail shown)`);
|
||||
parts.push(`- ${result.metadataOnly.length} metadata only`);
|
||||
parts.push(`Estimated tokens: ~${result.totalTokens.toLocaleString()}`);
|
||||
|
||||
if (result.metadataOnly.length > 0) {
|
||||
parts.push('');
|
||||
parts.push('NOTE: Some files excluded to stay within token budget.');
|
||||
parts.push('Use Read tool with specific file paths to see full content.');
|
||||
}
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
}
|
||||
@@ -120,7 +120,7 @@ export class EnhancedContext {
|
||||
originalTokenCount = cached.tokenCount;
|
||||
} else {
|
||||
// Load file
|
||||
const fileData = await plugins.smartfile.fs.toStringSync(fileAnalysis.path);
|
||||
const fileData = await plugins.fsInstance.file(fileAnalysis.path).encoding('utf8').read() as string;
|
||||
contents = fileData;
|
||||
originalTokenCount = this.countTokens(contents);
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import { ContextTrimmer } from './context-trimmer.js';
|
||||
import { LazyFileLoader } from './lazy-file-loader.js';
|
||||
import { ContextCache } from './context-cache.js';
|
||||
import { ContextAnalyzer } from './context-analyzer.js';
|
||||
import { DiffProcessor } from './diff-processor.js';
|
||||
import type {
|
||||
ContextMode,
|
||||
IContextConfig,
|
||||
@@ -24,7 +25,10 @@ import type {
|
||||
IFileAnalysis,
|
||||
IAnalysisResult,
|
||||
IIterativeConfig,
|
||||
IIterativeContextResult
|
||||
IIterativeContextResult,
|
||||
IDiffFileInfo,
|
||||
IProcessedDiff,
|
||||
IDiffProcessorOptions
|
||||
} from './types.js';
|
||||
|
||||
export {
|
||||
@@ -36,6 +40,7 @@ export {
|
||||
LazyFileLoader,
|
||||
ContextCache,
|
||||
ContextAnalyzer,
|
||||
DiffProcessor,
|
||||
};
|
||||
|
||||
// Types
|
||||
@@ -58,5 +63,8 @@ export type {
|
||||
IFileAnalysis,
|
||||
IAnalysisResult,
|
||||
IIterativeConfig,
|
||||
IIterativeContextResult
|
||||
IIterativeContextResult,
|
||||
IDiffFileInfo,
|
||||
IProcessedDiff,
|
||||
IDiffProcessorOptions
|
||||
};
|
||||
@@ -28,17 +28,24 @@ export class IterativeContextBuilder {
|
||||
private config: Required<IIterativeConfig>;
|
||||
private tokenBudget: number = 190000;
|
||||
private openaiInstance: plugins.smartai.OpenAiProvider;
|
||||
private externalOpenaiInstance?: plugins.smartai.OpenAiProvider;
|
||||
|
||||
/**
|
||||
* Creates a new IterativeContextBuilder
|
||||
* @param projectRoot - Root directory of the project
|
||||
* @param config - Iterative configuration
|
||||
* @param openaiInstance - Optional pre-configured OpenAI provider instance
|
||||
*/
|
||||
constructor(projectRoot: string, config?: Partial<IIterativeConfig>) {
|
||||
constructor(
|
||||
projectRoot: string,
|
||||
config?: Partial<IIterativeConfig>,
|
||||
openaiInstance?: plugins.smartai.OpenAiProvider
|
||||
) {
|
||||
this.projectRoot = projectRoot;
|
||||
this.lazyLoader = new LazyFileLoader(projectRoot);
|
||||
this.cache = new ContextCache(projectRoot);
|
||||
this.analyzer = new ContextAnalyzer(projectRoot);
|
||||
this.externalOpenaiInstance = openaiInstance;
|
||||
|
||||
// Default configuration
|
||||
this.config = {
|
||||
@@ -60,24 +67,30 @@ export class IterativeContextBuilder {
|
||||
await configManager.initialize(this.projectRoot);
|
||||
this.tokenBudget = configManager.getMaxTokens();
|
||||
|
||||
// Initialize OpenAI instance
|
||||
const qenvInstance = new plugins.qenv.Qenv();
|
||||
const openaiToken = await qenvInstance.getEnvVarOnDemand('OPENAI_TOKEN');
|
||||
if (!openaiToken) {
|
||||
throw new Error('OPENAI_TOKEN environment variable is required for iterative context building');
|
||||
// Use external OpenAI instance if provided, otherwise create a new one
|
||||
if (this.externalOpenaiInstance) {
|
||||
this.openaiInstance = this.externalOpenaiInstance;
|
||||
} else {
|
||||
// Initialize OpenAI instance from environment
|
||||
const qenvInstance = new plugins.qenv.Qenv();
|
||||
const openaiToken = await qenvInstance.getEnvVarOnDemand('OPENAI_TOKEN');
|
||||
if (!openaiToken) {
|
||||
throw new Error('OPENAI_TOKEN environment variable is required for iterative context building');
|
||||
}
|
||||
this.openaiInstance = new plugins.smartai.OpenAiProvider({
|
||||
openaiToken,
|
||||
});
|
||||
await this.openaiInstance.start();
|
||||
}
|
||||
this.openaiInstance = new plugins.smartai.OpenAiProvider({
|
||||
openaiToken,
|
||||
});
|
||||
await this.openaiInstance.start();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build context iteratively using AI decision making
|
||||
* @param taskType - Type of task being performed
|
||||
* @param additionalContext - Optional additional context (e.g., git diff for commit tasks)
|
||||
* @returns Complete iterative context result
|
||||
*/
|
||||
public async buildContextIteratively(taskType: TaskType): Promise<IIterativeContextResult> {
|
||||
public async buildContextIteratively(taskType: TaskType, additionalContext?: string): Promise<IIterativeContextResult> {
|
||||
const startTime = Date.now();
|
||||
logger.log('info', '🤖 Starting iterative context building...');
|
||||
logger.log('info', ` Task: ${taskType}, Budget: ${this.tokenBudget} tokens, Max iterations: ${this.config.maxIterations}`);
|
||||
@@ -100,6 +113,38 @@ export class IterativeContextBuilder {
|
||||
let loadedContent = '';
|
||||
const includedFiles: IFileInfo[] = [];
|
||||
|
||||
// If additional context (e.g., git diff) is provided, prepend it
|
||||
if (additionalContext) {
|
||||
// NOTE: additionalContext is expected to be pre-processed by DiffProcessor
|
||||
// which intelligently samples large diffs to stay within token budget (100k default)
|
||||
const MAX_DIFF_TOKENS = 200000; // Safety net for edge cases (DiffProcessor uses 100k budget)
|
||||
|
||||
const diffSection = `
|
||||
====== GIT DIFF ======
|
||||
|
||||
${additionalContext}
|
||||
|
||||
====== END OF GIT DIFF ======
|
||||
`;
|
||||
|
||||
// Validate token count (should already be under budget from DiffProcessor)
|
||||
const diffTokens = this.countTokens(diffSection);
|
||||
|
||||
if (diffTokens > MAX_DIFF_TOKENS) {
|
||||
logger.log('error', `❌ Pre-processed git diff exceeds safety limit (${diffTokens.toLocaleString()} tokens > ${MAX_DIFF_TOKENS.toLocaleString()} limit)`);
|
||||
logger.log('error', ` This should not happen - DiffProcessor should have limited to ~100k tokens.`);
|
||||
logger.log('error', ` Please check DiffProcessor configuration and output.`);
|
||||
throw new Error(
|
||||
`Pre-processed git diff size (${diffTokens.toLocaleString()} tokens) exceeds safety limit (${MAX_DIFF_TOKENS.toLocaleString()} tokens). ` +
|
||||
`This indicates a bug in DiffProcessor or misconfiguration.`
|
||||
);
|
||||
}
|
||||
|
||||
loadedContent = diffSection;
|
||||
totalTokensUsed += diffTokens;
|
||||
logger.log('info', `📝 Added pre-processed git diff to context (${diffTokens.toLocaleString()} tokens)`);
|
||||
}
|
||||
|
||||
// Phase 3: Iterative file selection and loading
|
||||
for (let iteration = 1; iteration <= this.config.maxIterations; iteration++) {
|
||||
const iterationStart = Date.now();
|
||||
@@ -418,7 +463,7 @@ Do not wrap the JSON in markdown code blocks or add any other text.`,
|
||||
}
|
||||
|
||||
// Load from disk
|
||||
const contents = await plugins.smartfile.fs.toStringSync(filePath);
|
||||
const contents = await plugins.fsInstance.file(filePath).encoding('utf8').read() as string;
|
||||
const tokenCount = this.countTokens(contents);
|
||||
const relativePath = plugins.path.relative(this.projectRoot, filePath);
|
||||
|
||||
|
||||
@@ -31,16 +31,32 @@ export class LazyFileLoader {
|
||||
|
||||
for (const globPattern of globs) {
|
||||
try {
|
||||
const smartFiles = await plugins.smartfile.fs.fileTreeToObject(this.projectRoot, globPattern);
|
||||
const fileArray = Array.isArray(smartFiles) ? smartFiles : [smartFiles];
|
||||
const virtualDir = await plugins.smartfileFactory.virtualDirectoryFromPath(this.projectRoot);
|
||||
// Filter files based on glob pattern using simple pattern matching
|
||||
const smartFiles = virtualDir.filter(file => {
|
||||
// Simple glob matching
|
||||
const relativePath = file.relative;
|
||||
if (globPattern.includes('**')) {
|
||||
// Handle ** patterns - match any path
|
||||
const pattern = globPattern.replace(/\*\*/g, '.*').replace(/\*/g, '[^/]*');
|
||||
return new RegExp(`^${pattern}$`).test(relativePath);
|
||||
} else if (globPattern.includes('*')) {
|
||||
// Handle single * patterns
|
||||
const pattern = globPattern.replace(/\*/g, '[^/]*');
|
||||
return new RegExp(`^${pattern}$`).test(relativePath);
|
||||
} else {
|
||||
// Exact match
|
||||
return relativePath === globPattern;
|
||||
}
|
||||
}).listFiles();
|
||||
|
||||
for (const smartFile of fileArray) {
|
||||
for (const smartFile of smartFiles) {
|
||||
try {
|
||||
const meta = await this.getMetadata(smartFile.path);
|
||||
const meta = await this.getMetadata(smartFile.absolutePath);
|
||||
metadata.push(meta);
|
||||
} catch (error) {
|
||||
// Skip files that can't be read
|
||||
console.warn(`Failed to get metadata for ${smartFile.path}:`, error.message);
|
||||
console.warn(`Failed to get metadata for ${smartFile.absolutePath}:`, error.message);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -104,7 +120,7 @@ export class LazyFileLoader {
|
||||
// Load files in parallel
|
||||
const loadPromises = metadata.map(async (meta) => {
|
||||
try {
|
||||
const contents = await plugins.smartfile.fs.toStringSync(meta.path);
|
||||
const contents = await plugins.fsInstance.file(meta.path).encoding('utf8').read() as string;
|
||||
const tokenCount = tokenizer(contents);
|
||||
|
||||
const fileInfo: IFileInfo = {
|
||||
@@ -138,7 +154,7 @@ export class LazyFileLoader {
|
||||
tokenizer: (content: string) => number
|
||||
): Promise<IFileInfo> {
|
||||
const meta = await this.getMetadata(filePath);
|
||||
const contents = await plugins.smartfile.fs.toStringSync(filePath);
|
||||
const contents = await plugins.fsInstance.file(filePath).encoding('utf8').read() as string;
|
||||
const tokenCount = tokenizer(contents);
|
||||
const relativePath = plugins.path.relative(this.projectRoot, filePath);
|
||||
|
||||
|
||||
@@ -9,14 +9,17 @@ import type { IIterativeContextResult, TaskType } from './types.js';
|
||||
export class TaskContextFactory {
|
||||
private projectDir: string;
|
||||
private configManager: ConfigManager;
|
||||
private openaiInstance?: any; // OpenAI provider instance
|
||||
|
||||
/**
|
||||
* Create a new TaskContextFactory
|
||||
* @param projectDirArg The project directory
|
||||
* @param openaiInstance Optional pre-configured OpenAI provider instance
|
||||
*/
|
||||
constructor(projectDirArg: string) {
|
||||
constructor(projectDirArg: string, openaiInstance?: any) {
|
||||
this.projectDir = projectDirArg;
|
||||
this.configManager = ConfigManager.getInstance();
|
||||
this.openaiInstance = openaiInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -32,7 +35,8 @@ export class TaskContextFactory {
|
||||
public async createContextForReadme(): Promise<IIterativeContextResult> {
|
||||
const iterativeBuilder = new IterativeContextBuilder(
|
||||
this.projectDir,
|
||||
this.configManager.getIterativeConfig()
|
||||
this.configManager.getIterativeConfig(),
|
||||
this.openaiInstance
|
||||
);
|
||||
await iterativeBuilder.initialize();
|
||||
return await iterativeBuilder.buildContextIteratively('readme');
|
||||
@@ -44,7 +48,8 @@ export class TaskContextFactory {
|
||||
public async createContextForDescription(): Promise<IIterativeContextResult> {
|
||||
const iterativeBuilder = new IterativeContextBuilder(
|
||||
this.projectDir,
|
||||
this.configManager.getIterativeConfig()
|
||||
this.configManager.getIterativeConfig(),
|
||||
this.openaiInstance
|
||||
);
|
||||
await iterativeBuilder.initialize();
|
||||
return await iterativeBuilder.buildContextIteratively('description');
|
||||
@@ -52,16 +57,16 @@ export class TaskContextFactory {
|
||||
|
||||
/**
|
||||
* Create context for commit message generation
|
||||
* @param gitDiff Optional git diff to include (currently not used in iterative mode)
|
||||
* @param gitDiff Optional git diff to include in the context
|
||||
*/
|
||||
public async createContextForCommit(gitDiff?: string): Promise<IIterativeContextResult> {
|
||||
const iterativeBuilder = new IterativeContextBuilder(
|
||||
this.projectDir,
|
||||
this.configManager.getIterativeConfig()
|
||||
this.configManager.getIterativeConfig(),
|
||||
this.openaiInstance
|
||||
);
|
||||
await iterativeBuilder.initialize();
|
||||
// Note: git diff could be incorporated into the iterative prompts if needed
|
||||
return await iterativeBuilder.buildContextIteratively('commit');
|
||||
return await iterativeBuilder.buildContextIteratively('commit', gitDiff);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -318,4 +318,7 @@ export interface IIterativeContextResult extends IContextResult {
|
||||
apiCallCount: number;
|
||||
/** Total duration in ms */
|
||||
totalDuration: number;
|
||||
}
|
||||
}
|
||||
|
||||
// Export DiffProcessor types
|
||||
export type { IDiffFileInfo, IProcessedDiff, IDiffProcessorOptions } from './diff-processor.js';
|
||||
@@ -6,10 +6,12 @@ export { path };
|
||||
// pushrocks scope
|
||||
import * as npmextra from '@push.rocks/npmextra';
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as smartagent from '@push.rocks/smartagent';
|
||||
import * as smartai from '@push.rocks/smartai';
|
||||
import * as smartcli from '@push.rocks/smartcli';
|
||||
import * as smartdelay from '@push.rocks/smartdelay';
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
import * as smartfs from '@push.rocks/smartfs';
|
||||
import * as smartgit from '@push.rocks/smartgit';
|
||||
import * as smartinteract from '@push.rocks/smartinteract';
|
||||
import * as smartlog from '@push.rocks/smartlog';
|
||||
@@ -21,10 +23,12 @@ import * as smarttime from '@push.rocks/smarttime';
|
||||
export {
|
||||
npmextra,
|
||||
qenv,
|
||||
smartagent,
|
||||
smartai,
|
||||
smartcli,
|
||||
smartdelay,
|
||||
smartfile,
|
||||
smartfs,
|
||||
smartgit,
|
||||
smartinteract,
|
||||
smartlog,
|
||||
@@ -34,6 +38,13 @@ export {
|
||||
smarttime,
|
||||
};
|
||||
|
||||
// Create a shared SmartFs instance for filesystem operations
|
||||
const smartFsNodeProvider = new smartfs.SmartFsProviderNode();
|
||||
export const fsInstance = new smartfs.SmartFs(smartFsNodeProvider);
|
||||
|
||||
// Create a shared SmartFileFactory for in-memory file operations
|
||||
export const smartfileFactory = smartfile.SmartFileFactory.nodeFs();
|
||||
|
||||
// @git.zone scope
|
||||
import * as tspublish from '@git.zone/tspublish';
|
||||
|
||||
|
||||
Reference in New Issue
Block a user