git.zone/tsdoc
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(docs): Update project metadata and documentation to reflect comprehensive AI-enhanced features and improved installation and usage instructions

Browse Source
This commit is contained in:
Philipp Kunz 2025-05-14 11:27:38 +00:00
parent 620737566f
commit ab273ea75c
21 changed files with 2305 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -18,3 +18,4 @@ dist/
dist_*/ dist_*/
# custom # custom
**/.claude/settings.local.json

8
changelog.md
View File

@ -1,5 +1,13 @@
# Changelog # Changelog
## 2025-05-14 - 1.5.0 - feat(docs)
Update project metadata and documentation to reflect comprehensive AI-enhanced features and improved installation and usage instructions
- Revised descriptions in package.json and npmextra.json to emphasize comprehensive documentation capabilities
- Expanded README with detailed installation options and extended usage examples for both CLI and API-like integrations
- Added new dependency (gpt-tokenizer) to support token counting for AI context building
- Adjusted keywords to better reflect project functionalities such as commit message automation and context trimming
## 2025-05-13 - 1.4.5 - fix(dependencies) ## 2025-05-13 - 1.4.5 - fix(dependencies)
Upgrade various dependency versions and update package manager configuration Upgrade various dependency versions and update package manager configuration

19
npmextra.json
View File

@ -9,18 +9,19 @@
"npmPackagename": "@git.zone/tsdoc", "npmPackagename": "@git.zone/tsdoc",
"license": "MIT", "license": "MIT",
"projectDomain": "git.zone", "projectDomain": "git.zone",
"description": "An advanced TypeScript documentation tool using AI to generate and enhance documentation for TypeScript projects.", "description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
"keywords": [ "keywords": [
"TypeScript", "TypeScript",
"documentation generation", "documentation",
"AI-enhanced documentation", "AI",
"CLI tool", "CLI",
"README",
"TypeDoc",
"commit messages",
"automation",
"code analysis", "code analysis",
"automated documentation", "context trimming",
"developer tools", "developer tools"
"API documentation",
"technical writing",
"code quality improvement"
] ]
} }
}, },

20
package.json
View File

@ -2,7 +2,7 @@
"name": "@git.zone/tsdoc", "name": "@git.zone/tsdoc",
"version": "1.4.5", "version": "1.4.5",
"private": false, "private": false,
"description": "An advanced TypeScript documentation tool using AI to generate and enhance documentation for TypeScript projects.", "description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
"type": "module", "type": "module",
"exports": { "exports": {
".": "./dist_ts/index.js" ".": "./dist_ts/index.js"
@ -41,6 +41,7 @@
"@push.rocks/smartpath": "^5.0.18", "@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartshell": "^3.0.5", "@push.rocks/smartshell": "^3.0.5",
"@push.rocks/smarttime": "^4.0.6", "@push.rocks/smarttime": "^4.0.6",
"gpt-tokenizer": "^2.9.0",
"typedoc": "^0.28.4", "typedoc": "^0.28.4",
"typescript": "^5.8.3" "typescript": "^5.8.3"
}, },
@ -61,15 +62,16 @@
], ],
"keywords": [ "keywords": [
"TypeScript", "TypeScript",
"documentation generation", "documentation",
"AI-enhanced documentation", "AI",
"CLI tool", "CLI",
"README",
"TypeDoc",
"commit messages",
"automation",
"code analysis", "code analysis",
"automated documentation", "context trimming",
"developer tools", "developer tools"
"API documentation",
"technical writing",
"code quality improvement"
], ],
"repository": { "repository": {
"type": "git", "type": "git",

8
pnpm-lock.yaml generated
View File

@ -53,6 +53,9 @@ importers:
'@push.rocks/smarttime': '@push.rocks/smarttime':
specifier: ^4.0.6 specifier: ^4.0.6
version: 4.1.1 version: 4.1.1
gpt-tokenizer:
specifier: ^2.9.0
version: 2.9.0
typedoc: typedoc:
specifier: ^0.28.4 specifier: ^0.28.4
version: 0.28.4(typescript@5.8.3) version: 0.28.4(typescript@5.8.3)
@ -2876,6 +2879,9 @@ packages:
resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==}
engines: {node: '>=14.16'} engines: {node: '>=14.16'}
gpt-tokenizer@2.9.0:
resolution: {integrity: sha512-YSpexBL/k4bfliAzMrRqn3M6+it02LutVyhVpDeMKrC/O9+pCe/5s8U2hYKa2vFLD5/vHhsKc8sOn/qGqII8Kg==}
graceful-fs@4.2.10: graceful-fs@4.2.10:
resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==}
@ -9282,6 +9288,8 @@ snapshots:
p-cancelable: 3.0.0 p-cancelable: 3.0.0
responselike: 3.0.0 responselike: 3.0.0
gpt-tokenizer@2.9.0: {}
graceful-fs@4.2.10: {} graceful-fs@4.2.10: {}
graceful-fs@4.2.11: {} graceful-fs@4.2.11: {}

846
readme.md
View File

@ -1,312 +1,726 @@
# @git.zone/tsdoc # @git.zone/tsdoc
An advanced TypeScript documentation tool using AI to generate and enhance documentation for TypeScript projects. An advanced TypeScript documentation tool using AI to generate and enhance documentation for TypeScript projects.
## Install ## Install
To install `@git.zone/tsdoc`, you can either install it globally or use it with `npx`. To install @git.zone/tsdoc, you have two options. You can install it globally so that the CLI commands are available throughout your system, or you can use it with npx if you prefer to keep the installation local to your project.
### Global Installation ### Global Installation
You can install `@git.zone/tsdoc` globally on your system using npm: To install the tool globally, run the following command in your terminal:
```bash ```bash
npm install -g @git.zone/tsdoc npm install -g @git.zone/tsdoc
``` ```
Installing globally ensures that the CLI commands (such as tsdoc, tsdoc typedoc, tsdoc aidoc, etc.) are available anywhere on your machine without the need to refer to the local node_modules folder.
### Usage with npx ### Usage with npx
If you prefer not to install it globally, you can use it with `npx`: If you prefer not to install the tool globally, you can invoke it using npx directly from your project. This method works well if you intend to use the tool on a per-project basis:
```bash ```bash
npx @git.zone/tsdoc <command> npx @git.zone/tsdoc <command>
``` ```
In the commands below, you will see how to use the various functionalities that @git.zone/tsdoc provides for generating intricate and enhanced documentation for your TypeScript projects.
## Usage ## Usage
`@git.zone/tsdoc` provides a comprehensive CLI tool and advanced AI-enhanced features to generate and enhance documentation for your TypeScript projects. The @git.zone/tsdoc module provides a very rich and interactive CLI interface together with a set of programmable classes that let you integrate documentation generation into your build processes or workflows. This section will walk you through every aspect of the module—from its basic CLI commands to its advanced internal API usage. All examples provided below use ESM syntax with TypeScript and are designed to be comprehensive. Every code snippet is written so you can easily copy, paste, and adapt to your project. The following guide is divided into several sections covering every major feature, tool integration, and customization options available in the module.
### Using the CLI -------------------------------------------------------------------
### Overview and Core Concepts
The primary interface for `@git.zone/tsdoc` is through its command-line tool. Below, we'll explore the different commands available and provide examples of how to use them. At its heart, @git.zone/tsdoc is a CLI tool that blends classic documentation generation (using libraries such as TypeDoc) with AI-enhanced techniques. The tool reads your project files, uses a context builder to optimize file content based on token usage and configurable trimming strategies, and then leverages an AI engine to generate enhanced documentation. This complete solution is designed to integrate smoothly into your project pipeline.
### Commands Key features include:
- **Auto-detection of documentation format**: The CLI attempts to determine the best documentation strategy for your project.
- **Support for TypeDoc generation**: Build TypeDoc-compatible documentation directly.
- **AI-Enhanced Documentation (AiDoc)**: Generate a README and project description using artificial intelligence that analyzes your project’s code and context.
- **Plugin Integration**: The module leverages a variety of plugins (smartfile, smartgit, smartcli, smartai, etc.) to streamline tasks such as file manipulation, CLI interaction, shell command execution, and logging.
- **Context Trimming and Optimization**: To manage token usage (especially for AI input), the module includes advanced context-building strategies that trim and summarize code files intelligently.
- **Robust Internal API**: While the primary user interface is through the CLI, the underlying classes (AiDoc, TypeDoc, Readme, etc.) can be used to build custom integrations or extend the tool’s functionality.
#### `tsdoc` - Auto-detect Documentation Format Below, you will find detailed explanations along with ESM/TypeScript code examples for all core use cases.
The standard command will automatically detect the documentation format of your project and generate the appropriate documentation. -------------------------------------------------------------------
### Command-Line Interface (CLI) Usage
The most common way to interact with @git.zone/tsdoc is via its command-line interface (CLI). The CLI is designed to auto-detect your project’s context and trigger the appropriate commands based on your needs. Below is a guide on how to use the CLI commands.
#### Basic Invocation
When you run the command without any arguments, the tool attempts to determine the appropriate documentation generation mode:
```bash ```bash
tsdoc tsdoc
``` ```
### Example This will scan the project directory and attempt to detect whether your project follows a TypeDoc convention or if it would benefit from an AI-enhanced documentation build. The auto-detection logic uses the project context (for example, the presence of a ts directory or specific configuration files) to decide the best course of action.
```typescript ##### Example Scenario
// In a TypeScript project, run the above command.
Imagine you have a TypeScript project with the following structure:
├── package.json
├── ts/
│   └── index.ts
└── readme.hints.md
When you execute:
```bash
tsdoc
``` ```
#### `tsdoc typedoc` - Generate Documentation using TypeDoc The tool will analyze the project directory, recognizing the ts/ folder, and it will route the command to the appropriate documentation generator, such as the TypeDoc generator if it detects valid structure.
The `typedoc` command will generate documentation compliant with the TypeDoc format. #### TypeDoc Command
For projects that require a traditional documentation format, you can explicitly generate documentation using the TypeDoc integration:
```bash ```bash
tsdoc typedoc --publicSubdir docs tsdoc typedoc --publicSubdir docs
``` ```
### Example This command instructs the module to generate HTML documentation using TypeDoc, placing the output into a public directory (or a custom subdirectory as specified).
**Inside a TypeScript file, this command can be mirrored by calling the TypeDoc class directly:**
```typescript ```typescript
import * as plugins from '@push.rocks/smartfile'; import { TypeDoc } from '@git.zone/tsdoc';
import * as path from 'path';
const tsconfigPath = plugins.path.join(__dirname, 'tsconfig.json'); const cwd = process.cwd();
const outputPath = plugins.path.join(__dirname, 'docs'); const typeDocInstance = new TypeDoc(cwd);
await plugins.smartshellInstance.exec( const compileDocumentation = async (): Promise<void> => {
`typedoc --tsconfig ${tsconfigPath} --out ${outputPath} index.ts`, try {
); // Specify the output subdirectory for documentation
await typeDocInstance.compile({
publicSubdir: 'docs'
});
console.log('Documentation successfully generated using TypeDoc.');
} catch (error) {
console.error('Error generating documentation with TypeDoc:', error);
}
};
compileDocumentation();
``` ```
#### `tsdoc aidoc` - Generate AI-Enhanced Documentation In this example, the script creates an instance of the TypeDoc class passing the current working directory. The compile method is then called with an options object, indicating that the public subdirectory should be named “docs.” The method spawns a shell command using the smart shell plugin to execute the TypeDoc binary.
The `aidoc` command will use AI to generate an enhanced README and update your project description. #### AI-Enhanced Documentation Command
One of the standout features of this module is its AI-enhanced documentation capabilities. The `aidoc` command integrates with an OpenAI interface to produce a more contextual and detailed README and project description. This is particularly useful when your project codebase has evolved and requires documentation updates based on the current source code.
To run the AI-enhanced documentation generation:
```bash ```bash
tsdoc aidoc tsdoc aidoc
``` ```
### Example In an ESM/TypeScript project, you can use the AiDoc class to programmatically run the same functionality:
```typescript ```typescript
import { AiDoc } from '@git.zone/tsdoc'; import { AiDoc } from '@git.zone/tsdoc';
(async () => { const buildEnhancedDocs = async (): Promise<void> => {
const aidoc = new AiDoc(); const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
await aidoc.start(); try {
await aidoc.buildReadme('./'); // Start the AI interface; this internally checks if the token is valid and persists it
await aidoc.buildDescription('./'); await aiDoc.start();
})();
// Build the README file for the project directory
console.log('Generating README file using AI...');
await aiDoc.buildReadme(process.cwd());
// Build a new project description based on the codebase
console.log('Generating updated project description...');
await aiDoc.buildDescription(process.cwd());
console.log('AI-enhanced documentation generated successfully.');
} catch (error) {
console.error('Failed to generate AI-enhanced documentation:', error);
}
};
buildEnhancedDocs();
``` ```
#### `tsdoc test` - Test Your Documentation Setup In the above snippet, we import the AiDoc class and create an instance with an OpenAI token. The methods start(), buildReadme(), and buildDescription() streamline the process of generating enhanced documentation by leveraging the underlying AI engine. This code example should serve as a blueprint for those wishing to integrate AI-driven documentation updates as part of their CI/CD pipelines.
The `test` command will test your current documentation setup, ensuring everything is configured correctly. #### Testing Your Documentation Setup
Before you commit changes to your project documentation, it is often worthwhile to run tests to ensure that your documentation generation process is behaving as expected. The module includes a `test` command:
```bash ```bash
tsdoc test tsdoc test
``` ```
### Example This command verifies that all components (CLI commands, TypeDoc compilation, AI integration, etc.) are properly configured.
```typescript Here is an example test script written in TypeScript using a test bundle:
import * as plugins from '@git.zone/tsdoc';
await plugins.runCli().catch((err) => {
console.error(err);
process.exit(1);
});
```
## Features in Depth
### Using Plugins
`@git.zone/tsdoc` extensively uses plugins to extend its capabilities.
### Available Plugins
- **npmextra**: Manage npm project-related configurations.
- **qenv**: Environment variable management.
- **smartai**: AI integration.
- **smartcli**: CLI helper.
- **smartdelay**: Simple delay utility.
- **smartfile**: File system utilities.
- **smartinteract**: Interaction helper.
- **smartlog**: Logging utility.
- **smartlogDestinationLocal**: Local file destination for logging.
- **smartpath**: Path utilities.
- **smartshell**: Shell command execution.
- **typedoc**: Documentation generation.
### Example Usage of Plugins
#### Path Management
```typescript
import * as path from 'path';
const packageDir = path.join(__dirname, '../');
const cwd = process.cwd();
const binDir = path.join(packageDir, './node_modules/.bin');
const assetsDir = path.join(packageDir, './assets');
const publicDir = path.join(cwd, './public');
const tsDir = path.join(cwd, './ts');
const tsconfigFile = path.join(assetsDir, './tsconfig.json');
const typedocOptionsFile = path.join(assetsDir, './typedoc.json');
```
#### Logging
```typescript
import * as plugins from './plugins';
const logger = new plugins.smartlog.Smartlog({
logContext: {
company: 'Some Company',
companyunit: 'Some CompanyUnit',
containerName: 'Some Containername',
environment: 'local',
runtime: 'node',
zone: 'gitzone',
},
minimumLogLevel: 'silly',
});
logger.addLogDestination(new plugins.smartlogDestinationLocal.DestinationLocal());
```
## Advanced Usage
### Using `TypeDoc` Class
The `TypeDoc` class provides utility methods to compile TypeScript documentation.
```typescript
import { TypeDoc } from '@git.zone/tsdoc/classes.typedoc';
const typeDocInstance = new TypeDoc(paths.cwd);
await typeDocInstance.compile({
publicSubdir: 'docs',
});
```
### Using `AiDoc` Class
The `AiDoc` class integrates with AI services to enhance your documentation.
#### Initializing and Using AI
```typescript
import { AiDoc } from '@git.zone/tsdoc';
const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
await aiDoc.start();
await aiDoc.buildReadme('./');
await aiDoc.buildDescription('./');
```
#### Retrieving AI Tokens
```typescript
import * as plugins from '@git.zone/tsdoc/plugins';
const qenv = new plugins.qenv.Qenv();
const openaiToken = await qenv.getEnvVarOnDemand('OPENAI_TOKEN');
```
### Testing
The provided tests demonstrate how to verify the functionality of the `@git.zone/tsdoc` tool.
#### Example Test Script
```typescript ```typescript
import { expect, tap } from '@push.rocks/tapbundle'; import { expect, tap } from '@push.rocks/tapbundle';
import * as tsdoc from '../ts/index'; import { AiDoc } from '@git.zone/tsdoc';
tap.test('should create AiDoc instance', async () => { tap.test('AiDoc instance creation', async () => {
const aidoc = new tsdoc.AiDoc({ OPENAI_TOKEN: 'dummy-token' }); const aidoc = new AiDoc({ OPENAI_TOKEN: 'dummy-token' });
expect(aidoc).toBeInstanceOf(tsdoc.AiDoc); expect(aidoc).toBeInstanceOf(AiDoc);
}); });
tap.test('should start AiDoc', async () => { tap.test('Running AI documentation generation', async () => {
const aidoc = new tsdoc.AiDoc({ OPENAI_TOKEN: 'dummy-token' }); const aidoc = new AiDoc({ OPENAI_TOKEN: 'dummy-token' });
await aidoc.start(); await aidoc.start();
await aidoc.buildReadme('./');
await aidoc.buildDescription('./'); // Attempt buildReadme and buildDescription synchronously for test coverage
await aidoc.buildReadme(process.cwd());
await aidoc.buildDescription(process.cwd());
// If no errors are thrown, we assume the process works as expected
expect(true).toBe(true);
}); });
tap.start(); tap.start();
``` ```
### Internals This test script demonstrates how to automate the validation process by using the provided AiDoc class. Using a testing framework like tap ensures that your documentation generation remains robust even as new features are added or as the project evolves.
The `@git.zone/tsdoc` consists of several internal classes and utilities that streamline its functionality. -------------------------------------------------------------------
### Advanced Usage Scenarios
#### File Paths Management Beyond using the CLI, @git.zone/tsdoc provides various classes and plugins that allow you to deeply integrate documentation generation within your project. The following sections document advanced usage scenarios where you programmatically interact with different components.
Located in `ts/paths.ts`, the file defines various directories and file paths used by the tool. #### 1. Deep Dive into AiDoc Functionality
The AiDoc class is the core of the AI-enhanced documentation generation. It manages interactions with the OpenAI API, handles token validations, and integrates with project-specific configurations.
Consider the following advanced usage example:
```typescript ```typescript
import * as plugins from './plugins'; import { AiDoc } from '@git.zone/tsdoc';
import * as path from 'path';
export const packageDir = plugins.path.join( const generateProjectDocs = async () => {
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), // Create an instance of the AiDoc class with a configuration object
'../', // that includes your OpenAI token. This token will be used to query the AI.
); const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
export const cwd = process.cwd();
export const binDir = plugins.path.join(packageDir, './node_modules/.bin');
export const assetsDir = plugins.path.join(packageDir, './assets');
export const publicDir = plugins.path.join(cwd, './public');
export const tsDir = plugins.path.join(cwd, './ts');
export const tsconfigFile = plugins.path.join(assetsDir, './tsconfig.json');
export const typedocOptionsFile = plugins.path.join(assetsDir, './typedoc.json');
```
#### Utility Commands // Initialize the AI documentation system.
await aiDoc.start();
Define utility commands that streamline various processes. // Build the README file for the current project.
console.log('Building README for the project...');
await aiDoc.buildReadme(process.cwd());
##### Example: SmartCLI Usage // Build an updated project description based on the analysis of the source files.
console.log('Building project description...');
await aiDoc.buildDescription(process.cwd());
```typescript // You can also generate a commit message based on code changes by using the next commit object generation.
// Import required modules and plugins try {
import * as plugins from './plugins'; console.log('Generating commit message based on your project changes...');
import * as paths from './paths'; const nextCommit = await aiDoc.buildNextCommitObject(process.cwd());
console.log('Next commit message object:', nextCommit);
// TypeDoc and AiDoc classes } catch (error) {
import { TypeDoc } from './classes.typedoc'; console.error('Error generating commit message:', error);
import { AiDoc } from './classes.aidoc'; }
// Export a run function
export const run = async () => {
const tsdocCli = new plugins.smartcli.Smartcli();
tsdocCli.standardCommand().subscribe(async (argvArg) => {
switch (true) {
case await TypeDoc.isTypeDocDir(paths.cwd):
tsdocCli.triggerCommand('typedoc', argvArg);
break;
default:
console.error(`Cannot determine docs format at ${paths.cwd}`);
}
});
tsdocCli.addCommand('typedoc').subscribe(async (argvArg) => {
const typeDocInstance = new TypeDoc(paths.cwd);
await typeDocInstance.compile({
publicSubdir: argvArg.publicSubdir,
});
});
tsdocCli.addCommand('aidoc').subscribe(async (argvArg) => {
const aidocInstance = new AiDoc(argvArg);
await aidocInstance.start();
await aidocInstance.buildReadme(paths.cwd);
await aidocInstance.buildDescription(paths.cwd);
});
tsdocCli.startParse();
}; };
generateProjectDocs();
``` ```
By leveraging these functionalities, you can configure and extend `@git.zone/tsdoc` to fit your specific documentation generation needs. In this example, the AiDoc class handles multiple tasks:
### Further Enhancements - It starts by validating and printing the sanitized token.
- It generates and writes the README file based on dynamic analysis.
- It updates the project description stored in your configuration files.
- It even integrates with Git to produce a suggested commit message that factors in the current state of the project directory.
The `@git.zone/tsdoc` tool is designed to be extensible. Explore the source files and plugins to add more functionality or integrate with other tools as needed. This README provides an extensive overview of the commands and features but it's always beneficial to dive into the source code to understand the intricacies and potential customizations. Happy documenting! Internally, methods such as buildReadme() interact with the ProjectContext class to gather files and determine the relevant context. This context is trimmed and processed based on token budgets, thus ensuring that the AI interface only receives the information it can effectively process.
#### 2. Interacting with the TypeDoc Class Programmatically
The TypeDoc class does not merely wrap the standard TypeDoc tool. It adds a layer of automation by preparing the TypeScript environment, generating a temporary tsconfig file, and invoking TypeDoc with the proper configuration. You can use this functionality to conditionally generate documentation or integrate it into your build steps.
Below is another example demonstrating the integration:
```typescript
import { TypeDoc } from '@git.zone/tsdoc';
import * as path from 'path';
const generateTypeDocDocs = async () => {
// Assume you are in the root directory of your TypeScript project
const cwd = process.cwd();
// Create an instance of the TypeDoc class
const typeDocInstance = new TypeDoc(cwd);
// Prepare additional options if necessary (e.g., setting a public subdirectory for docs)
const options = { publicSubdir: 'documentation' };
// Compile your TypeScript project documentation.
// The compile method handles creating the tsconfig file, running the shell command, and cleaning up afterward.
try {
console.log('Compiling TypeScript documentation using TypeDoc...');
await typeDocInstance.compile(options);
console.log('Documentation generated at:', path.join(cwd, 'public', 'documentation'));
} catch (error) {
console.error('Error compiling TypeDoc documentation:', error);
}
};
generateTypeDocDocs();
```
This script clearly demonstrates how TypeDoc automation is structured inside the module. By invoking the compile() method, the class takes care of setting directory paths, preparing command arguments, and executing the underlying TypeDoc binary using the smart shell plugin.
#### 3. Customizing Context Building
One of the critical functionalities within @git.zone/tsdoc is its ability to build a smart context for documentation generation. The module not only collects file content from your project (like package.json, readme.hints.md, and other source files) but also intelligently trims and summarizes these contents to fit within token limits for AI processing.
Consider the following deep-dive example into the context building process:
```typescript
import { EnhancedContext } from '@git.zone/tsdoc';
import { ConfigManager } from '@git.zone/tsdoc/dist/ts/context/config-manager.js';
const buildProjectContext = async () => {
const projectDir = process.cwd();
// Create an instance of the EnhancedContext class to optimize file content for AI use.
const enhancedContext = new EnhancedContext(projectDir);
// Initialize the context builder. This ensures that configuration (e.g., token budgets, trimming options) is loaded.
await enhancedContext.initialize();
// Optionally, you can choose to set a custom token budget and context mode.
enhancedContext.setTokenBudget(100000); // for example, limit tokens to 100K
enhancedContext.setContextMode('trimmed');
// Build the context string from selected files in your project.
const contextResult = await enhancedContext.buildContext('readme');
console.log('Context generated with token count:', contextResult.tokenCount);
console.log('Token savings due to trimming:', contextResult.tokenSavings);
// The context string includes file boundaries and token information.
console.log('Generated Context:', contextResult.context);
};
buildProjectContext();
```
In this example:
- The EnhancedContext class is initialized with the project directory.
- Configuration is loaded via the ConfigManager, which reads parameters from npmextra.json.
- The context builder then gathers files (such as package.json, readme hints, TypeScript sources, etc.), trims unnecessary content, and builds a context string.
- Finally, it prints the overall token counts and savings, giving you valuable feedback on how the context was optimized for the AI input.
This detailed context-building mechanism is essential for managing large TypeScript projects. It ensures that the AI engine can process relevant code data without being overwhelmed by too many tokens.
#### 4. Working with the Readme Class for Automatic README Generation
The Readme class in @git.zone/tsdoc takes the AI-enhanced documentation further by not only generating a project-level README but also iterating over submodules within your project. This ensures that every published module has its own complete, AI-generated README.
Here is an advanced example demonstrating how to trigger README generation for the main project and its submodules:
```typescript
import { AiDoc } from '@git.zone/tsdoc';
import * as path from 'path';
import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
import { readFileSync } from 'fs';
const buildProjectReadme = async () => {
const projectDir = process.cwd();
// Create an instance of the AiDoc class to handle AI-enhanced docs generation
const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
// Start the AI interface
await aiDoc.start();
// Build the primary README for the project directory
console.log('Generating primary README...');
await aiDoc.buildReadme(projectDir);
// Logging function to verify submodule processing
logger.log('info', `Primary README generated in ${projectDir}`);
// Assume that submodules are organized in distinct directories.
// Here we simulate the process of scanning subdirectories and triggering README generation for each.
const subModules = ['submodule1', 'submodule2'];
// Loop through each submodule directory to generate its README.
for (const subModule of subModules) {
const subModuleDir = path.join(projectDir, subModule);
logger.log('info', `Generating README for submodule: ${subModule}`);
// Each submodule README is generated independently.
await aiDoc.buildReadme(subModuleDir);
// Optionally, read the generated README content for verification.
const readmePath = path.join(subModuleDir, 'readme.md');
try {
const readmeContent = readFileSync(readmePath, 'utf8');
logger.log('info', `Generated README for ${subModule}:\n${readmeContent.substring(0, 200)}...`);
} catch (error) {
logger.log('error', `Failed to read README for ${subModule}: ${error}`);
}
}
};
buildProjectReadme();
```
In this example, the script:
- Starts by building the AI-enhanced README for the entire project.
- Then iterates over a list of submodule directories and generates READMEs for each.
- Uses the logging utility to provide immediate feedback on the generation process.
- Optionally, reads back a snippet of the generated file to verify successful documentation generation.
This approach ensures that projects with multiple submodules or packages maintain a consistent and high-quality documentation standard across every component.
-------------------------------------------------------------------
### Plugin-Based Architecture and Integrations
Under the hood, @git.zone/tsdoc leverages a number of smaller, focused plugins that extend its functionality. These plugins facilitate file system operations, shell command execution, environment variable management, and logging. The modular design makes it easy to extend or customize the tool according to your needs.
The relevant plugins include:
- smartai: Provides the API integration with OpenAI.
- smartcli: Handles CLI input parsing and command setup.
- smartdelay: Manages asynchronous delays and debouncing.
- smartfile: Offers an abstraction over file I/O operations.
- smartgit: Facilitates integration with git repositories (e.g., retrieving diffs, commit status).
- smartinteract: Eases interaction with the user (prompting for tokens, confirming actions).
- smartlog and smartlogDestinationLocal: Provide comprehensive logging mechanisms.
- smartpath, smartshell, and smarttime: Manage file paths, execute shell commands, and process time data respectively.
Below is a sample snippet illustrating how you might directly interact with a few of these plugins to, for example, run a custom shell command or log events:
```typescript
import * as plugins from '@git.zone/tsdoc/dist/ts/plugins.js';
import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
const runCustomCommand = async () => {
// Create an instance of the smart shell utility
const smartshellInstance = new plugins.smartshell.Smartshell({
executor: 'bash',
pathDirectories: [plugins.smartpath.join(process.cwd(), 'node_modules/.bin')]
});
try {
// Execute a sample shell command, e.g., listing files in the current directory
const output = await smartshellInstance.exec('ls -la');
logger.log('info', `Shell command output:\n${output}`);
} catch (error) {
logger.log('error', 'Error executing custom shell command:', error);
}
};
runCustomCommand();
```
This example shows how to:
- Import the necessary plugins.
- Set up a smartshell instance by specifying the shell executor and the directories where executables are located.
- Execute a command (in this case, listing directory contents) and log the results using the provided logging plugin.
Such examples demonstrate the flexibility provided by the module’s internal API. They also illustrate that even if you choose not to use the CLI, you can still leverage the @git.zone/tsdoc functionality programmatically in a highly integrated fashion.
-------------------------------------------------------------------
### Handling Git Commit Messages with AI
A unique feature of this tool is its capacity to assist with creating smart commit messages based on code changes. The Commit class (found within the aidocs_classes directory) ties together output from smartgit and AiDoc to suggest commit messages that are both descriptive and formatted according to conventional commit guidelines.
Consider this example where you generate a commit message based on the diff from your git repository:
```typescript
import { AiDoc } from '@git.zone/tsdoc';
const generateCommitMessage = async () => {
// Create an instance of AiDoc
const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
// Initialize the AI service
await aiDoc.start();
try {
// Generate and retrieve the next commit object based on the uncommitted changes in the repository.
const commitObject = await aiDoc.buildNextCommitObject(process.cwd());
console.log('Recommended commit object:', commitObject);
// The commit object is structured with the following fields:
// - recommendedNextVersionLevel: Indicates whether the commit is a fix, feature, or breaking change.
// - recommendedNextVersionScope: The scope of changes.
// - recommendedNextVersionMessage: A short commit message.
// - recommendedNextVersionDetails: A list of details explaining the changes.
// - recommendedNextVersion: A computed version string.
} catch (error) {
console.error('Error generating commit message:', error);
}
};
generateCommitMessage();
```
The process works as follows:
1. The AiDoc instance is created and started.
2. The tool uses the smartgit plugin to fetch uncommitted changes from the repository.
3. It then builds a context string incorporating file diffs and project metadata.
4. Finally, the OpenAI API is queried to produce a commit message formatted as JSON. This JSON object is parsed and can be used directly in your git workflow.
This advanced integration assists teams in maintaining consistent commit message standards while reducing the manual burden of summarizing code changes.
-------------------------------------------------------------------
### Detailed Explanation of Internal Mechanics
If you are curious about the intricate inner workings of @git.zone/tsdoc and wish to extend or debug its behavior, here is an in-depth explanation of some internal mechanisms.
#### Context Trimming Strategy
Managing token count is critical when interfacing with APIs that have strict limits. The module uses a multi-step process:
- It gathers various files (such as package.json, ts files, readme hints).
- It sorts the files and calculates the token count using the GPT tokenizer.
- It applies trimming strategies such as removing function implementations or comments in TypeScript files, based on a configurable set of parameters.
- Finally, it constructs a unified context string that includes file boundaries for clarity.
For example, the ContextTrimmer class carries out these transformations:
```typescript
import { ContextTrimmer } from '@git.zone/tsdoc/dist/ts/context/context-trimmer.js';
const trimFileContent = (filePath: string, content: string): string => {
// Create an instance of ContextTrimmer with default configuration
const trimmer = new ContextTrimmer({
removeImplementations: true,
preserveJSDoc: true,
maxFunctionLines: 5,
removeComments: true,
removeBlankLines: true
});
// Trim the content based on the file type and configured options
const trimmedContent = trimmer.trimFile(filePath, content, 'trimmed');
return trimmedContent;
};
// Example usage with a TypeScript file
const tsFileContent = `
/**
* This function calculates the sum of two numbers.
*/
export const add = (a: number, b: number): number => {
// Calculation logic
return a + b;
};
`;
const trimmedTSContent = trimFileContent('src/math.ts', tsFileContent);
console.log('Trimmed TypeScript File Content:\n', trimmedTSContent);
```
This process helps in reducing the number of tokens before sending the data to the AI API while preserving the essential context needed for documentation generation.
#### Dynamic Configuration Management
The module’s configuration is stored in the npmextra.json file and includes settings for context building, trimming strategies, and task-specific options. The ConfigManager class reads these settings and merges them with default values. This dynamic configuration system ensures that the behavior of the documentation tool can be easily adjusted without altering the source code.
```typescript
import { ConfigManager } from '@git.zone/tsdoc/dist/ts/context/config-manager.js';
const updateDocumentationConfig = async () => {
const projectDir = process.cwd();
const configManager = ConfigManager.getInstance();
// Initialize the configuration manager with the project directory
await configManager.initialize(projectDir);
// Retrieve the current configuration
let currentConfig = configManager.getConfig();
console.log('Current context configuration:', currentConfig);
// If you want to change some parameters (e.g., maxTokens), update and then save the new configuration
const newConfig = { maxTokens: 150000 };
await configManager.updateConfig(newConfig);
console.log('Configuration updated successfully.');
};
updateDocumentationConfig();
```
In this snippet, the ConfigManager:
- Loads current configuration from npmextra.json.
- Allows updates to specific keys (such as token limits).
- Persists these changes back to the file system using the smartfile plugin.
#### Logging and Diagnostic Output
Throughout its execution, @git.zone/tsdoc logs important information such as token counts, file statistics, and shell command outputs. This logging is accomplished through a combination of the smartlog and smartlogDestinationLocal plugins. The following example illustrates how logging can help diagnose execution issues:
```typescript
import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
const logDiagnosticInfo = () => {
logger.log('info', 'Starting documentation generation process...');
// Log additional contextual information
logger.log('debug', 'Project directory:', process.cwd());
logger.log('debug', 'Token budget set for context building:', 150000);
// Simulate a long-running process
setTimeout(() => {
logger.log('info', 'Documentation generation process completed successfully.');
}, 2000);
};
logDiagnosticInfo();
```
Using comprehensive logging, the tool provides feedback not only during normal execution but also in error scenarios, allowing developers to troubleshoot and optimize their documentation generation workflow.
-------------------------------------------------------------------
### Integrating @git.zone/tsdoc into a Continuous Integration Pipeline
For teams looking to integrate documentation generation into their CI processes, @git.zone/tsdoc can be harnessed by scripting the CLI commands or by embedding the class-based API directly into your build scripts. Here’s an example of a CI script written in TypeScript that runs as part of a GitHub Action or similar workflow:
```typescript
import { runCli } from '@git.zone/tsdoc';
import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
const runDocumentationPipeline = async () => {
try {
logger.log('info', 'Starting the documentation pipeline...');
// Run the CLI which automatically detects the project context and generates docs.
await runCli();
logger.log('info', 'Documentation pipeline completed successfully.');
} catch (error) {
logger.log('error', 'Documentation pipeline encountered an error:', error);
process.exit(1);
}
};
runDocumentationPipeline();
```
In a CI environment, you can invoke this script to ensure that documentation is generated or updated as part of your deployment process. The process includes building the README, updating project descriptions, and generating TypeDoc documentation if the project structure warrants it.
-------------------------------------------------------------------
### Comprehensive Workflow Example
Below is a full-fledged example that combines many of the above functionalities into a single workflow. This script is intended to be run as part of your build process or as a standalone command, and it demonstrates how to initialize all parts of the module, generate documentation for the main project and its submodules, update configuration, and log key diagnostics.
```typescript
import { AiDoc } from '@git.zone/tsdoc';
import { TypeDoc } from '@git.zone/tsdoc';
import { ConfigManager } from '@git.zone/tsdoc/dist/ts/context/config-manager.js';
import { EnhancedContext } from '@git.zone/tsdoc/dist/ts/context/enhanced-context.js';
import * as path from 'path';
import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
const runFullDocumentationWorkflow = async () => {
const projectDir = process.cwd();
// Initialize configuration management
const configManager = ConfigManager.getInstance();
await configManager.initialize(projectDir);
logger.log('info', `Loaded configuration for project at ${projectDir}`);
// Step 1: Generate conventional TypeDoc documentation
const typeDocInstance = new TypeDoc(projectDir);
try {
logger.log('info', 'Starting TypeDoc documentation generation...');
await typeDocInstance.compile({ publicSubdir: 'docs' });
logger.log('info', `TypeDoc documentation generated in ${path.join(projectDir, 'public', 'docs')}`);
} catch (error) {
logger.log('error', 'Error during TypeDoc generation:', error);
}
// Step 2: Run AI-enhanced documentation generation
const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
await aiDoc.start();
// Generate main README and updated project description
try {
logger.log('info', 'Generating main README via AI-enhanced documentation...');
await aiDoc.buildReadme(projectDir);
logger.log('info', 'Main README generated successfully.');
logger.log('info', 'Generating updated project description...');
await aiDoc.buildDescription(projectDir);
logger.log('info', 'Project description updated successfully.');
} catch (error) {
logger.log('error', 'Error generating AI-enhanced documentation:', error);
}
// Step 3: Generate contextual data using EnhancedContext
const enhancedContext = new EnhancedContext(projectDir);
await enhancedContext.initialize();
enhancedContext.setContextMode('trimmed');
enhancedContext.setTokenBudget(150000);
const contextResult = await enhancedContext.buildContext('readme');
logger.log('info', `Context built successfully. Total tokens: ${contextResult.tokenCount}. Savings: ${contextResult.tokenSavings}`);
// Step 4: Process submodules (if any) and generate READMEs
const subModules = ['submodule1', 'submodule2'];
for (const subModule of subModules) {
const subModuleDir = path.join(projectDir, subModule);
logger.log('info', `Processing submodule: ${subModule}`);
try {
await aiDoc.buildReadme(subModuleDir);
logger.log('info', `Submodule README generated for ${subModule}`);
} catch (error) {
logger.log('error', `Failed to generate README for ${subModule}:`, error);
}
}
// Optional: Generate a commit message suggestion based on current changes
try {
logger.log('info', 'Generating commit message suggestion...');
const commitObject = await aiDoc.buildNextCommitObject(projectDir);
logger.log('info', 'Suggested commit message object:', commitObject);
} catch (error) {
logger.log('error', 'Error generating commit message suggestion:', error);
}
logger.log('info', 'Full documentation workflow completed successfully.');
};
runFullDocumentationWorkflow();
```
This comprehensive workflow showcases the integration of various facets of the @git.zone/tsdoc module:
- Loading and updating configuration via the ConfigManager.
- Generating static documentation using TypeDoc.
- Enhancing documentation with AI through the AiDoc class.
- Optimizing project context with the EnhancedContext class.
- Iterating over submodules to ensure all parts of your project are documented.
- Providing useful diagnostic logging for every step.
-------------------------------------------------------------------
### Wrapping Up the Usage Guide
The examples provided above demonstrate that @git.zone/tsdoc is not simply a CLI tool—it is a complete documentation framework designed to adapt to your workflow. Whether you are a developer looking to automate documentation updates in your CI pipeline or a team seeking an AI-powered enhancement for your project metadata, this module offers a wide range of interfaces and hooks for you to leverage.
Key takeaways:
- The CLI handles most routine tasks automatically while also exposing commands for specific documentation generation strategies.
- Programmatic usage allows deep integration with your project’s build and commit processes.
- The internal architecture—built on plugins, context optimization, and extensive logging—ensures that the tool can scale with project complexity.
- Advanced users can customize context trimming, file inclusion rules, and even modify AI queries to better suit their project’s needs.
Each code example provided here is written using modern ESM syntax and TypeScript to ensure compatibility with current development practices. Since the module is designed with extensibility in mind, developers are encouraged to explore the source code (especially the classes in the ts/ and ts/aidocs_classes directories) for further customization opportunities.
By integrating @git.zone/tsdoc into your workflow, you ensure that your project documentation remains accurate, comprehensive, and reflective of your latest code changes—whether you are generating a simple README or a complex API documentation set enhanced by AI insights.
Happy documenting!
## License and Legal Information ## License and Legal Information

314
readme.plan.md Normal file
View File

@ -0,0 +1,314 @@
# TSDocs Context Optimization Plan
## Problem Statement
For large TypeScript projects, the context generated for AI-based documentation creation becomes too large, potentially exceeding even o4-mini's 200K token limit. This affects the ability to effectively generate:
- Project documentation (README.md)
- API descriptions and keywords
- Commit messages and changelogs
Current implementation simply includes all TypeScript files and key project files, but lacks intelligent selection, prioritization, or content reduction mechanisms.
## Analysis of Approaches
### 1. Smart Content Selection
**Description:** Intelligently select only files that are necessary for the specific task being performed, using heuristic rules.
**Advantages:**
- Simple to implement
- Predictable behavior
- Can be fine-tuned for different operations
**Disadvantages:**
- Requires manual tuning of rules
- May miss important context in complex projects
- Static approach lacks adaptability
**Implementation Complexity:** Medium
### 2. File Prioritization
**Description:** Rank files by relevance using git history, file size, import/export analysis, and relationship to the current task.
**Advantages:**
- Adaptively includes the most relevant files first
- Maintains context for frequently changed or central files
- Can leverage git history for additional signals
**Disadvantages:**
- Complexity in determining accurate relevance scores
- Requires analyzing project structure
- May require scanning imports/exports for dependency analysis
**Implementation Complexity:** High
### 3. Chunking Strategy
**Description:** Process the project in logical segments, generating intermediate results that are then combined to create the final output.
**Advantages:**
- Can handle projects of any size
- Focused context for each specific part
- May improve quality by focusing on specific areas deeply
**Disadvantages:**
- Complex orchestration of multiple AI calls
- Challenge in maintaining consistency across chunks
- May increase time and cost for processing
**Implementation Complexity:** High
### 4. Dynamic Context Trimming
**Description:** Automatically reduce context by removing non-essential code while preserving structure. Techniques include:
- Removing implementation details but keeping interfaces and type definitions
- Truncating large functions while keeping signatures
- Removing comments and whitespace (except JSDoc)
- Keeping only imports/exports for context files
**Advantages:**
- Preserves full project structure
- Flexible token usage based on importance
- Good balance between completeness and token efficiency
**Disadvantages:**
- Potential to remove important implementation details
- Risk of missing context needed for specific tasks
- Complex rules for what to trim vs keep
**Implementation Complexity:** Medium
### 5. Embeddings-Based Retrieval
**Description:** Create vector embeddings of project files and retrieve only the most relevant ones for a specific task using semantic similarity.
**Advantages:**
- Highly adaptive to different types of requests
- Leverages semantic understanding of content
- Can scale to extremely large projects
**Disadvantages:**
- Requires setting up and managing embeddings database
- Added complexity of running vector similarity searches
- Higher resource requirements for maintaining embeddings
**Implementation Complexity:** Very High
### 6. Task-Specific Contexts
**Description:** Create separate optimized contexts for different tasks (readme, commit messages, etc.) with distinct file selection and processing strategies.
**Advantages:**
- Highly optimized for each specific task
- Efficient token usage for each operation
- Improved quality through task-focused contexts
**Disadvantages:**
- Maintenance of multiple context building strategies
- More complex configuration
- Potential duplication in implementation
**Implementation Complexity:** Medium
### 7. Recursive Summarization
**Description:** Summarize larger files first, then include these summaries in the final context along with smaller files included in full.
**Advantages:**
- Can handle arbitrary project sizes
- Preserves essential information from all files
- Balanced approach to token usage
**Disadvantages:**
- Quality loss from summarization
- Increased processing time from multiple AI calls
- Complex orchestration logic
**Implementation Complexity:** High
## Implementation Strategy
We propose a phased implementation approach, starting with the most impactful and straightforward approaches, then building toward more complex solutions as needed:
### Phase 1: Foundation (1-2 weeks)
1. **Implement Dynamic Context Trimming**
- Create a `ContextProcessor` class that takes SmartFile objects and applies trimming rules
- Implement configurable trimming rules (remove implementations, keep signatures)
- Add a configuration option to control trimming aggressiveness
- Support preserving JSDoc comments while removing other comments
2. **Enhance Token Monitoring**
- Track token usage per file to identify problematic files
- Implement token budgeting to stay within limits
- Add detailed token reporting for optimization
### Phase 2: Smart Selection (2-3 weeks)
3. **Implement Task-Specific Contexts**
- Create specialized context builders for readme, commit messages, and descriptions
- Customize file selection rules for each task
- Add configuration options for task-specific settings
4. **Add Smart Content Selection**
- Implement heuristic rules for file importance
- Create configuration for inclusion/exclusion patterns
- Add ability to focus on specific directories or modules
### Phase 3: Advanced Techniques (3-4 weeks)
5. **Implement File Prioritization**
- Add git history analysis to identify frequently changed files
- Implement dependency analysis to identify central files
- Create a scoring system for file relevance
6. **Add Optional Recursive Summarization**
- Implement file summarization for large files
- Create a hybrid approach that mixes full files and summaries
- Add configuration to control summarization thresholds
### Phase 4: Research-Based Approaches (Future Consideration)
7. **Research and Evaluate Embeddings-Based Retrieval**
- Prototype embeddings creation for TypeScript files
- Evaluate performance and accuracy
- Implement if benefits justify the complexity
8. **Explore Chunking Strategies**
- Research effective chunking approaches for documentation
- Prototype and evaluate performance
- Implement if benefits justify the complexity
## Technical Design
### Core Components
1. **ContextBuilder** - Enhanced version of current ProjectContext
```typescript
interface IContextBuilder {
buildContext(): Promise<string>;
getTokenCount(): number;
setContextMode(mode: 'normal' | 'trimmed' | 'summarized'): void;
setTokenBudget(maxTokens: number): void;
setPrioritizationStrategy(strategy: IPrioritizationStrategy): void;
}
```
2. **FileProcessor** - Handles per-file processing and trimming
```typescript
interface IFileProcessor {
processFile(file: SmartFile): Promise<string>;
setProcessingMode(mode: 'full' | 'trim' | 'summarize'): void;
getTokenCount(): number;
}
```
3. **PrioritizationStrategy** - Ranks files by importance
```typescript
interface IPrioritizationStrategy {
rankFiles(files: SmartFile[], context: string): Promise<SmartFile[]>;
setImportanceMetrics(metrics: IImportanceMetrics): void;
}
```
4. **TaskContextFactory** - Creates optimized contexts for specific tasks
```typescript
interface ITaskContextFactory {
createContextForReadme(projectDir: string): Promise<string>;
createContextForCommit(projectDir: string, diff: string): Promise<string>;
createContextForDescription(projectDir: string): Promise<string>;
}
```
### Configuration Options
The system will support configuration via a new section in `npmextra.json`:
```json
{
"tsdoc": {
"context": {
"maxTokens": 190000,
"defaultMode": "dynamic",
"taskSpecificSettings": {
"readme": {
"mode": "full",
"includePaths": ["src/", "lib/"],
"excludePaths": ["test/", "examples/"]
},
"commit": {
"mode": "trimmed",
"focusOnChangedFiles": true
},
"description": {
"mode": "summarized",
"includePackageInfo": true
}
},
"trimming": {
"removeImplementations": true,
"preserveInterfaces": true,
"preserveTypeDefs": true,
"preserveJSDoc": true,
"maxFunctionLines": 5
}
}
}
}
```
## Cost-Benefit Analysis
### Cost Considerations
1. **Development costs**
- Initial implementation of foundational components (~30-40 hours)
- Testing and validation across different project sizes (~10-15 hours)
- Documentation and configuration examples (~5 hours)
2. **Operational costs**
- Potential increased processing time for context preparation
- Additional API calls for summarization or embeddings approaches
- Monitoring and maintenance of the system
### Benefits
1. **Scalability**
- Support for projects of any size, up to and beyond o4-mini's 200K token limit
- Future-proof design that can adapt to different models and token limits
2. **Quality improvements**
- More focused contexts lead to better AI outputs
- Task-specific optimization improves relevance
- Consistent performance regardless of project size
3. **User experience**
- Predictable behavior for all project sizes
- Transparent token usage reporting
- Configuration options for different usage patterns
## First Deliverable
For immediate improvements, we recommend implementing Dynamic Context Trimming and Task-Specific Contexts first, as these offer the best balance of impact and implementation complexity.
### Implementation Plan for Dynamic Context Trimming
1. Create a basic `ContextTrimmer` class that processes TypeScript files:
- Remove function bodies but keep signatures
- Preserve interface and type definitions
- Keep imports and exports
- Preserve JSDoc comments
2. Integrate with the existing ProjectContext class:
- Add a trimming mode option
- Apply trimming during the context building process
- Track and report token savings
3. Modify the CLI to support trimming options:
- Add a `--trim` flag to enable trimming
- Add a `--trim-level` option for controlling aggressiveness
- Show token usage with and without trimming
This approach could reduce token usage by 40-70% while preserving the essential structure of the codebase, making it suitable for large projects while maintaining high-quality AI outputs.

4
ts/00_commitinfo_data.ts
View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@git.zone/tsdoc', name: '@git.zone/tsdoc',
version: '1.4.5', version: '1.5.0',
description: 'An advanced TypeScript documentation tool using AI to generate and enhance documentation for TypeScript projects.' description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
} }

26
ts/aidocs_classes/commit.ts
View File

@ -31,15 +31,27 @@ export class Commit {
'pnpm-lock.yaml', 'pnpm-lock.yaml',
'package-lock.json', 'package-lock.json',
]); ]);
const projectContext = new ProjectContext(this.projectDir); // Use the new TaskContextFactory for optimized context
let contextString = await projectContext.update(); const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
contextString = ` await taskContextFactory.initialize();
${contextString}
Below is the diff of the uncommitted changes. If nothing is changed, there are no changes: // Generate context specifically for commit task
const contextResult = await taskContextFactory.createContextForCommit(
diffStringArray[0] ? diffStringArray.join('\n\n') : 'No changes.'
);
${diffStringArray[0] ? diffStringArray.join('\n\n') : 'No changes.'} // Get the optimized context string
`; let contextString = contextResult.context;
// Log token usage statistics
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
// Check for token overflow against model limits
const MODEL_TOKEN_LIMIT = 200000; // o4-mini
if (contextResult.tokenCount > MODEL_TOKEN_LIMIT * 0.9) {
console.log(`⚠️ Warning: Context size (${contextResult.tokenCount} tokens) is close to or exceeds model limit (${MODEL_TOKEN_LIMIT} tokens).`);
console.log(`The model may not be able to process all information effectively.`);
}
let result = await this.aiDocsRef.openaiInstance.chat({ let result = await this.aiDocsRef.openaiInstance.chat({
systemMessage: ` systemMessage: `

21
ts/aidocs_classes/description.ts
View File

@ -18,9 +18,16 @@ export class Description {
} }
public async build() { public async build() {
// we can now assemble the directory structure. // Use the new TaskContextFactory for optimized context
const projectContext = new ProjectContext(this.projectDir); const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
const contextString = await projectContext.update(); await taskContextFactory.initialize();
// Generate context specifically for description task
const contextResult = await taskContextFactory.createContextForDescription();
const contextString = contextResult.context;
// Log token usage statistics
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
let result = await this.aiDocsRef.openaiInstance.chat({ let result = await this.aiDocsRef.openaiInstance.chat({
systemMessage: ` systemMessage: `
@ -48,7 +55,11 @@ Don't wrap the JSON in three ticks json!!!
result.message.replace('```json', '').replace('```', ''), result.message.replace('```json', '').replace('```', ''),
); );
const npmextraJson = (await projectContext.gatherFiles()).smartfilesNpmextraJSON; // Create a standard ProjectContext instance for file operations
const projectContext = new ProjectContext(this.projectDir);
const files = await projectContext.gatherFiles();
const npmextraJson = files.smartfilesNpmextraJSON;
const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString()); const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString());
npmextraJsonContent.gitzone.module.description = resultObject.description; npmextraJsonContent.gitzone.module.description = resultObject.description;
@ -58,7 +69,7 @@ Don't wrap the JSON in three ticks json!!!
await npmextraJson.write(); await npmextraJson.write();
// do the same with packageJson // do the same with packageJson
const packageJson = (await projectContext.gatherFiles()).smartfilePackageJSON; const packageJson = files.smartfilePackageJSON;
const packageJsonContent = JSON.parse(packageJson.contents.toString()); const packageJsonContent = JSON.parse(packageJson.contents.toString());
packageJsonContent.description = resultObject.description; packageJsonContent.description = resultObject.description;
packageJsonContent.keywords = resultObject.keywords; packageJsonContent.keywords = resultObject.keywords;

43
ts/aidocs_classes/projectcontext.ts
View File

@ -5,6 +5,8 @@ export class ProjectContext {
// INSTANCE // INSTANCE
public projectDir: string; public projectDir: string;
private tokenCount: number = 0;
private contextString: string = '';
constructor(projectDirArg: string) { constructor(projectDirArg: string) {
this.projectDir = projectDirArg; this.projectDir = projectDirArg;
@ -63,6 +65,24 @@ ${smartfile.contents.toString()}
.join('\n'); .join('\n');
} }
/**
* Calculate the token count for a string using the GPT tokenizer
* @param text The text to count tokens for
* @param model The model to use for token counting (default: gpt-3.5-turbo)
* @returns The number of tokens in the text
*/
public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
try {
// Use the gpt-tokenizer library to count tokens
const tokens = plugins.gptTokenizer.encode(text);
return tokens.length;
} catch (error) {
console.error('Error counting tokens:', error);
// Provide a rough estimate (4 chars per token) if tokenization fails
return Math.ceil(text.length / 4);
}
}
private async buildContext(dirArg: string) { private async buildContext(dirArg: string) {
const files = await this.gatherFiles(); const files = await this.gatherFiles();
let context = await this.convertFilesToContext([ let context = await this.convertFilesToContext([
@ -73,10 +93,33 @@ ${smartfile.contents.toString()}
...files.smartfilesMod, ...files.smartfilesMod,
...files.smartfilesTest, ...files.smartfilesTest,
]); ]);
// Count tokens in the context
this.contextString = context;
this.tokenCount = this.countTokens(context);
// console.log(context); // console.log(context);
return context; return context;
} }
/**
* Get the token count for the current context
* @returns The number of tokens in the context
*/
public getTokenCount(): number {
return this.tokenCount;
}
/**
* Get both the context string and its token count
* @returns An object containing the context string and token count
*/
public getContextWithTokenCount(): { context: string; tokenCount: number } {
return {
context: this.contextString,
tokenCount: this.tokenCount
};
}
public async update() { public async update() {
const result = await this.buildContext(this.projectDir); const result = await this.buildContext(this.projectDir);
return result; return result;

14
ts/aidocs_classes/readme.ts
View File

@ -17,11 +17,19 @@ export class Readme {
public async build() { public async build() {
let finalReadmeString = ``; let finalReadmeString = ``;
// we can now assemble the directory structure. // Use the new TaskContextFactory for optimized context
const projectContext = new ProjectContext(this.projectDir); const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
const contextString = await projectContext.update(); await taskContextFactory.initialize();
// Generate context specifically for readme task
const contextResult = await taskContextFactory.createContextForReadme();
const contextString = contextResult.context;
// Log token usage statistics
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
// lets first check legal before introducung any cost // lets first check legal before introducung any cost
const projectContext = new ProjectContext(this.projectDir);
const npmExtraJson = JSON.parse( const npmExtraJson = JSON.parse(
(await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString() (await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString()
); );

33
ts/classes.aidoc.ts
View File

@ -94,4 +94,37 @@ export class AiDoc {
const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg); const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
return await projectContextInstance.gatherFiles(); return await projectContextInstance.gatherFiles();
} }
/**
* Get the context with token count information
* @param projectDirArg The path to the project directory
* @returns An object containing the context string and its token count
*/
public async getProjectContextWithTokenCount(projectDirArg: string) {
const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
await projectContextInstance.update();
return projectContextInstance.getContextWithTokenCount();
}
/**
* Get just the token count for a project's context
* @param projectDirArg The path to the project directory
* @returns The number of tokens in the project context
*/
public async getProjectContextTokenCount(projectDirArg: string) {
const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
await projectContextInstance.update();
return projectContextInstance.getTokenCount();
}
/**
* Count tokens in a text string using GPT tokenizer
* @param text The text to count tokens for
* @param model The model to use for tokenization (default: gpt-3.5-turbo)
* @returns The number of tokens in the text
*/
public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
const projectContextInstance = new aiDocsClasses.ProjectContext('');
return projectContextInstance.countTokens(text, model);
}
} }

128
ts/cli.ts
View File

@ -4,6 +4,7 @@ import { logger } from './logging.js';
import { TypeDoc } from './classes.typedoc.js'; import { TypeDoc } from './classes.typedoc.js';
import { AiDoc } from './classes.aidoc.js'; import { AiDoc } from './classes.aidoc.js';
import * as context from './context/index.js';
export const run = async () => { export const run = async () => {
const tsdocCli = new plugins.smartcli.Smartcli(); const tsdocCli = new plugins.smartcli.Smartcli();
@ -30,6 +31,18 @@ export const run = async () => {
tsdocCli.addCommand('aidoc').subscribe(async (argvArg) => { tsdocCli.addCommand('aidoc').subscribe(async (argvArg) => {
const aidocInstance = new AiDoc(); const aidocInstance = new AiDoc();
await aidocInstance.start(); await aidocInstance.start();
// Get context token count if requested
if (argvArg.tokens || argvArg.showTokens) {
logger.log('info', `Calculating context token count...`);
const tokenCount = await aidocInstance.getProjectContextTokenCount(paths.cwd);
logger.log('ok', `Total context token count: ${tokenCount}`);
if (argvArg.tokensOnly) {
return; // Exit early if we only want token count
}
}
logger.log('info', `Generating new readme...`); logger.log('info', `Generating new readme...`);
logger.log('info', `This may take some time...`); logger.log('info', `This may take some time...`);
await aidocInstance.buildReadme(paths.cwd); await aidocInstance.buildReadme(paths.cwd);
@ -38,6 +51,121 @@ export const run = async () => {
await aidocInstance.buildDescription(paths.cwd); await aidocInstance.buildDescription(paths.cwd);
}); });
tsdocCli.addCommand('tokens').subscribe(async (argvArg) => {
const aidocInstance = new AiDoc();
await aidocInstance.start();
logger.log('info', `Calculating context token count...`);
// Determine context mode based on args
let contextMode: context.ContextMode = 'full';
if (argvArg.trim || argvArg.trimmed) {
contextMode = 'trimmed';
} else if (argvArg.summarize || argvArg.summarized) {
contextMode = 'summarized';
}
// Get task type if specified
let taskType: context.TaskType | undefined = undefined;
if (argvArg.task) {
if (['readme', 'commit', 'description'].includes(argvArg.task)) {
taskType = argvArg.task as context.TaskType;
} else {
logger.log('warn', `Unknown task type: ${argvArg.task}. Using default context.`);
}
}
// Use enhanced context
const taskFactory = new context.TaskContextFactory(paths.cwd);
await taskFactory.initialize();
let contextResult: context.IContextResult;
if (argvArg.all) {
// Show stats for all task types
const stats = await taskFactory.getTokenStats();
logger.log('ok', 'Token statistics by task:');
for (const [task, data] of Object.entries(stats)) {
logger.log('info', `\n${task.toUpperCase()}:`);
logger.log('info', ` Tokens: ${data.tokenCount}`);
logger.log('info', ` Token savings: ${data.savings}`);
logger.log('info', ` Files: ${data.includedFiles} included, ${data.trimmedFiles} trimmed, ${data.excludedFiles} excluded`);
// Calculate percentage of model context
const o4MiniPercentage = (data.tokenCount / 200000 * 100).toFixed(2);
logger.log('info', ` Context usage: ${o4MiniPercentage}% of o4-mini (200K tokens)`);
}
return;
}
if (taskType) {
// Get context for specific task
contextResult = await taskFactory.createContextForTask(taskType);
} else {
// Get generic context with specified mode
const enhancedContext = new context.EnhancedContext(paths.cwd);
await enhancedContext.initialize();
enhancedContext.setContextMode(contextMode);
if (argvArg.maxTokens) {
enhancedContext.setTokenBudget(parseInt(argvArg.maxTokens, 10));
}
contextResult = await enhancedContext.buildContext();
}
// Display results
logger.log('ok', `Total context token count: ${contextResult.tokenCount}`);
logger.log('info', `Files included: ${contextResult.includedFiles.length}`);
logger.log('info', `Files trimmed: ${contextResult.trimmedFiles.length}`);
logger.log('info', `Files excluded: ${contextResult.excludedFiles.length}`);
logger.log('info', `Token savings: ${contextResult.tokenSavings}`);
if (argvArg.detailed) {
// Show more detailed info about the context and token usage
const o4MiniPercentage = (contextResult.tokenCount / 200000 * 100).toFixed(2);
logger.log('info', `Token usage: ${o4MiniPercentage}% of o4-mini 200K token context window`);
if (argvArg.model) {
// Show percentages for different models
if (argvArg.model === 'gpt4') {
const gpt4Percentage = (contextResult.tokenCount / 8192 * 100).toFixed(2);
logger.log('info', `Token usage (GPT-4): ${gpt4Percentage}% of 8192 token context window`);
} else if (argvArg.model === 'gpt35') {
const gpt35Percentage = (contextResult.tokenCount / 4096 * 100).toFixed(2);
logger.log('info', `Token usage (GPT-3.5): ${gpt35Percentage}% of 4096 token context window`);
}
}
// Estimate cost (approximate values)
const o4MiniInputCost = 0.00005; // per 1K tokens for o4-mini
const estimatedCost = (contextResult.tokenCount / 1000 * o4MiniInputCost).toFixed(6);
logger.log('info', `Estimated input cost: $${estimatedCost} (o4-mini)`);
if (argvArg.listFiles) {
// List files included in context
logger.log('info', '\nIncluded files:');
contextResult.includedFiles.forEach(file => {
logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`);
});
logger.log('info', '\nTrimmed files:');
contextResult.trimmedFiles.forEach(file => {
logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`);
});
if (contextResult.excludedFiles.length > 0) {
logger.log('info', '\nExcluded files:');
contextResult.excludedFiles.forEach(file => {
logger.log('info', ` ${file.relativePath} (${file.tokenCount} tokens)`);
});
}
}
}
});
tsdocCli.addCommand('test').subscribe((argvArg) => { tsdocCli.addCommand('test').subscribe((argvArg) => {
tsdocCli.triggerCommand('typedoc', argvArg); tsdocCli.triggerCommand('typedoc', argvArg);
process.on('exit', async () => { process.on('exit', async () => {

209
ts/context/config-manager.ts Normal file
View File

@ -0,0 +1,209 @@
import * as plugins from '../plugins.js';
import type { IContextConfig, ITrimConfig, ITaskConfig, TaskType, ContextMode } from './types.js';
/**
* Manages configuration for context building
*/
export class ConfigManager {
private static instance: ConfigManager;
private config: IContextConfig;
private projectDir: string = '';
/**
* Get the singleton instance of ConfigManager
*/
public static getInstance(): ConfigManager {
if (!ConfigManager.instance) {
ConfigManager.instance = new ConfigManager();
}
return ConfigManager.instance;
}
/**
* Private constructor for singleton pattern
*/
private constructor() {
this.config = this.getDefaultConfig();
}
/**
* Initialize the config manager with a project directory
* @param projectDir The project directory
*/
public async initialize(projectDir: string): Promise<void> {
this.projectDir = projectDir;
await this.loadConfig();
}
/**
* Get the default configuration
*/
private getDefaultConfig(): IContextConfig {
return {
maxTokens: 190000, // Default for o4-mini with some buffer
defaultMode: 'trimmed',
taskSpecificSettings: {
readme: {
mode: 'trimmed',
includePaths: ['ts/', 'src/'],
excludePaths: ['test/', 'node_modules/']
},
commit: {
mode: 'trimmed',
focusOnChangedFiles: true
},
description: {
mode: 'trimmed',
includePackageInfo: true
}
},
trimming: {
removeImplementations: true,
preserveInterfaces: true,
preserveTypeDefs: true,
preserveJSDoc: true,
maxFunctionLines: 5,
removeComments: true,
removeBlankLines: true
}
};
}
/**
* Load configuration from npmextra.json
*/
private async loadConfig(): Promise<void> {
try {
if (!this.projectDir) {
return;
}
// Create KeyValueStore for this project
// We'll just use smartfile directly instead of KeyValueStore
// Read the npmextra.json file
const npmextraJsonFile = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'npmextra.json')
);
const npmextraContent = JSON.parse(npmextraJsonFile.contents.toString());
// Check for tsdoc context configuration
if (npmextraContent?.tsdoc?.context) {
// Merge with default config
this.config = this.mergeConfigs(this.config, npmextraContent.tsdoc.context);
}
} catch (error) {
console.error('Error loading context configuration:', error);
}
}
/**
* Merge configurations, with userConfig taking precedence
* @param defaultConfig The default configuration
* @param userConfig The user configuration
*/
private mergeConfigs(defaultConfig: IContextConfig, userConfig: Partial<IContextConfig>): IContextConfig {
const result: IContextConfig = { ...defaultConfig };
// Merge top-level properties
if (userConfig.maxTokens !== undefined) result.maxTokens = userConfig.maxTokens;
if (userConfig.defaultMode !== undefined) result.defaultMode = userConfig.defaultMode;
// Merge task-specific settings
if (userConfig.taskSpecificSettings) {
result.taskSpecificSettings = result.taskSpecificSettings || {};
// For each task type, merge settings
(['readme', 'commit', 'description'] as TaskType[]).forEach(taskType => {
if (userConfig.taskSpecificSettings?.[taskType]) {
result.taskSpecificSettings![taskType] = {
...result.taskSpecificSettings![taskType],
...userConfig.taskSpecificSettings[taskType]
};
}
});
}
// Merge trimming configuration
if (userConfig.trimming) {
result.trimming = {
...result.trimming,
...userConfig.trimming
};
}
return result;
}
/**
* Get the complete configuration
*/
public getConfig(): IContextConfig {
return this.config;
}
/**
* Get the trimming configuration
*/
public getTrimConfig(): ITrimConfig {
return this.config.trimming || {};
}
/**
* Get configuration for a specific task
* @param taskType The type of task
*/
public getTaskConfig(taskType: TaskType): ITaskConfig {
// Get task-specific config or empty object
const taskConfig = this.config.taskSpecificSettings?.[taskType] || {};
// If mode is not specified, use default mode
if (!taskConfig.mode) {
taskConfig.mode = this.config.defaultMode;
}
return taskConfig;
}
/**
* Get the maximum tokens allowed for context
*/
public getMaxTokens(): number {
return this.config.maxTokens || 190000;
}
/**
* Update the configuration
* @param config The new configuration
*/
public async updateConfig(config: Partial<IContextConfig>): Promise<void> {
// Merge with existing config
this.config = this.mergeConfigs(this.config, config);
try {
if (!this.projectDir) {
return;
}
// Read the existing npmextra.json file
const npmextraJsonPath = plugins.path.join(this.projectDir, 'npmextra.json');
let npmextraContent = {};
if (await plugins.smartfile.fs.fileExists(npmextraJsonPath)) {
const npmextraJsonFile = await plugins.smartfile.SmartFile.fromFilePath(npmextraJsonPath);
npmextraContent = JSON.parse(npmextraJsonFile.contents.toString()) || {};
}
// Update the tsdoc context configuration
const typedContent = npmextraContent as any;
if (!typedContent.tsdoc) typedContent.tsdoc = {};
typedContent.tsdoc.context = this.config;
// Write back to npmextra.json
const updatedContent = JSON.stringify(npmextraContent, null, 2);
await plugins.smartfile.memory.toFs(updatedContent, npmextraJsonPath);
} catch (error) {
console.error('Error updating context configuration:', error);
}
}
}

246
ts/context/context-trimmer.ts Normal file
View File

@ -0,0 +1,246 @@
import * as plugins from '../plugins.js';
import type { ITrimConfig, ContextMode } from './types.js';
/**
* Class responsible for trimming file contents to reduce token usage
* while preserving important information for context
*/
export class ContextTrimmer {
private config: ITrimConfig;
/**
* Create a new ContextTrimmer with the given configuration
* @param config The trimming configuration
*/
constructor(config?: ITrimConfig) {
this.config = {
removeImplementations: true,
preserveInterfaces: true,
preserveTypeDefs: true,
preserveJSDoc: true,
maxFunctionLines: 5,
removeComments: true,
removeBlankLines: true,
...config
};
}
/**
* Trim a file's contents based on the configuration
* @param filePath The path to the file
* @param content The file's contents
* @param mode The context mode to use
* @returns The trimmed file contents
*/
public trimFile(filePath: string, content: string, mode: ContextMode = 'trimmed'): string {
// If mode is 'full', return the original content
if (mode === 'full') {
return content;
}
// Process based on file type
if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) {
return this.trimTypeScriptFile(content);
} else if (filePath.endsWith('.md')) {
return this.trimMarkdownFile(content);
} else if (filePath.endsWith('.json')) {
return this.trimJsonFile(content);
}
// Default to returning the original content for unknown file types
return content;
}
/**
* Trim a TypeScript file to reduce token usage
* @param content The TypeScript file contents
* @returns The trimmed file contents
*/
private trimTypeScriptFile(content: string): string {
let result = content;
// Step 1: Preserve JSDoc comments if configured
const jsDocComments: string[] = [];
if (this.config.preserveJSDoc) {
const jsDocRegex = /\/\*\*[\s\S]*?\*\//g;
const matches = result.match(jsDocRegex) || [];
jsDocComments.push(...matches);
}
// Step 2: Remove comments if configured
if (this.config.removeComments) {
// Remove single-line comments
result = result.replace(/\/\/.*$/gm, '');
// Remove multi-line comments (except JSDoc if preserveJSDoc is true)
if (!this.config.preserveJSDoc) {
result = result.replace(/\/\*[\s\S]*?\*\//g, '');
} else {
// Only remove non-JSDoc comments
result = result.replace(/\/\*(?!\*)[\s\S]*?\*\//g, '');
}
}
// Step 3: Remove function implementations if configured
if (this.config.removeImplementations) {
// Match function and method bodies
result = result.replace(
/(\b(function|constructor|async function)\s+[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
(match, start, funcType, body, end) => {
// Keep function signature and opening brace, replace body with comment
return `${start} /* implementation removed */ ${end}`;
}
);
// Match arrow function bodies
result = result.replace(
/(\([^)]*\)\s*=>\s*{)([\s\S]*?)(})/g,
(match, start, body, end) => {
return `${start} /* implementation removed */ ${end}`;
}
);
// Match method declarations
result = result.replace(
/(^\s*[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/gm,
(match, start, body, end) => {
return `${start} /* implementation removed */ ${end}`;
}
);
// Match class methods
result = result.replace(
/(\b(public|private|protected|static|async)?\s+[\w$]+\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
(match, start, modifier, body, end) => {
return `${start} /* implementation removed */ ${end}`;
}
);
} else if (this.config.maxFunctionLines && this.config.maxFunctionLines > 0) {
// If not removing implementations completely, limit the number of lines
// Match function and method bodies
result = result.replace(
/(\b(function|constructor|async function)\s+[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
(match, start, funcType, body, end) => {
return this.limitFunctionBody(start, body, end);
}
);
// Match arrow function bodies
result = result.replace(
/(\([^)]*\)\s*=>\s*{)([\s\S]*?)(})/g,
(match, start, body, end) => {
return this.limitFunctionBody(start, body, end);
}
);
// Match method declarations
result = result.replace(
/(^\s*[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/gm,
(match, start, body, end) => {
return this.limitFunctionBody(start, body, end);
}
);
// Match class methods
result = result.replace(
/(\b(public|private|protected|static|async)?\s+[\w$]+\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
(match, start, modifier, body, end) => {
return this.limitFunctionBody(start, body, end);
}
);
}
// Step 4: Remove blank lines if configured
if (this.config.removeBlankLines) {
result = result.replace(/^\s*[\r\n]/gm, '');
}
// Step 5: Restore preserved JSDoc comments
if (this.config.preserveJSDoc && jsDocComments.length > 0) {
// This is a placeholder; we already preserved JSDoc comments in the regex steps
}
return result;
}
/**
* Limit a function body to a maximum number of lines
* @param start The function signature and opening brace
* @param body The function body
* @param end The closing brace
* @returns The limited function body
*/
private limitFunctionBody(start: string, body: string, end: string): string {
const lines = body.split('\n');
if (lines.length > this.config.maxFunctionLines!) {
const limitedBody = lines.slice(0, this.config.maxFunctionLines!).join('\n');
return `${start}${limitedBody}\n // ... (${lines.length - this.config.maxFunctionLines!} lines trimmed)\n${end}`;
}
return `${start}${body}${end}`;
}
/**
* Trim a Markdown file to reduce token usage
* @param content The Markdown file contents
* @returns The trimmed file contents
*/
private trimMarkdownFile(content: string): string {
// For markdown files, we generally want to keep most content
// but we can remove lengthy code blocks if needed
return content;
}
/**
* Trim a JSON file to reduce token usage
* @param content The JSON file contents
* @returns The trimmed file contents
*/
private trimJsonFile(content: string): string {
try {
// Parse the JSON
const json = JSON.parse(content);
// For package.json, keep only essential information
if ('name' in json && 'version' in json && 'dependencies' in json) {
const essentialKeys = [
'name', 'version', 'description', 'author', 'license',
'main', 'types', 'exports', 'type'
];
const trimmedJson: any = {};
essentialKeys.forEach(key => {
if (key in json) {
trimmedJson[key] = json[key];
}
});
// Add dependency information without versions
if ('dependencies' in json) {
trimmedJson.dependencies = Object.keys(json.dependencies).reduce((acc, dep) => {
acc[dep] = '*'; // Replace version with wildcard
return acc;
}, {} as Record<string, string>);
}
// Return the trimmed JSON
return JSON.stringify(trimmedJson, null, 2);
}
// For other JSON files, leave as is
return content;
} catch (error) {
// If there's an error parsing the JSON, return the original content
return content;
}
}
/**
* Update the trimmer configuration
* @param config The new configuration to apply
*/
public updateConfig(config: ITrimConfig): void {
this.config = {
...this.config,
...config
};
}
}

343
ts/context/enhanced-context.ts Normal file
View File

@ -0,0 +1,343 @@
import * as plugins from '../plugins.js';
import type { ContextMode, IContextResult, IFileInfo, TaskType } from './types.js';
import { ContextTrimmer } from './context-trimmer.js';
import { ConfigManager } from './config-manager.js';
/**
* Enhanced ProjectContext that supports context optimization strategies
*/
export class EnhancedContext {
private projectDir: string;
private trimmer: ContextTrimmer;
private configManager: ConfigManager;
private contextMode: ContextMode = 'trimmed';
private tokenBudget: number = 190000; // Default for o4-mini
private contextResult: IContextResult = {
context: '',
tokenCount: 0,
includedFiles: [],
trimmedFiles: [],
excludedFiles: [],
tokenSavings: 0
};
/**
* Create a new EnhancedContext
* @param projectDirArg The project directory
*/
constructor(projectDirArg: string) {
this.projectDir = projectDirArg;
this.configManager = ConfigManager.getInstance();
this.trimmer = new ContextTrimmer(this.configManager.getTrimConfig());
}
/**
* Initialize the context builder
*/
public async initialize(): Promise<void> {
await this.configManager.initialize(this.projectDir);
this.tokenBudget = this.configManager.getMaxTokens();
this.trimmer.updateConfig(this.configManager.getTrimConfig());
}
/**
* Set the context mode
* @param mode The context mode to use
*/
public setContextMode(mode: ContextMode): void {
this.contextMode = mode;
}
/**
* Set the token budget
* @param maxTokens The maximum tokens to use
*/
public setTokenBudget(maxTokens: number): void {
this.tokenBudget = maxTokens;
}
/**
* Gather files from the project
* @param includePaths Optional paths to include
* @param excludePaths Optional paths to exclude
*/
public async gatherFiles(includePaths?: string[], excludePaths?: string[]): Promise<Record<string, plugins.smartfile.SmartFile | plugins.smartfile.SmartFile[]>> {
const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'package.json'),
this.projectDir,
);
const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'readme.md'),
this.projectDir,
);
const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'readme.hints.md'),
this.projectDir,
);
const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(
plugins.path.join(this.projectDir, 'npmextra.json'),
this.projectDir,
);
// Use provided include paths or default to all TypeScript files
const includeGlobs = includePaths?.map(path => `${path}/**/*.ts`) || ['ts*/**/*.ts'];
// Get TypeScript files
const smartfilesModPromises = includeGlobs.map(glob =>
plugins.smartfile.fs.fileTreeToObject(this.projectDir, glob)
);
const smartfilesModArrays = await Promise.all(smartfilesModPromises);
// Flatten the arrays
const smartfilesMod: plugins.smartfile.SmartFile[] = [];
smartfilesModArrays.forEach(array => {
smartfilesMod.push(...array);
});
// Get test files if not excluded
let smartfilesTest: plugins.smartfile.SmartFile[] = [];
if (!excludePaths?.includes('test/')) {
smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(
this.projectDir,
'test/**/*.ts',
);
}
return {
smartfilePackageJSON,
smartfilesReadme,
smartfilesReadmeHints,
smartfilesNpmextraJSON,
smartfilesMod,
smartfilesTest,
};
}
/**
* Convert files to context string
* @param files The files to convert
* @param mode The context mode to use
*/
public async convertFilesToContext(
files: plugins.smartfile.SmartFile[],
mode: ContextMode = this.contextMode
): Promise<string> {
// Reset context result
this.contextResult = {
context: '',
tokenCount: 0,
includedFiles: [],
trimmedFiles: [],
excludedFiles: [],
tokenSavings: 0
};
let totalTokenCount = 0;
let totalOriginalTokens = 0;
// Sort files by importance (for now just a simple alphabetical sort)
// Later this could be enhanced with more sophisticated prioritization
const sortedFiles = [...files].sort((a, b) => a.relative.localeCompare(b.relative));
const processedFiles: string[] = [];
for (const smartfile of sortedFiles) {
// Calculate original token count
const originalContent = smartfile.contents.toString();
const originalTokenCount = this.countTokens(originalContent);
totalOriginalTokens += originalTokenCount;
// Apply trimming based on mode
let processedContent = originalContent;
if (mode !== 'full') {
processedContent = this.trimmer.trimFile(
smartfile.relative,
originalContent,
mode
);
}
// Calculate new token count
const processedTokenCount = this.countTokens(processedContent);
// Check if we have budget for this file
if (totalTokenCount + processedTokenCount > this.tokenBudget) {
// We don't have budget for this file
this.contextResult.excludedFiles.push({
path: smartfile.path,
contents: originalContent,
relativePath: smartfile.relative,
tokenCount: originalTokenCount
});
continue;
}
// Format the file for context
const formattedContent = `
====== START OF FILE ${smartfile.relative} ======
${processedContent}
====== END OF FILE ${smartfile.relative} ======
`;
processedFiles.push(formattedContent);
totalTokenCount += processedTokenCount;
// Track file in appropriate list
const fileInfo: IFileInfo = {
path: smartfile.path,
contents: processedContent,
relativePath: smartfile.relative,
tokenCount: processedTokenCount
};
if (mode === 'full' || processedContent === originalContent) {
this.contextResult.includedFiles.push(fileInfo);
} else {
this.contextResult.trimmedFiles.push(fileInfo);
this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
}
}
// Join all processed files
const context = processedFiles.join('\n');
// Update context result
this.contextResult.context = context;
this.contextResult.tokenCount = totalTokenCount;
return context;
}
/**
* Build context for the project
* @param taskType Optional task type for task-specific context
*/
public async buildContext(taskType?: TaskType): Promise<IContextResult> {
// Initialize if needed
if (this.tokenBudget === 0) {
await this.initialize();
}
// Get task-specific configuration if a task type is provided
if (taskType) {
const taskConfig = this.configManager.getTaskConfig(taskType);
if (taskConfig.mode) {
this.setContextMode(taskConfig.mode);
}
}
// Gather files
const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
const files = await this.gatherFiles(
taskConfig?.includePaths,
taskConfig?.excludePaths
);
// Convert files to context
// Create an array of all files to process
const allFiles: plugins.smartfile.SmartFile[] = [];
// Add individual files
if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
// Add arrays of files
if (files.smartfilesMod) {
if (Array.isArray(files.smartfilesMod)) {
allFiles.push(...files.smartfilesMod);
} else {
allFiles.push(files.smartfilesMod);
}
}
if (files.smartfilesTest) {
if (Array.isArray(files.smartfilesTest)) {
allFiles.push(...files.smartfilesTest);
} else {
allFiles.push(files.smartfilesTest);
}
}
const context = await this.convertFilesToContext(allFiles);
return this.contextResult;
}
/**
* Update the context with git diff information for commit tasks
* @param gitDiff The git diff to include
*/
public updateWithGitDiff(gitDiff: string): IContextResult {
// If we don't have a context yet, return empty result
if (!this.contextResult.context) {
return this.contextResult;
}
// Add git diff to context
const diffSection = `
====== GIT DIFF ======
${gitDiff}
====== END GIT DIFF ======
`;
const diffTokenCount = this.countTokens(diffSection);
// Update context and token count
this.contextResult.context += diffSection;
this.contextResult.tokenCount += diffTokenCount;
return this.contextResult;
}
/**
* Count tokens in a string
* @param text The text to count tokens for
* @param model The model to use for token counting
*/
public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
try {
// Use the gpt-tokenizer library to count tokens
const tokens = plugins.gptTokenizer.encode(text);
return tokens.length;
} catch (error) {
console.error('Error counting tokens:', error);
// Provide a rough estimate if tokenization fails
return Math.ceil(text.length / 4);
}
}
/**
* Get the context result
*/
public getContextResult(): IContextResult {
return this.contextResult;
}
/**
* Get the token count for the current context
*/
public getTokenCount(): number {
return this.contextResult.tokenCount;
}
/**
* Get both the context string and its token count
*/
public getContextWithTokenCount(): { context: string; tokenCount: number } {
return {
context: this.contextResult.context,
tokenCount: this.contextResult.tokenCount
};
}
}

32
ts/context/index.ts Normal file
View File

@ -0,0 +1,32 @@
import { EnhancedContext } from './enhanced-context.js';
import { TaskContextFactory } from './task-context-factory.js';
import { ConfigManager } from './config-manager.js';
import { ContextTrimmer } from './context-trimmer.js';
import type {
ContextMode,
IContextConfig,
IContextResult,
IFileInfo,
ITrimConfig,
ITaskConfig,
TaskType
} from './types.js';
export {
// Classes
EnhancedContext,
TaskContextFactory,
ConfigManager,
ContextTrimmer,
};
// Types
export type {
ContextMode,
IContextConfig,
IContextResult,
IFileInfo,
ITrimConfig,
ITaskConfig,
TaskType
};

138
ts/context/task-context-factory.ts Normal file
View File

@ -0,0 +1,138 @@
import * as plugins from '../plugins.js';
import { EnhancedContext } from './enhanced-context.js';
import { ConfigManager } from './config-manager.js';
import type { IContextResult, TaskType } from './types.js';
/**
* Factory class for creating task-specific context
*/
export class TaskContextFactory {
private projectDir: string;
private configManager: ConfigManager;
/**
* Create a new TaskContextFactory
* @param projectDirArg The project directory
*/
constructor(projectDirArg: string) {
this.projectDir = projectDirArg;
this.configManager = ConfigManager.getInstance();
}
/**
* Initialize the factory
*/
public async initialize(): Promise<void> {
await this.configManager.initialize(this.projectDir);
}
/**
* Create context for README generation
*/
public async createContextForReadme(): Promise<IContextResult> {
const contextBuilder = new EnhancedContext(this.projectDir);
await contextBuilder.initialize();
// Get README-specific configuration
const taskConfig = this.configManager.getTaskConfig('readme');
if (taskConfig.mode) {
contextBuilder.setContextMode(taskConfig.mode);
}
// Build the context for README task
return await contextBuilder.buildContext('readme');
}
/**
* Create context for description generation
*/
public async createContextForDescription(): Promise<IContextResult> {
const contextBuilder = new EnhancedContext(this.projectDir);
await contextBuilder.initialize();
// Get description-specific configuration
const taskConfig = this.configManager.getTaskConfig('description');
if (taskConfig.mode) {
contextBuilder.setContextMode(taskConfig.mode);
}
// Build the context for description task
return await contextBuilder.buildContext('description');
}
/**
* Create context for commit message generation
* @param gitDiff Optional git diff to include
*/
public async createContextForCommit(gitDiff?: string): Promise<IContextResult> {
const contextBuilder = new EnhancedContext(this.projectDir);
await contextBuilder.initialize();
// Get commit-specific configuration
const taskConfig = this.configManager.getTaskConfig('commit');
if (taskConfig.mode) {
contextBuilder.setContextMode(taskConfig.mode);
}
// Build the context for commit task
const contextResult = await contextBuilder.buildContext('commit');
// If git diff is provided, add it to the context
if (gitDiff) {
contextBuilder.updateWithGitDiff(gitDiff);
}
return contextBuilder.getContextResult();
}
/**
* Create context for any task type
* @param taskType The task type to create context for
* @param additionalContent Optional additional content to include
*/
public async createContextForTask(
taskType: TaskType,
additionalContent?: string
): Promise<IContextResult> {
switch (taskType) {
case 'readme':
return this.createContextForReadme();
case 'description':
return this.createContextForDescription();
case 'commit':
return this.createContextForCommit(additionalContent);
default:
// Generic context for unknown task types
const contextBuilder = new EnhancedContext(this.projectDir);
await contextBuilder.initialize();
return await contextBuilder.buildContext();
}
}
/**
* Get token stats for all task types
*/
public async getTokenStats(): Promise<Record<TaskType, {
tokenCount: number;
savings: number;
includedFiles: number;
trimmedFiles: number;
excludedFiles: number;
}>> {
const taskTypes: TaskType[] = ['readme', 'description', 'commit'];
const stats: Record<TaskType, any> = {} as any;
for (const taskType of taskTypes) {
const result = await this.createContextForTask(taskType);
stats[taskType] = {
tokenCount: result.tokenCount,
savings: result.tokenSavings,
includedFiles: result.includedFiles.length,
trimmedFiles: result.trimmedFiles.length,
excludedFiles: result.excludedFiles.length
};
}
return stats;
}
}

95
ts/context/types.ts Normal file
View File

@ -0,0 +1,95 @@
/**
* Context processing mode to control how context is built
*/
export type ContextMode = 'full' | 'trimmed' | 'summarized';
/**
* Configuration for context trimming
*/
export interface ITrimConfig {
/** Whether to remove function implementations */
removeImplementations?: boolean;
/** Whether to preserve interface definitions */
preserveInterfaces?: boolean;
/** Whether to preserve type definitions */
preserveTypeDefs?: boolean;
/** Whether to preserve JSDoc comments */
preserveJSDoc?: boolean;
/** Maximum lines to keep for function bodies (if not removing completely) */
maxFunctionLines?: number;
/** Whether to remove normal comments (non-JSDoc) */
removeComments?: boolean;
/** Whether to remove blank lines */
removeBlankLines?: boolean;
}
/**
* Task types that require different context optimization
*/
export type TaskType = 'readme' | 'commit' | 'description';
/**
* Configuration for different tasks
*/
export interface ITaskConfig {
/** The context mode to use for this task */
mode?: ContextMode;
/** File paths to include for this task */
includePaths?: string[];
/** File paths to exclude for this task */
excludePaths?: string[];
/** For commit tasks, whether to focus on changed files */
focusOnChangedFiles?: boolean;
/** For description tasks, whether to include package info */
includePackageInfo?: boolean;
}
/**
* Complete context configuration
*/
export interface IContextConfig {
/** Maximum tokens to use for context */
maxTokens?: number;
/** Default context mode */
defaultMode?: ContextMode;
/** Task-specific settings */
taskSpecificSettings?: {
[key in TaskType]?: ITaskConfig;
};
/** Trimming configuration */
trimming?: ITrimConfig;
}
/**
* Basic file information interface
*/
export interface IFileInfo {
/** The file path */
path: string;
/** The file contents */
contents: string;
/** The file's relative path from the project root */
relativePath: string;
/** The estimated token count of the file */
tokenCount?: number;
/** The file's importance score (higher is more important) */
importanceScore?: number;
}
/**
* Result of context building
*/
export interface IContextResult {
/** The generated context string */
context: string;
/** The total token count of the context */
tokenCount: number;
/** Files included in the context */
includedFiles: IFileInfo[];
/** Files that were trimmed */
trimmedFiles: IFileInfo[];
/** Files that were excluded */
excludedFiles: IFileInfo[];
/** Token savings from trimming */
tokenSavings: number;
}

3
ts/plugins.ts
View File

@ -41,5 +41,6 @@ export { tspublish };
// third party scope // third party scope
import * as typedoc from 'typedoc'; import * as typedoc from 'typedoc';
import * as gptTokenizer from 'gpt-tokenizer';
export { typedoc }; export { typedoc, gptTokenizer };