git.zone/tsdoc
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Jürgen Kunz
2025-12-15 14:34:02 +00:00
parent 9cae46e2fe
commit bcded1eafa
22 changed files with 288 additions and 3620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
ts/aidocs_classes/commit.ts
View File

@@ -1,7 +1,7 @@
import * as plugins from '../plugins.js';
import { AiDoc } from '../classes.aidoc.js';
import { ProjectContext } from './projectcontext.js';
import { DiffProcessor } from '../context/diff-processor.js';
import { DiffProcessor } from '../classes.diffprocessor.js';
export interface INextCommitObject {
recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project
@@ -114,32 +114,10 @@ export class Commit {
processedDiffString = 'No changes.';
}
// Use the new TaskContextFactory for optimized context
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
this.projectDir,
this.aiDocsRef.openaiInstance
);
await taskContextFactory.initialize();
// Generate context specifically for commit task
const contextResult = await taskContextFactory.createContextForCommit(processedDiffString);
// Get the optimized context string
let contextString = contextResult.context;
// Log token usage statistics
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
// Check for token overflow against model limits
const MODEL_TOKEN_LIMIT = 200000; // o4-mini
if (contextResult.tokenCount > MODEL_TOKEN_LIMIT * 0.9) {
console.log(`⚠️ Warning: Context size (${contextResult.tokenCount} tokens) is close to or exceeds model limit (${MODEL_TOKEN_LIMIT} tokens).`);
console.log(`The model may not be able to process all information effectively.`);
}
// Use DualAgentOrchestrator for commit message generation with Guardian validation
// Use DualAgentOrchestrator for commit message generation
// Note: No filesystem tool needed - the diff already contains all change information
const commitOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
openaiToken: this.aiDocsRef.getOpenaiToken(),
smartAiInstance: this.aiDocsRef.smartAiInstance,
defaultProvider: 'openai',
guardianPolicyPrompt: `
You validate commit messages for semantic versioning compliance.
@@ -154,7 +132,7 @@ APPROVE if:
REJECT with specific feedback if:
- Version level doesn't match the scope of changes (e.g., "feat" for a typo fix should be "fix")
- Message is vague, unprofessional, or contains sensitive information
- JSON is malformed or missing required fields (recommendedNextVersionLevel, recommendedNextVersionScope, recommendedNextVersionMessage, recommendedNextVersionDetails, recommendedNextVersion)
- JSON is malformed or missing required fields
`,
});
@@ -162,9 +140,12 @@ REJECT with specific feedback if:
const commitTaskPrompt = `
You create a commit message for a git commit.
The commit message should be based on the files in the project.
You should not include any licensing information.
You should not include any personal information.
Project directory: ${this.projectDir}
Analyze the git diff below to understand what changed and generate a commit message.
You should not include any licensing information or personal information.
Never mention CLAUDE code, or codex.
Important: Answer only in valid JSON.
@@ -173,21 +154,20 @@ Your answer should be parseable with JSON.parse() without modifying anything.
Here is the structure of the JSON you should return:
interface {
recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project
recommendedNextVersionScope: string; // the recommended scope name of the next version, like "core" or "cli", or specific class names.
recommendedNextVersionMessage: string; // the commit message. Don't put fix() feat() or BREAKING CHANGE in the message. Please just the message itself.
recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level
recommendedNextVersionScope: string; // scope name like "core", "cli", or specific class names
recommendedNextVersionMessage: string; // the commit message (don't include fix/feat prefix)
recommendedNextVersionDetails: string[]; // detailed bullet points for the changelog
recommendedNextVersion: string; // the recommended next version of the project, x.x.x
recommendedNextVersion: string; // the recommended next version x.x.x
}
For the recommendedNextVersionDetails, please only add a detail entries to the array if it has an obvious value to the reader.
For recommendedNextVersionDetails, only add entries that have obvious value to the reader.
You are being given the files of the project. You should use them to create the commit message.
Also you are given a diff.
Never mention CLAUDE code, or codex.
Here is the git diff showing what changed:
Project context and diff:
${contextString}
${processedDiffString}
Generate the commit message based on these changes.
`;
const commitResult = await commitOrchestrator.run(commitTaskPrompt);
@@ -214,7 +194,7 @@ ${contextString}
// Use DualAgentOrchestrator for changelog generation with Guardian validation
const changelogOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
openaiToken: this.aiDocsRef.getOpenaiToken(),
smartAiInstance: this.aiDocsRef.smartAiInstance,
defaultProvider: 'openai',
guardianPolicyPrompt: `
You validate changelog generation.

108
ts/aidocs_classes/description.ts
View File

@@ -18,50 +18,72 @@ export class Description {
}
public async build() {
// Use the new TaskContextFactory for optimized context
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
this.projectDir,
this.aiDocsRef.openaiInstance
);
await taskContextFactory.initialize();
// Generate context specifically for description task
const contextResult = await taskContextFactory.createContextForDescription();
const contextString = contextResult.context;
// Log token usage statistics
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
let result = await this.aiDocsRef.openaiInstance.chat({
systemMessage: `
You create a json adhering the following interface:
{
description: string; // a sensible short, one sentence description of the project
keywords: string[]; // an array of tags that describe the project
}
The description should be based on what you understand from the project's files.
The keywords should be based on use cases you see from the files.
Don't be cheap about the way you think.
Important: Answer only in valid JSON.
You answer should be parseable with JSON.parse() without modifying anything.
Don't wrap the JSON in three ticks json!!!
`,
messageHistory: [],
userMessage: contextString,
});
console.log(result.message);
const resultObject: IDescriptionInterface = JSON.parse(
result.message.replace('```json', '').replace('```', ''),
);
// Create a standard ProjectContext instance for file operations
// Gather project context upfront to avoid token explosion from filesystem tool
const projectContext = new ProjectContext(this.projectDir);
const files = await projectContext.gatherFiles();
const contextString = await projectContext.convertFilesToContext([
files.smartfilePackageJSON,
files.smartfilesNpmextraJSON,
...files.smartfilesMod.slice(0, 10), // Limit to first 10 source files for description
]);
// Use DualAgentOrchestrator for description generation
const descriptionOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
smartAiInstance: this.aiDocsRef.smartAiInstance,
defaultProvider: 'openai',
guardianPolicyPrompt: `
You validate description generation.
APPROVE if:
- JSON is valid and parseable
- Description is a clear, concise one-sentence summary
- Keywords are relevant to the project's use cases
- Both description and keywords fields are present
REJECT if:
- JSON is malformed or wrapped in markdown code blocks
- Description is too long or vague
- Keywords are irrelevant or generic
`,
});
await descriptionOrchestrator.start();
const descriptionTaskPrompt = `
You create a project description and keywords for an npm package.
Analyze the project files provided below to understand the codebase, then generate a description and keywords.
Your response must be valid JSON adhering to this interface:
{
description: string; // a sensible short, one sentence description of the project
keywords: string[]; // an array of tags that describe the project based on use cases
}
Important: Answer only in valid JSON.
Your answer should be parseable with JSON.parse() without modifying anything.
Don't wrap the JSON in \`\`\`json\`\`\` - just return the raw JSON object.
Here are the project files:
${contextString}
Generate the description based on these files.
`;
const descriptionResult = await descriptionOrchestrator.run(descriptionTaskPrompt);
await descriptionOrchestrator.stop();
if (!descriptionResult.success) {
throw new Error(`Description generation failed: ${descriptionResult.status}`);
}
console.log(descriptionResult.result);
const resultObject: IDescriptionInterface = JSON.parse(
descriptionResult.result.replace('```json', '').replace('```', ''),
);
// Use the already gathered files for updates
const npmextraJson = files.smartfilesNpmextraJSON;
const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString());
@@ -82,6 +104,6 @@ Don't wrap the JSON in three ticks json!!!
console.log(`\n======================\n`);
console.log(JSON.stringify(resultObject, null, 2));
console.log(`\n======================\n`);
return result.message;
return descriptionResult.result;
}
}

21
ts/aidocs_classes/projectcontext.ts
View File

@@ -64,21 +64,14 @@ ${smartfile.contents.toString()}
}
/**
* Calculate the token count for a string using the GPT tokenizer
* @param text The text to count tokens for
* @param model The model to use for token counting (default: gpt-3.5-turbo)
* @returns The number of tokens in the text
* Estimate token count for a string
* Uses a rough estimate of 4 characters per token
* @param text The text to estimate tokens for
* @returns Estimated number of tokens
*/
public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
try {
// Use the gpt-tokenizer library to count tokens
const tokens = plugins.gptTokenizer.encode(text);
return tokens.length;
} catch (error) {
console.error('Error counting tokens:', error);
// Provide a rough estimate (4 chars per token) if tokenization fails
return Math.ceil(text.length / 4);
}
public countTokens(text: string): number {
// Rough estimate: ~4 characters per token for English text
return Math.ceil(text.length / 4);
}
private async buildContext(dirArg: string) {

239
ts/aidocs_classes/readme.ts
View File

@@ -17,20 +17,6 @@ export class Readme {
public async build() {
let finalReadmeString = ``;
// Use the new TaskContextFactory for optimized context
const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(
this.projectDir,
this.aiDocsRef.openaiInstance
);
await taskContextFactory.initialize();
// Generate context specifically for readme task
const contextResult = await taskContextFactory.createContextForReadme();
const contextString = contextResult.context;
// Log token usage statistics
console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
// lets first check legal before introducung any cost
const projectContext = new ProjectContext(this.projectDir);
const npmExtraJson = JSON.parse(
@@ -42,50 +28,88 @@ export class Readme {
console.log(error);
}
let result = await this.aiDocsRef.openaiInstance.chat({
systemMessage: `
You create markdown readmes for npm projects. You only output the markdown readme.
// Gather project context upfront to avoid token explosion from filesystem tool
const contextString = await projectContext.convertFilesToContext([
(await projectContext.gatherFiles()).smartfilePackageJSON,
(await projectContext.gatherFiles()).smartfilesReadme,
(await projectContext.gatherFiles()).smartfilesReadmeHints,
(await projectContext.gatherFiles()).smartfilesNpmextraJSON,
...(await projectContext.gatherFiles()).smartfilesMod,
]);
The Readme should follow the following template:
// Use DualAgentOrchestrator for readme generation
const readmeOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
smartAiInstance: this.aiDocsRef.smartAiInstance,
defaultProvider: 'openai',
guardianPolicyPrompt: `
You validate README generation.
APPROVE if:
- README follows proper markdown format
- Contains Install and Usage sections
- Code examples are correct TypeScript/ESM syntax
- Documentation is comprehensive and helpful
REJECT if:
- README is incomplete or poorly formatted
- Contains licensing information (added separately)
- Uses CommonJS syntax instead of ESM
- Contains "in conclusion" or similar filler
`,
});
await readmeOrchestrator.start();
const readmeTaskPrompt = `
You create markdown READMEs for npm projects. You only output the markdown readme.
Analyze the project files provided below to understand the codebase, then generate a comprehensive README.
The README should follow this template:
# Project Name
[
The name is the module name of package.json
The description is in the description field of package.json
]
[The name from package.json and description]
## Install
[
Write a short text on how to install the project
]
[Short text on how to install the project]
## Usage
[
[
Give code examples here.
Construct sensible scenarios for the user.
Make sure to show a complete set of features of the module.
Don't omit use cases.
It does not matter how much time you need.
ALWAYS USE ESM SYNTAX AND TYPESCRIPT.
DON'T CHICKEN OUT. Write at least 4000 words. More if necessary.
If there is already a readme, take the Usage section as base. Remove outdated content, and expand and improve upon the valid parts.
Super important: Check for completenes.
Don't include any licensing information. This will be added in a later step.
Avoid "in conclusions".
Good to know:
* npmextra.json contains overall module information.
* readme.hints.md provides valuable hints about module ideas.
Write at least 4000 words. More if necessary.
If there is already a readme, take the Usage section as base. Remove outdated content, expand and improve.
Check for completeness.
Don't include any licensing information. This will be added later.
Avoid "in conclusion" statements.
]
`,
messageHistory: [],
userMessage: contextString,
});
finalReadmeString += result.message + '\n' + legalInfo;
Here are the project files:
${contextString}
Generate the README based on these files.
`;
const readmeResult = await readmeOrchestrator.run(readmeTaskPrompt);
await readmeOrchestrator.stop();
if (!readmeResult.success) {
throw new Error(`README generation failed: ${readmeResult.status}`);
}
// Clean up markdown formatting if wrapped in code blocks
let resultMessage = readmeResult.result
.replace(/^```markdown\n?/i, '')
.replace(/\n?```$/i, '');
finalReadmeString += resultMessage + '\n' + legalInfo;
console.log(`\n======================\n`);
console.log(result.message);
console.log(resultMessage);
console.log(`\n======================\n`);
const readme = (await projectContext.gatherFiles()).smartfilesReadme;
@@ -96,60 +120,95 @@ The Readme should follow the following template:
const tsPublishInstance = new plugins.tspublish.TsPublish();
const subModules = await tsPublishInstance.getModuleSubDirs(paths.cwd);
logger.log('info', `Found ${Object.keys(subModules).length} sub modules`);
for (const subModule of Object.keys(subModules)) {
logger.log('info', `Building readme for ${subModule}`);
const subModuleContextString = await projectContext.update();
let result = await this.aiDocsRef.openaiInstance.chat({
systemMessage: `
You create markdown readmes for npm projects. You only output the markdown readme.
IMPORTANT: YOU ARE NOW CREATING THE README FOR THE FOLLOWING SUB MODULE: ${subModule} !!!!!!!!!!!
The Sub Module will be published with the following data:
${JSON.stringify(await plugins.fsInstance.file(plugins.path.join(paths.cwd, subModule, 'tspublish.json')).encoding('utf8').read(), null, 2)}
const tspublishData = await plugins.fsInstance
.file(plugins.path.join(paths.cwd, subModule, 'tspublish.json'))
.encoding('utf8')
.read();
The Readme should follow the following template:
# Project Name
[
The name is the module name of package.json
The description is in the description field of package.json
]
## Install
[
Write a short text on how to install the project
]
## Usage
[
Give code examples here.
Construct sensible scenarios for the user.
Make sure to show a complete set of features of the module.
Don't omit use cases.
It does not matter how much time you need.
ALWAYS USE ESM SYNTAX AND TYPESCRIPT.
DON'T CHICKEN OUT. Write at least 4000 words. More if necessary.
If there is already a readme, take the Usage section as base. Remove outdated content, and expand and improve upon the valid parts.
Super important: Check for completenes.
Don't include any licensing information. This will be added in a later step.
Avoid "in conclusions".
Good to know:
* npmextra.json contains overall module information.
* readme.hints.md provides valuable hints about module ideas.
* Your output lands directly in the readme.md file.
* Don't use \`\`\` at the beginning or the end. It'll cause problems. Only use it for codeblocks. You are directly writing markdown. No need to introduce it weirdly.
]
`,
messageHistory: [],
userMessage: subModuleContextString,
// Gather submodule context
const subModuleContext = new ProjectContext(plugins.path.join(paths.cwd, subModule));
let subModuleContextString = '';
try {
const subModuleFiles = await subModuleContext.gatherFiles();
subModuleContextString = await subModuleContext.convertFilesToContext([
subModuleFiles.smartfilePackageJSON,
subModuleFiles.smartfilesNpmextraJSON,
...subModuleFiles.smartfilesMod,
]);
} catch (e) {
// Submodule may not have all files, continue with what we have
logger.log('warn', `Could not gather full context for ${subModule}`);
}
// Create a new orchestrator for each submodule
const subModuleOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
smartAiInstance: this.aiDocsRef.smartAiInstance,
defaultProvider: 'openai',
guardianPolicyPrompt: `
You validate README generation for submodules.
APPROVE comprehensive, well-formatted markdown with ESM TypeScript examples.
REJECT incomplete READMEs or those with licensing info.
`,
});
const subModuleReadmeString = result.message + '\n' + legalInfo;
await plugins.fsInstance.file(plugins.path.join(paths.cwd, subModule, 'readme.md')).encoding('utf8').write(subModuleReadmeString);
logger.log('success', `Built readme for ${subModule}`);
await subModuleOrchestrator.start();
const subModulePrompt = `
You create markdown READMEs for npm projects. You only output the markdown readme.
SUB MODULE: ${subModule}
IMPORTANT: YOU ARE CREATING THE README FOR THIS SUB MODULE: ${subModule}
The Sub Module will be published with:
${JSON.stringify(tspublishData, null, 2)}
Generate a README following the template:
# Project Name
[name and description from package.json]
## Install
[installation instructions]
## Usage
[
Code examples with complete features.
ESM TypeScript syntax only.
Write at least 4000 words.
No licensing information.
No "in conclusion".
]
Don't use \`\`\` at the beginning or end. Only for code blocks.
Here are the submodule files:
${subModuleContextString}
Generate the README based on these files.
`;
const subModuleResult = await subModuleOrchestrator.run(subModulePrompt);
await subModuleOrchestrator.stop();
if (subModuleResult.success) {
const subModuleReadmeString = subModuleResult.result
.replace(/^```markdown\n?/i, '')
.replace(/\n?```$/i, '') + '\n' + legalInfo;
await plugins.fsInstance
.file(plugins.path.join(paths.cwd, subModule, 'readme.md'))
.encoding('utf8')
.write(subModuleReadmeString);
logger.log('success', `Built readme for ${subModule}`);
} else {
logger.log('error', `Failed to build readme for ${subModule}: ${subModuleResult.status}`);
}
}
return result.message;
return resultMessage;
}
}