diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2f4c20b..57af0e7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,9 +59,6 @@ importers: '@push.rocks/smarttime': specifier: ^4.1.1 version: 4.1.1 - gpt-tokenizer: - specifier: ^3.4.0 - version: 3.4.0 typedoc: specifier: ^0.28.15 version: 0.28.15(typescript@5.9.3) @@ -3032,9 +3029,6 @@ packages: resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} engines: {node: '>=14.16'} - gpt-tokenizer@3.4.0: - resolution: {integrity: sha512-wxFLnhIXTDjYebd9A9pGl3e31ZpSypbpIJSOswbgop5jLte/AsZVDvjlbEuVFlsqZixVKqbcoNmRlFDf6pz/UQ==} - graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} @@ -9687,8 +9681,6 @@ snapshots: p-cancelable: 3.0.0 responselike: 3.0.0 - gpt-tokenizer@3.4.0: {} - graceful-fs@4.2.10: {} graceful-fs@4.2.11: {} diff --git a/ts/aidocs_classes/description.ts b/ts/aidocs_classes/description.ts index ffaaaa3..bd1b992 100644 --- a/ts/aidocs_classes/description.ts +++ b/ts/aidocs_classes/description.ts @@ -1,6 +1,7 @@ import type { AiDoc } from '../classes.aidoc.js'; import * as plugins from '../plugins.js'; import { ProjectContext } from './projectcontext.js'; +import { logger } from '../logging.js'; interface IDescriptionInterface { description: string; @@ -18,43 +19,60 @@ export class Description { } public async build() { - // Gather project context upfront to avoid token explosion from filesystem tool - const projectContext = new ProjectContext(this.projectDir); - const files = await projectContext.gatherFiles(); - const contextString = await projectContext.convertFilesToContext([ - files.smartfilePackageJSON, - files.smartfilesNpmextraJSON, - ...files.smartfilesMod.slice(0, 10), // Limit to first 10 source files for description - ]); - - // Use DualAgentOrchestrator for description generation + // Use DualAgentOrchestrator with filesystem tool for agent-driven exploration const descriptionOrchestrator = new plugins.smartagent.DualAgentOrchestrator({ smartAiInstance: this.aiDocsRef.smartAiInstance, defaultProvider: 'openai', + maxIterations: 15, + maxResultChars: 10000, // Limit tool output to prevent token explosion + maxHistoryMessages: 15, // Limit history window + logPrefix: '[Description]', + onProgress: (event) => logger.log(event.logLevel, event.logMessage), guardianPolicyPrompt: ` -You validate description generation. +You validate description generation tool calls and outputs. -APPROVE if: +APPROVE tool calls for: +- Reading package.json, npmextra.json, or source files in the ts/ directory +- Listing directory contents to understand project structure +- Using tree to see project structure + +REJECT tool calls for: +- Reading files outside the project directory +- Writing, deleting, or modifying any files +- Any destructive operations + +For final output, APPROVE if: - JSON is valid and parseable - Description is a clear, concise one-sentence summary - Keywords are relevant to the project's use cases - Both description and keywords fields are present -REJECT if: +REJECT final output if: - JSON is malformed or wrapped in markdown code blocks - Description is too long or vague - Keywords are irrelevant or generic `, }); + // Register scoped filesystem tool for agent exploration + descriptionOrchestrator.registerScopedFilesystemTool(this.projectDir); + await descriptionOrchestrator.start(); const descriptionTaskPrompt = ` You create a project description and keywords for an npm package. -Analyze the project files provided below to understand the codebase, then generate a description and keywords. +PROJECT DIRECTORY: ${this.projectDir} -Your response must be valid JSON adhering to this interface: +Use the filesystem tool to explore the project and understand what it does: +1. First, use tree to see the project structure +2. Read package.json to understand the package name and current description +3. Read npmextra.json if it exists for additional metadata +4. Read key source files in ts/ directory to understand the implementation + +Then generate a description and keywords based on your exploration. + +Your FINAL response must be valid JSON adhering to this interface: { description: string; // a sensible short, one sentence description of the project keywords: string[]; // an array of tags that describe the project based on use cases @@ -63,12 +81,6 @@ Your response must be valid JSON adhering to this interface: Important: Answer only in valid JSON. Your answer should be parseable with JSON.parse() without modifying anything. Don't wrap the JSON in \`\`\`json\`\`\` - just return the raw JSON object. - -Here are the project files: - -${contextString} - -Generate the description based on these files. `; const descriptionResult = await descriptionOrchestrator.run(descriptionTaskPrompt); @@ -83,7 +95,11 @@ Generate the description based on these files. descriptionResult.result.replace('```json', '').replace('```', ''), ); - // Use the already gathered files for updates + // Use ProjectContext to get file handles for writing + const projectContext = new ProjectContext(this.projectDir); + const files = await projectContext.gatherFiles(); + + // Update npmextra.json const npmextraJson = files.smartfilesNpmextraJSON; const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString()); @@ -93,7 +109,7 @@ Generate the description based on these files. npmextraJson.contents = Buffer.from(JSON.stringify(npmextraJsonContent, null, 2)); await npmextraJson.write(); - // do the same with packageJson + // Update package.json const packageJson = files.smartfilePackageJSON; const packageJsonContent = JSON.parse(packageJson.contents.toString()); packageJsonContent.description = resultObject.description; diff --git a/ts/aidocs_classes/readme.ts b/ts/aidocs_classes/readme.ts index c825454..3d1b017 100644 --- a/ts/aidocs_classes/readme.ts +++ b/ts/aidocs_classes/readme.ts @@ -17,7 +17,7 @@ export class Readme { public async build() { let finalReadmeString = ``; - // lets first check legal before introducung any cost + // First check legal info before introducing any cost const projectContext = new ProjectContext(this.projectDir); const npmExtraJson = JSON.parse( (await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString() @@ -28,29 +28,36 @@ export class Readme { console.log(error); } - // Gather project context upfront to avoid token explosion from filesystem tool - const contextString = await projectContext.convertFilesToContext([ - (await projectContext.gatherFiles()).smartfilePackageJSON, - (await projectContext.gatherFiles()).smartfilesReadme, - (await projectContext.gatherFiles()).smartfilesReadmeHints, - (await projectContext.gatherFiles()).smartfilesNpmextraJSON, - ...(await projectContext.gatherFiles()).smartfilesMod, - ]); - - // Use DualAgentOrchestrator for readme generation + // Use DualAgentOrchestrator with filesystem tool for agent-driven exploration const readmeOrchestrator = new plugins.smartagent.DualAgentOrchestrator({ smartAiInstance: this.aiDocsRef.smartAiInstance, defaultProvider: 'openai', + maxIterations: 25, + maxResultChars: 15000, // Limit tool output to prevent token explosion + maxHistoryMessages: 20, // Limit history window + logPrefix: '[README]', + onProgress: (event) => logger.log(event.logLevel, event.logMessage), guardianPolicyPrompt: ` -You validate README generation. +You validate README generation tool calls and outputs. -APPROVE if: +APPROVE tool calls for: +- Reading any files within the project directory (package.json, ts/*.ts, readme.md, etc.) +- Using tree to see project structure +- Using glob to find source files +- Listing directory contents + +REJECT tool calls for: +- Reading files outside the project directory +- Writing, deleting, or modifying any files +- Any destructive operations + +For final README output, APPROVE if: - README follows proper markdown format - Contains Install and Usage sections - Code examples are correct TypeScript/ESM syntax - Documentation is comprehensive and helpful -REJECT if: +REJECT final output if: - README is incomplete or poorly formatted - Contains licensing information (added separately) - Uses CommonJS syntax instead of ESM @@ -58,14 +65,25 @@ REJECT if: `, }); + // Register scoped filesystem tool for agent exploration + readmeOrchestrator.registerScopedFilesystemTool(this.projectDir); + await readmeOrchestrator.start(); const readmeTaskPrompt = ` You create markdown READMEs for npm projects. You only output the markdown readme. -Analyze the project files provided below to understand the codebase, then generate a comprehensive README. +PROJECT DIRECTORY: ${this.projectDir} -The README should follow this template: +Use the filesystem tool to explore the project and understand what it does: +1. First, use tree to see the project structure (maxDepth: 3) +2. Read package.json to understand the package name, description, and dependencies +3. Read the existing readme.md if it exists (use it as a base, improve and expand) +4. Read readme.hints.md if it exists (contains hints for documentation) +5. Read key source files in ts/ directory to understand the API and implementation +6. Focus on exported classes, interfaces, and functions + +Then generate a comprehensive README following this template: # Project Name [The name from package.json and description] @@ -86,12 +104,6 @@ The README should follow this template: Don't include any licensing information. This will be added later. Avoid "in conclusion" statements. ] - -Here are the project files: - -${contextString} - -Generate the README based on these files. `; const readmeResult = await readmeOrchestrator.run(readmeTaskPrompt); @@ -124,48 +136,58 @@ Generate the README based on these files. for (const subModule of Object.keys(subModules)) { logger.log('info', `Building readme for ${subModule}`); + const subModulePath = plugins.path.join(paths.cwd, subModule); const tspublishData = await plugins.fsInstance - .file(plugins.path.join(paths.cwd, subModule, 'tspublish.json')) + .file(plugins.path.join(subModulePath, 'tspublish.json')) .encoding('utf8') .read(); - // Gather submodule context - const subModuleContext = new ProjectContext(plugins.path.join(paths.cwd, subModule)); - let subModuleContextString = ''; - try { - const subModuleFiles = await subModuleContext.gatherFiles(); - subModuleContextString = await subModuleContext.convertFilesToContext([ - subModuleFiles.smartfilePackageJSON, - subModuleFiles.smartfilesNpmextraJSON, - ...subModuleFiles.smartfilesMod, - ]); - } catch (e) { - // Submodule may not have all files, continue with what we have - logger.log('warn', `Could not gather full context for ${subModule}`); - } - - // Create a new orchestrator for each submodule + // Create a new orchestrator with filesystem tool for each submodule const subModuleOrchestrator = new plugins.smartagent.DualAgentOrchestrator({ smartAiInstance: this.aiDocsRef.smartAiInstance, defaultProvider: 'openai', + maxIterations: 20, + maxResultChars: 12000, + maxHistoryMessages: 15, + logPrefix: `[README:${subModule}]`, + onProgress: (event) => logger.log(event.logLevel, event.logMessage), guardianPolicyPrompt: ` You validate README generation for submodules. -APPROVE comprehensive, well-formatted markdown with ESM TypeScript examples. +APPROVE tool calls for: +- Reading any files within the submodule directory +- Using tree to see structure +- Using glob to find source files + +REJECT tool calls for: +- Reading files outside the submodule directory +- Writing, deleting, or modifying any files +- Any destructive operations + +APPROVE final README if comprehensive, well-formatted markdown with ESM TypeScript examples. REJECT incomplete READMEs or those with licensing info. `, }); + // Register scoped filesystem tool for the submodule directory + subModuleOrchestrator.registerScopedFilesystemTool(subModulePath); + await subModuleOrchestrator.start(); const subModulePrompt = ` You create markdown READMEs for npm projects. You only output the markdown readme. SUB MODULE: ${subModule} +SUB MODULE DIRECTORY: ${subModulePath} IMPORTANT: YOU ARE CREATING THE README FOR THIS SUB MODULE: ${subModule} The Sub Module will be published with: ${JSON.stringify(tspublishData, null, 2)} +Use the filesystem tool to explore the submodule: +1. Use tree to see the submodule structure +2. Read package.json to understand the submodule +3. Read source files in ts/ directory to understand the implementation + Generate a README following the template: # Project Name @@ -184,12 +206,6 @@ Generate a README following the template: ] Don't use \`\`\` at the beginning or end. Only for code blocks. - -Here are the submodule files: - -${subModuleContextString} - -Generate the README based on these files. `; const subModuleResult = await subModuleOrchestrator.run(subModulePrompt); @@ -200,7 +216,7 @@ Generate the README based on these files. .replace(/^```markdown\n?/i, '') .replace(/\n?```$/i, '') + '\n' + legalInfo; await plugins.fsInstance - .file(plugins.path.join(paths.cwd, subModule, 'readme.md')) + .file(plugins.path.join(subModulePath, 'readme.md')) .encoding('utf8') .write(subModuleReadmeString); logger.log('success', `Built readme for ${subModule}`);