update

pnpm-lock.yaml

@@ -59,9 +59,6 @@ importers:
       '@push.rocks/smarttime':
         specifier: ^4.1.1
         version: 4.1.1
-      gpt-tokenizer:
-        specifier: ^3.4.0
-        version: 3.4.0
       typedoc:
         specifier: ^0.28.15
         version: 0.28.15(typescript@5.9.3)
@@ -3032,9 +3029,6 @@ packages:
     resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==}
     engines: {node: '>=14.16'}
 
-  gpt-tokenizer@3.4.0:
-    resolution: {integrity: sha512-wxFLnhIXTDjYebd9A9pGl3e31ZpSypbpIJSOswbgop5jLte/AsZVDvjlbEuVFlsqZixVKqbcoNmRlFDf6pz/UQ==}
-
   graceful-fs@4.2.10:
     resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==}
 
@@ -9687,8 +9681,6 @@ snapshots:
       p-cancelable: 3.0.0
       responselike: 3.0.0
 
-  gpt-tokenizer@3.4.0: {}
-
   graceful-fs@4.2.10: {}
 
   graceful-fs@4.2.11: {}

ts/aidocs_classes/description.ts

@@ -1,6 +1,7 @@
 import type { AiDoc } from '../classes.aidoc.js';
 import * as plugins from '../plugins.js';
 import { ProjectContext } from './projectcontext.js';
+import { logger } from '../logging.js';
 
 interface IDescriptionInterface {
   description: string;
@@ -18,43 +19,60 @@ export class Description {
   }
 
   public async build() {
-    // Gather project context upfront to avoid token explosion from filesystem tool
+    // Use DualAgentOrchestrator with filesystem tool for agent-driven exploration
-    const projectContext = new ProjectContext(this.projectDir);
-    const files = await projectContext.gatherFiles();
-    const contextString = await projectContext.convertFilesToContext([
-      files.smartfilePackageJSON,
-      files.smartfilesNpmextraJSON,
-      ...files.smartfilesMod.slice(0, 10), // Limit to first 10 source files for description
-    ]);
-
-    // Use DualAgentOrchestrator for description generation
     const descriptionOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
       smartAiInstance: this.aiDocsRef.smartAiInstance,
       defaultProvider: 'openai',
+      maxIterations: 15,
+      maxResultChars: 10000, // Limit tool output to prevent token explosion
+      maxHistoryMessages: 15, // Limit history window
+      logPrefix: '[Description]',
+      onProgress: (event) => logger.log(event.logLevel, event.logMessage),
       guardianPolicyPrompt: `
-You validate description generation.
+You validate description generation tool calls and outputs.
 
-APPROVE if:
+APPROVE tool calls for:
+- Reading package.json, npmextra.json, or source files in the ts/ directory
+- Listing directory contents to understand project structure
+- Using tree to see project structure
+
+REJECT tool calls for:
+- Reading files outside the project directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+For final output, APPROVE if:
 - JSON is valid and parseable
 - Description is a clear, concise one-sentence summary
 - Keywords are relevant to the project's use cases
 - Both description and keywords fields are present
 
-REJECT if:
+REJECT final output if:
 - JSON is malformed or wrapped in markdown code blocks
 - Description is too long or vague
 - Keywords are irrelevant or generic
 `,
     });
 
+    // Register scoped filesystem tool for agent exploration
+    descriptionOrchestrator.registerScopedFilesystemTool(this.projectDir);
+
     await descriptionOrchestrator.start();
 
     const descriptionTaskPrompt = `
 You create a project description and keywords for an npm package.
 
-Analyze the project files provided below to understand the codebase, then generate a description and keywords.
+PROJECT DIRECTORY: ${this.projectDir}
 
-Your response must be valid JSON adhering to this interface:
+Use the filesystem tool to explore the project and understand what it does:
+1. First, use tree to see the project structure
+2. Read package.json to understand the package name and current description
+3. Read npmextra.json if it exists for additional metadata
+4. Read key source files in ts/ directory to understand the implementation
+
+Then generate a description and keywords based on your exploration.
+
+Your FINAL response must be valid JSON adhering to this interface:
 {
   description: string; // a sensible short, one sentence description of the project
   keywords: string[]; // an array of tags that describe the project based on use cases
@@ -63,12 +81,6 @@ Your response must be valid JSON adhering to this interface:
 Important: Answer only in valid JSON.
 Your answer should be parseable with JSON.parse() without modifying anything.
 Don't wrap the JSON in \`\`\`json\`\`\` - just return the raw JSON object.
-
-Here are the project files:
-
-${contextString}
-
-Generate the description based on these files.
 `;
 
     const descriptionResult = await descriptionOrchestrator.run(descriptionTaskPrompt);
@@ -83,7 +95,11 @@ Generate the description based on these files.
       descriptionResult.result.replace('```json', '').replace('```', ''),
     );
 
-    // Use the already gathered files for updates
+    // Use ProjectContext to get file handles for writing
+    const projectContext = new ProjectContext(this.projectDir);
+    const files = await projectContext.gatherFiles();
+
+    // Update npmextra.json
     const npmextraJson = files.smartfilesNpmextraJSON;
     const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString());
 
@@ -93,7 +109,7 @@ Generate the description based on these files.
     npmextraJson.contents = Buffer.from(JSON.stringify(npmextraJsonContent, null, 2));
     await npmextraJson.write();
 
-    // do the same with packageJson
+    // Update package.json
     const packageJson = files.smartfilePackageJSON;
     const packageJsonContent = JSON.parse(packageJson.contents.toString());
     packageJsonContent.description = resultObject.description;

ts/aidocs_classes/readme.ts

@@ -17,7 +17,7 @@ export class Readme {
   public async build() {
     let finalReadmeString = ``;
 
-    // lets first check legal before introducung any cost
+    // First check legal info before introducing any cost
     const projectContext = new ProjectContext(this.projectDir);
     const npmExtraJson = JSON.parse(
       (await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString()
@@ -28,29 +28,36 @@ export class Readme {
       console.log(error);
     }
 
-    // Gather project context upfront to avoid token explosion from filesystem tool
+    // Use DualAgentOrchestrator with filesystem tool for agent-driven exploration
-    const contextString = await projectContext.convertFilesToContext([
-      (await projectContext.gatherFiles()).smartfilePackageJSON,
-      (await projectContext.gatherFiles()).smartfilesReadme,
-      (await projectContext.gatherFiles()).smartfilesReadmeHints,
-      (await projectContext.gatherFiles()).smartfilesNpmextraJSON,
-      ...(await projectContext.gatherFiles()).smartfilesMod,
-    ]);
-
-    // Use DualAgentOrchestrator for readme generation
     const readmeOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
       smartAiInstance: this.aiDocsRef.smartAiInstance,
       defaultProvider: 'openai',
+      maxIterations: 25,
+      maxResultChars: 15000, // Limit tool output to prevent token explosion
+      maxHistoryMessages: 20, // Limit history window
+      logPrefix: '[README]',
+      onProgress: (event) => logger.log(event.logLevel, event.logMessage),
       guardianPolicyPrompt: `
-You validate README generation.
+You validate README generation tool calls and outputs.
 
-APPROVE if:
+APPROVE tool calls for:
+- Reading any files within the project directory (package.json, ts/*.ts, readme.md, etc.)
+- Using tree to see project structure
+- Using glob to find source files
+- Listing directory contents
+
+REJECT tool calls for:
+- Reading files outside the project directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+For final README output, APPROVE if:
 - README follows proper markdown format
 - Contains Install and Usage sections
 - Code examples are correct TypeScript/ESM syntax
 - Documentation is comprehensive and helpful
 
-REJECT if:
+REJECT final output if:
 - README is incomplete or poorly formatted
 - Contains licensing information (added separately)
 - Uses CommonJS syntax instead of ESM
@@ -58,14 +65,25 @@ REJECT if:
 `,
     });
 
+    // Register scoped filesystem tool for agent exploration
+    readmeOrchestrator.registerScopedFilesystemTool(this.projectDir);
+
     await readmeOrchestrator.start();
 
     const readmeTaskPrompt = `
 You create markdown READMEs for npm projects. You only output the markdown readme.
 
-Analyze the project files provided below to understand the codebase, then generate a comprehensive README.
+PROJECT DIRECTORY: ${this.projectDir}
 
-The README should follow this template:
+Use the filesystem tool to explore the project and understand what it does:
+1. First, use tree to see the project structure (maxDepth: 3)
+2. Read package.json to understand the package name, description, and dependencies
+3. Read the existing readme.md if it exists (use it as a base, improve and expand)
+4. Read readme.hints.md if it exists (contains hints for documentation)
+5. Read key source files in ts/ directory to understand the API and implementation
+6. Focus on exported classes, interfaces, and functions
+
+Then generate a comprehensive README following this template:
 
 # Project Name
 [The name from package.json and description]
@@ -86,12 +104,6 @@ The README should follow this template:
   Don't include any licensing information. This will be added later.
   Avoid "in conclusion" statements.
 ]
-
-Here are the project files:
-
-${contextString}
-
-Generate the README based on these files.
 `;
 
     const readmeResult = await readmeOrchestrator.run(readmeTaskPrompt);
@@ -124,48 +136,58 @@ Generate the README based on these files.
     for (const subModule of Object.keys(subModules)) {
       logger.log('info', `Building readme for ${subModule}`);
 
+      const subModulePath = plugins.path.join(paths.cwd, subModule);
       const tspublishData = await plugins.fsInstance
-        .file(plugins.path.join(paths.cwd, subModule, 'tspublish.json'))
+        .file(plugins.path.join(subModulePath, 'tspublish.json'))
         .encoding('utf8')
         .read();
 
-      // Gather submodule context
+      // Create a new orchestrator with filesystem tool for each submodule
-      const subModuleContext = new ProjectContext(plugins.path.join(paths.cwd, subModule));
-      let subModuleContextString = '';
-      try {
-        const subModuleFiles = await subModuleContext.gatherFiles();
-        subModuleContextString = await subModuleContext.convertFilesToContext([
-          subModuleFiles.smartfilePackageJSON,
-          subModuleFiles.smartfilesNpmextraJSON,
-          ...subModuleFiles.smartfilesMod,
-        ]);
-      } catch (e) {
-        // Submodule may not have all files, continue with what we have
-        logger.log('warn', `Could not gather full context for ${subModule}`);
-      }
-
-      // Create a new orchestrator for each submodule
       const subModuleOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
         smartAiInstance: this.aiDocsRef.smartAiInstance,
         defaultProvider: 'openai',
+        maxIterations: 20,
+        maxResultChars: 12000,
+        maxHistoryMessages: 15,
+        logPrefix: `[README:${subModule}]`,
+        onProgress: (event) => logger.log(event.logLevel, event.logMessage),
         guardianPolicyPrompt: `
 You validate README generation for submodules.
 
-APPROVE comprehensive, well-formatted markdown with ESM TypeScript examples.
+APPROVE tool calls for:
+- Reading any files within the submodule directory
+- Using tree to see structure
+- Using glob to find source files
+
+REJECT tool calls for:
+- Reading files outside the submodule directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+APPROVE final README if comprehensive, well-formatted markdown with ESM TypeScript examples.
 REJECT incomplete READMEs or those with licensing info.
 `,
       });
 
+      // Register scoped filesystem tool for the submodule directory
+      subModuleOrchestrator.registerScopedFilesystemTool(subModulePath);
+
       await subModuleOrchestrator.start();
 
       const subModulePrompt = `
 You create markdown READMEs for npm projects. You only output the markdown readme.
 SUB MODULE: ${subModule}
+SUB MODULE DIRECTORY: ${subModulePath}
 
 IMPORTANT: YOU ARE CREATING THE README FOR THIS SUB MODULE: ${subModule}
 The Sub Module will be published with:
 ${JSON.stringify(tspublishData, null, 2)}
 
+Use the filesystem tool to explore the submodule:
+1. Use tree to see the submodule structure
+2. Read package.json to understand the submodule
+3. Read source files in ts/ directory to understand the implementation
+
 Generate a README following the template:
 
 # Project Name
@@ -184,12 +206,6 @@ Generate a README following the template:
 ]
 
 Don't use \`\`\` at the beginning or end. Only for code blocks.
-
-Here are the submodule files:
-
-${subModuleContextString}
-
-Generate the README based on these files.
 `;
 
       const subModuleResult = await subModuleOrchestrator.run(subModulePrompt);
@@ -200,7 +216,7 @@ Generate the README based on these files.
           .replace(/^```markdown\n?/i, '')
           .replace(/\n?```$/i, '') + '\n' + legalInfo;
         await plugins.fsInstance
-          .file(plugins.path.join(paths.cwd, subModule, 'readme.md'))
+          .file(plugins.path.join(subModulePath, 'readme.md'))
           .encoding('utf8')
           .write(subModuleReadmeString);
         logger.log('success', `Built readme for ${subModule}`);