update

ts/aidocs_classes/readme.ts

@@ -17,7 +17,7 @@ export class Readme {
   public async build() {
     let finalReadmeString = ``;
 
-    // lets first check legal before introducung any cost
+    // First check legal info before introducing any cost
     const projectContext = new ProjectContext(this.projectDir);
     const npmExtraJson = JSON.parse(
       (await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString()
@@ -28,29 +28,36 @@ export class Readme {
       console.log(error);
     }
 
-    // Gather project context upfront to avoid token explosion from filesystem tool
-    const contextString = await projectContext.convertFilesToContext([
-      (await projectContext.gatherFiles()).smartfilePackageJSON,
-      (await projectContext.gatherFiles()).smartfilesReadme,
-      (await projectContext.gatherFiles()).smartfilesReadmeHints,
-      (await projectContext.gatherFiles()).smartfilesNpmextraJSON,
-      ...(await projectContext.gatherFiles()).smartfilesMod,
-    ]);
-
-    // Use DualAgentOrchestrator for readme generation
+    // Use DualAgentOrchestrator with filesystem tool for agent-driven exploration
     const readmeOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
       smartAiInstance: this.aiDocsRef.smartAiInstance,
       defaultProvider: 'openai',
+      maxIterations: 25,
+      maxResultChars: 15000, // Limit tool output to prevent token explosion
+      maxHistoryMessages: 20, // Limit history window
+      logPrefix: '[README]',
+      onProgress: (event) => logger.log(event.logLevel, event.logMessage),
       guardianPolicyPrompt: `
-You validate README generation.
+You validate README generation tool calls and outputs.
 
-APPROVE if:
+APPROVE tool calls for:
+- Reading any files within the project directory (package.json, ts/*.ts, readme.md, etc.)
+- Using tree to see project structure
+- Using glob to find source files
+- Listing directory contents
+
+REJECT tool calls for:
+- Reading files outside the project directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+For final README output, APPROVE if:
 - README follows proper markdown format
 - Contains Install and Usage sections
 - Code examples are correct TypeScript/ESM syntax
 - Documentation is comprehensive and helpful
 
-REJECT if:
+REJECT final output if:
 - README is incomplete or poorly formatted
 - Contains licensing information (added separately)
 - Uses CommonJS syntax instead of ESM
@@ -58,14 +65,25 @@ REJECT if:
 `,
     });
 
+    // Register scoped filesystem tool for agent exploration
+    readmeOrchestrator.registerScopedFilesystemTool(this.projectDir);
+
     await readmeOrchestrator.start();
 
     const readmeTaskPrompt = `
 You create markdown READMEs for npm projects. You only output the markdown readme.
 
-Analyze the project files provided below to understand the codebase, then generate a comprehensive README.
+PROJECT DIRECTORY: ${this.projectDir}
 
-The README should follow this template:
+Use the filesystem tool to explore the project and understand what it does:
+1. First, use tree to see the project structure (maxDepth: 3)
+2. Read package.json to understand the package name, description, and dependencies
+3. Read the existing readme.md if it exists (use it as a base, improve and expand)
+4. Read readme.hints.md if it exists (contains hints for documentation)
+5. Read key source files in ts/ directory to understand the API and implementation
+6. Focus on exported classes, interfaces, and functions
+
+Then generate a comprehensive README following this template:
 
 # Project Name
 [The name from package.json and description]
@@ -86,12 +104,6 @@ The README should follow this template:
   Don't include any licensing information. This will be added later.
   Avoid "in conclusion" statements.
 ]
-
-Here are the project files:
-
-${contextString}
-
-Generate the README based on these files.
 `;
 
     const readmeResult = await readmeOrchestrator.run(readmeTaskPrompt);
@@ -124,48 +136,58 @@ Generate the README based on these files.
     for (const subModule of Object.keys(subModules)) {
       logger.log('info', `Building readme for ${subModule}`);
 
+      const subModulePath = plugins.path.join(paths.cwd, subModule);
       const tspublishData = await plugins.fsInstance
-        .file(plugins.path.join(paths.cwd, subModule, 'tspublish.json'))
+        .file(plugins.path.join(subModulePath, 'tspublish.json'))
         .encoding('utf8')
         .read();
 
-      // Gather submodule context
-      const subModuleContext = new ProjectContext(plugins.path.join(paths.cwd, subModule));
-      let subModuleContextString = '';
-      try {
-        const subModuleFiles = await subModuleContext.gatherFiles();
-        subModuleContextString = await subModuleContext.convertFilesToContext([
-          subModuleFiles.smartfilePackageJSON,
-          subModuleFiles.smartfilesNpmextraJSON,
-          ...subModuleFiles.smartfilesMod,
-        ]);
-      } catch (e) {
-        // Submodule may not have all files, continue with what we have
-        logger.log('warn', `Could not gather full context for ${subModule}`);
-      }
-
-      // Create a new orchestrator for each submodule
+      // Create a new orchestrator with filesystem tool for each submodule
       const subModuleOrchestrator = new plugins.smartagent.DualAgentOrchestrator({
         smartAiInstance: this.aiDocsRef.smartAiInstance,
         defaultProvider: 'openai',
+        maxIterations: 20,
+        maxResultChars: 12000,
+        maxHistoryMessages: 15,
+        logPrefix: `[README:${subModule}]`,
+        onProgress: (event) => logger.log(event.logLevel, event.logMessage),
         guardianPolicyPrompt: `
 You validate README generation for submodules.
 
-APPROVE comprehensive, well-formatted markdown with ESM TypeScript examples.
+APPROVE tool calls for:
+- Reading any files within the submodule directory
+- Using tree to see structure
+- Using glob to find source files
+
+REJECT tool calls for:
+- Reading files outside the submodule directory
+- Writing, deleting, or modifying any files
+- Any destructive operations
+
+APPROVE final README if comprehensive, well-formatted markdown with ESM TypeScript examples.
 REJECT incomplete READMEs or those with licensing info.
 `,
       });
 
+      // Register scoped filesystem tool for the submodule directory
+      subModuleOrchestrator.registerScopedFilesystemTool(subModulePath);
+
       await subModuleOrchestrator.start();
 
       const subModulePrompt = `
 You create markdown READMEs for npm projects. You only output the markdown readme.
 SUB MODULE: ${subModule}
+SUB MODULE DIRECTORY: ${subModulePath}
 
 IMPORTANT: YOU ARE CREATING THE README FOR THIS SUB MODULE: ${subModule}
 The Sub Module will be published with:
 ${JSON.stringify(tspublishData, null, 2)}
 
+Use the filesystem tool to explore the submodule:
+1. Use tree to see the submodule structure
+2. Read package.json to understand the submodule
+3. Read source files in ts/ directory to understand the implementation
+
 Generate a README following the template:
 
 # Project Name
@@ -184,12 +206,6 @@ Generate a README following the template:
 ]
 
 Don't use \`\`\` at the beginning or end. Only for code blocks.
-
-Here are the submodule files:
-
-${subModuleContextString}
-
-Generate the README based on these files.
 `;
 
       const subModuleResult = await subModuleOrchestrator.run(subModulePrompt);
@@ -200,7 +216,7 @@ Generate the README based on these files.
           .replace(/^```markdown\n?/i, '')
           .replace(/\n?```$/i, '') + '\n' + legalInfo;
         await plugins.fsInstance
-          .file(plugins.path.join(paths.cwd, subModule, 'readme.md'))
+          .file(plugins.path.join(subModulePath, 'readme.md'))
           .encoding('utf8')
           .write(subModuleReadmeString);
         logger.log('success', `Built readme for ${subModule}`);