1.5.0

.gitea/workflows/default_nottags.yaml

@@ -0,0 +1,66 @@
+name: Default (not tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: code.foss.global/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+
+      - name: Run npm prepare
+        run: npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build

.gitea/workflows/default_tags.yaml

@@ -0,0 +1,124 @@
+name: Default (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: code.foss.global/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Release
+        run: |
+          npmci node install stable
+          npmci npm publish
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @ship.zone/npmci
+          npmci npm prepare
+
+      - name: Code quality
+        run: |
+          npmci command npm install -g typescript
+          npmci npm install
+
+      - name: Trigger
+        run: npmci trigger
+
+      - name: Build docs and upload artifacts
+        run: |
+          npmci node install stable
+          npmci npm install
+          pnpm install -g @git.zone/tsdoc
+          npmci command tsdoc
+        continue-on-error: true

.gitignore

@@ -15,8 +15,7 @@ node_modules/
 
 # builds
 dist/
-dist_web/
+dist_*/
-dist_serve/
-dist_ts_web/
 
 # custom
+**/.claude/settings.local.json

.gitlab-ci.yml

@@ -1,125 +0,0 @@
-# gitzone standard
-image: hosttoday/ht-docker-node:npmci
-
-cache:
-  paths:
-  - .npmci_cache/
-  key: "$CI_BUILD_STAGE"
-
-stages:
-- security
-- test
-- release
-- metadata
-
-# ====================
-# security stage
-# ====================
-mirror:
-  stage: security
-  script:
-  - npmci git mirror
-  tags:
-  - docker
-  - notpriv
-
-snyk:
-  stage: security
-  script:
-    - npmci npm prepare
-    - npmci command npm install -g snyk
-    - npmci command npm install --ignore-scripts
-    - npmci command snyk test
-  tags:
-  - docker
-  - notpriv
-
-# ====================
-# test stage
-# ====================
-
-testLTS:
-  stage: test
-  script:
-  - npmci npm prepare
-  - npmci node install lts
-  - npmci npm install
-  - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - docker
-  - notpriv
-    
-testSTABLE:
-  stage: test
-  script:
-  - npmci npm prepare
-  - npmci node install stable
-  - npmci npm install
-  - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - docker
-  - notpriv
-
-release:
-  stage: release
-  script:
-  - npmci node install stable
-  - npmci npm publish
-  only:
-  - tags
-  tags:
-  - docker
-  - notpriv
-
-# ====================
-# metadata stage
-# ====================
-codequality:
-  stage: metadata
-  image: docker:stable
-  allow_failure: true
-  services:
-    - docker:stable-dind
-  script:
-    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
-    - docker run
-        --env SOURCE_CODE="$PWD"
-        --volume "$PWD":/code
-        --volume /var/run/docker.sock:/var/run/docker.sock
-        "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
-  artifacts:
-    paths: [codeclimate.json]
-  tags:
-  - docker
-  - priv
-
-trigger:
-  stage: metadata
-  script:
-  - npmci trigger
-  only:
-  - tags
-  tags:
-  - docker
-  - notpriv
-
-pages:
-  image: hosttoday/ht-docker-node:npmci
-  stage: metadata
-  script:
-    - npmci command npm install -g typedoc typescript
-    - npmci npm prepare
-    - npmci npm install
-    - npmci command typedoc --module "commonjs" --target "ES2016" --out public/ ts/
-  tags:
-    - docker
-    - notpriv
-  only:
-    - tags
-  artifacts:
-    expire_in: 1 week
-    paths:
-    - public
-  allow_failure: true

.snyk

@@ -1,13 +0,0 @@
-# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.13.5
-# ignores vulnerabilities until expiry date; change duration by modifying expiry date
-ignore:
-  SNYK-JS-MARKED-174116:
-    - typedoc > marked:
-        reason: None given
-        expires: '2019-06-13T06:50:33.594Z'
-  'npm:shelljs:20140723':
-    - typedoc > shelljs:
-        reason: None given
-        expires: '2019-06-13T06:50:33.594Z'
-patch: {}

.vscode/launch.json

@@ -0,0 +1,11 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "command": "npm test",
+      "name": "Run npm test",
+      "request": "launch",
+      "type": "node-terminal"
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,26 @@
+{
+  "json.schemas": [
+    {
+      "fileMatch": ["/npmextra.json"],
+      "schema": {
+        "type": "object",
+        "properties": {
+          "npmci": {
+            "type": "object",
+            "description": "settings for npmci"
+          },
+          "gitzone": {
+            "type": "object",
+            "description": "settings for gitzone",
+            "properties": {
+              "projectType": {
+                "type": "string",
+                "enum": ["website", "element", "service", "npm", "wcc"]
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}

changelog.md

@@ -0,0 +1,129 @@
+# Changelog
+
+## 2025-05-14 - 1.5.0 - feat(docs)
+Update project metadata and documentation to reflect comprehensive AI-enhanced features and improved installation and usage instructions
+
+- Revised descriptions in package.json and npmextra.json to emphasize comprehensive documentation capabilities
+- Expanded README with detailed installation options and extended usage examples for both CLI and API-like integrations
+- Added new dependency (gpt-tokenizer) to support token counting for AI context building
+- Adjusted keywords to better reflect project functionalities such as commit message automation and context trimming
+
+## 2025-05-13 - 1.4.5 - fix(dependencies)
+Upgrade various dependency versions and update package manager configuration
+
+- Bump @git.zone/tsbuild from ^2.1.80 to ^2.3.2
+- Upgrade @push.rocks/tapbundle from ^5.0.23 to ^6.0.3
+- Update @types/node from ^22.8.1 to ^22.15.17
+- Bump @push.rocks/smartai from ^0.4.2 to ^0.5.4
+- Upgrade @push.rocks/smartlog from ^3.0.7 to ^3.0.9
+- Update typedoc from ^0.27.9 to ^0.28.4
+- Bump typescript from ^5.5.2 to ^5.8.3
+- Add packageManager field with pnpm@10.10.0 specification
+
+## 2025-02-25 - 1.4.4 - fix(dependencies)
+Update dependencies to latest versions
+
+- Updated '@push.rocks/smartai' from '^0.0.17' to '^0.4.2'
+- Updated 'typedoc' from '^0.26.1' to '^0.27.9'
+
+## 2025-01-14 - 1.4.3 - fix(aidocs_classes)
+Improve readme generation instructions to ensure correct markdown formatting.
+
+- Added guidance to avoid using backticks at the beginning and end of readme generation to prevent markdown issues.
+- Clarified that the output is directly written to readme.md and backticks should only be used for code blocks.
+
+## 2024-10-28 - 1.4.2 - fix(cli)
+Ensure async completion for aidoc readme and description generation
+
+- Added await statements for asynchronous methods buildReadme and buildDescription in the aidoc command.
+
+## 2024-10-28 - 1.4.1 - fix(readme)
+Correct async call to getModuleSubDirs in readme generation.
+
+- Fixed an issue with asynchronous handling in readme generation for submodules.
+- Ensured that getModuleSubDirs function is called with await to handle promises properly.
+
+## 2024-10-28 - 1.4.0 - feat(aidocs)
+Added support for building readmes for sub-modules in aidocs
+
+- Updated the `Readme` class to handle monorepo projects by generating readmes for sub-modules.
+- Integrated `tspublish` to identify sub-modules for readme generation.
+
+## 2024-06-24 - 1.3.12 - fix(aidocs)
+Fix changelog generation by handling leading newlines
+
+- Fixed handling of leading newlines in the changelog to ensure proper formatting.
+
+## 2024-06-23 - 1.3.11 - fix(core)
+Fixed new changelog formatting issue to retain consistent spacing.
+
+- Adjusted the new changelog generation to ensure consistent spacing for improved readability.
+
+## 2024-06-23 - 1.3.10 - fix(aidocs_classes)
+Fix changelog format to remove extra newline
+
+- Updated `ts/aidocs_classes/commit.ts` to fix the changelog format.
+
+## 2024-06-23 - 1.3.9 - fix(aidoc)
+Fix changelog generation by properly stripping markdown code fences
+
+- Corrected the changelog generation code to ensure markdown code fences are properly stripped.
+
+
+## 2024-06-23 - 1.3.8 - fix(changelog)
+Fix changelog generation by properly stripping markdown code fences
+
+- Corrected the changelog generation code to ensure markdown code fences are properly stripped.
+
+## 2024-06-23 - 1.3.7 - fix(aidoc)
+Update to include package-lock.json in uncommitted changes check
+
+- Modified the getUncommittedDiff method call in commit.ts to include package-lock.json along with pnpm-lock.yaml
+
+
+## 2024-06-23 - 1.3.6 - fix(commit)
+Fixed issue with retrieving uncommitted diffs in git repository
+
+- Revised logic to correctly handle uncommitted changes by using an array for `getUncommittedDiff` method
+- Ensured proper handling and representation of uncommitted changes in the output
+
+
+## 2024-06-23 - 1.3.5 - fix(aidocs_classes)
+Refactor and enhance changelog formatting
+
+- Updated the `commit.ts` file to improve the changelog formatting and ensure consistency.
+- Enhanced the changelog instructions to include summarizing messages for omitted commits.
+- Removed unnecessary console logging in `projectcontext.ts`.
+
+
+```markdown
+## 2024-06-23 - 1.3.3 - fix(aidocs_classes)
+Fix changelog formatting issue in commit class
+
+## 2024-06-23 - 1.3.2 - fix(aidocs_classes)
+Fix minor bugs and update dependencies in aidocs_classes
+
+## 2024-06-23 - 1.3.1 - fix(aidocs_classes)
+Fix typo in INextCommitObject interface and update date format in changelog generation.
+
+## 2024-06-23 - 1.3.0 - fix(aidocs_classes)
+Fix typo in INextCommitObject interface
+
+## 2024-06-23 - 1.2.4 - feat(core)
+Added smarttime dependency and improved changelog generation
+
+## 2024-06-23 - 1.2.3 - fix(logging)
+Refactor logger initialization to use commitinfo data
+
+## 2024-06-23 - 1.2.2 - fix(aidocs)
+Fix bug in AiDoc class causing undefined token handling
+
+## 2024-06-23 - 1.2.0 - fix(core)
+Fixed usage of plugins in project context and readme generation
+
+## 2024-06-23 - 1.1.42 - feat(aidocs_classes)
+Enhance changelog generation by supporting complete generation in the absence of previous changelog files
+
+## 2024-06-23 - 1.1.41 - fix(aidocs_classes)
+Improve commit message generation by handling empty diffs and updating changelog instructions
+```

cli.child.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+process.env.CLI_CALL = 'true';
+import * as cliTool from './ts/index.js';
+cliTool.runCli();

cli.js

@@ -1,3 +1,4 @@
 #!/usr/bin/env node
 process.env.CLI_CALL = 'true';
-require('./dist/index');
+const cliTool = await import('./dist_ts/index.js');
+cliTool.runCli();

cli.ts.js

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
 process.env.CLI_CALL = 'true';
-require('@gitzone/tsrun');
+
-require('./ts/index');
+import * as tsrun from '@git.zone/tsrun';
+tsrun.runPath('./cli.child.js', import.meta.url);

npmextra.json

@@ -1,17 +1,35 @@
 {
   "gitzone": {
+    "projectType": "npm",
     "module": {
       "githost": "gitlab.com",
       "gitscope": "gitzone",
       "gitrepo": "tsdoc",
       "shortDescription": "a tool for better documentation",
-      "npmPackagename": "@gitzone/tsdoc",
+      "npmPackagename": "@git.zone/tsdoc",
       "license": "MIT",
-      "projectDomain": "git.zone"
+      "projectDomain": "git.zone",
+      "description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
+      "keywords": [
+        "TypeScript",
+        "documentation",
+        "AI",
+        "CLI",
+        "README",
+        "TypeDoc",
+        "commit messages",
+        "automation",
+        "code analysis",
+        "context trimming",
+        "developer tools"
+      ]
     }
   },
   "npmci": {
     "npmGlobalTools": [],
     "npmAccessLevel": "public"
+  },
+  "tsdoc": {
+    "legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH  \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
   }
 }

package.json

@@ -1,37 +1,85 @@
 {
-  "name": "@gitzone/tsdoc",
+  "name": "@git.zone/tsdoc",
-  "version": "1.0.15",
+  "version": "1.5.0",
   "private": false,
-  "description": "a tool for better documentation",
+  "description": "A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.",
-  "main": "dist/index.js",
+  "type": "module",
-  "typings": "dist/index.d.ts",
+  "exports": {
-  "author": "Lossless GmbH",
+    ".": "./dist_ts/index.js"
+  },
+  "author": "Task Venture Capital GmbH",
   "license": "MIT",
   "bin": {
     "tsdoc": "cli.js"
   },
   "scripts": {
-    "test": "(tstest test/) && (node ./cli.ts.js) && rm -rf public/",
+    "test": "(tstest test/) && npm run testCli",
-    "build": "(tsbuild)",
+    "testCli": "(node ./cli.ts.js) && (node ./cli.ts.js aidocs)",
-    "buildMkdocs": "(cd mkdocs/originalrepo && docker rmi -f mkdocs && docker build -t mkdocs .)",
+    "build": "(tsbuild --web --allowimplicitany)",
-    "format": "(gitzone format)"
+    "buildDocs": "tsdoc"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.0.22",
+    "@git.zone/tsbuild": "^2.3.2",
-    "@gitzone/tstest": "^1.0.22",
+    "@git.zone/tsrun": "^1.2.46",
-    "@pushrocks/tapbundle": "^3.0.7",
+    "@git.zone/tstest": "^1.0.90",
-    "@types/node": "^12.0.2",
+    "@push.rocks/tapbundle": "^6.0.3",
-    "tslint": "^5.11.0",
+    "@types/node": "^22.15.17"
-    "tslint-config-prettier": "^1.15.0"
   },
   "dependencies": {
-    "@pushrocks/early": "^3.0.3",
+    "@git.zone/tspublish": "^1.5.5",
-    "@pushrocks/smartcli": "^3.0.7",
+    "@push.rocks/early": "^4.0.3",
-    "@pushrocks/smartfile": "^7.0.2",
+    "@push.rocks/npmextra": "^5.0.23",
-    "@pushrocks/smartlog": "^2.0.19",
+    "@push.rocks/qenv": "^6.0.5",
-    "@pushrocks/smartlog-destination-local": "^7.0.5",
+    "@push.rocks/smartai": "^0.5.4",
-    "@pushrocks/smartshell": "^2.0.20",
+    "@push.rocks/smartcli": "^4.0.11",
-    "typedoc": "^0.14.2",
+    "@push.rocks/smartdelay": "^3.0.5",
-    "typescript": "^3.4.5"
+    "@push.rocks/smartfile": "^11.0.20",
-  }
+    "@push.rocks/smartgit": "^3.1.0",
+    "@push.rocks/smartinteract": "^2.0.15",
+    "@push.rocks/smartlog": "^3.0.9",
+    "@push.rocks/smartlog-destination-local": "^9.0.2",
+    "@push.rocks/smartpath": "^5.0.18",
+    "@push.rocks/smartshell": "^3.0.5",
+    "@push.rocks/smarttime": "^4.0.6",
+    "gpt-tokenizer": "^2.9.0",
+    "typedoc": "^0.28.4",
+    "typescript": "^5.8.3"
+  },
+  "files": [
+    "ts/**/*",
+    "ts_web/**/*",
+    "dist/**/*",
+    "dist_*/**/*",
+    "dist_ts/**/*",
+    "dist_ts_web/**/*",
+    "assets/**/*",
+    "cli.js",
+    "npmextra.json",
+    "readme.md"
+  ],
+  "browserslist": [
+    "last 1 chrome versions"
+  ],
+  "keywords": [
+    "TypeScript",
+    "documentation",
+    "AI",
+    "CLI",
+    "README",
+    "TypeDoc",
+    "commit messages",
+    "automation",
+    "code analysis",
+    "context trimming",
+    "developer tools"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/gitzone/tsdoc.git"
+  },
+  "bugs": {
+    "url": "https://gitlab.com/gitzone/tsdoc/issues"
+  },
+  "homepage": "https://gitlab.com/gitzone/tsdoc#readme",
+  "packageManager": "pnpm@10.10.0+sha512.d615db246fe70f25dcfea6d8d73dee782ce23e2245e3c4f6f888249fb568149318637dca73c2c5c8ef2a4ca0d5657fb9567188bfab47f566d1ee6ce987815c39"
 }

readme.hints.md

@@ -0,0 +1,5 @@
+* module needs to be installed globally
+* alternatively can be used through npx, if installed locally
+* cli parameters are concluded from ./ts/cli.ts
+* this module is not intended for API use.
+* Read carefully through the TypeScript files. Don't make stuff up.

readme.md

@@ -1,26 +1,742 @@
-# @gitzone/tsdoc
+# @git.zone/tsdoc  
-a tool for better documentation
+An advanced TypeScript documentation tool using AI to generate and enhance documentation for TypeScript projects.
 
-## Availabililty and Links
+## Install
-* [npmjs.org (npm package)](https://www.npmjs.com/package/@gitzone/tsdoc)
-* [gitlab.com (source)](https://gitlab.com/gitzone/tsdoc)
-* [github.com (source mirror)](https://github.com/gitzone/tsdoc)
-* [docs (typedoc)](https://gitzone.gitlab.io/tsdoc/)
 
-## Status for master
+To install @git.zone/tsdoc, you have two options. You can install it globally so that the CLI commands are available throughout your system, or you can use it with npx if you prefer to keep the installation local to your project.
-[![build status](https://gitlab.com/gitzone/tsdoc/badges/master/build.svg)](https://gitlab.com/gitzone/tsdoc/commits/master)
+
-[![coverage report](https://gitlab.com/gitzone/tsdoc/badges/master/coverage.svg)](https://gitlab.com/gitzone/tsdoc/commits/master)
+### Global Installation
-[![npm downloads per month](https://img.shields.io/npm/dm/@gitzone/tsdoc.svg)](https://www.npmjs.com/package/@gitzone/tsdoc)
+
-[![Known Vulnerabilities](https://snyk.io/test/npm/@gitzone/tsdoc/badge.svg)](https://snyk.io/test/npm/@gitzone/tsdoc)
+To install the tool globally, run the following command in your terminal:
-[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
+
-[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
+```bash
-[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/)
+npm install -g @git.zone/tsdoc
+```
+
+Installing globally ensures that the CLI commands (such as tsdoc, tsdoc typedoc, tsdoc aidoc, etc.) are available anywhere on your machine without the need to refer to the local node_modules folder.
+
+### Usage with npx
+
+If you prefer not to install the tool globally, you can invoke it using npx directly from your project. This method works well if you intend to use the tool on a per-project basis:
+
+```bash
+npx @git.zone/tsdoc <command>
+```
+
+In the commands below, you will see how to use the various functionalities that @git.zone/tsdoc provides for generating intricate and enhanced documentation for your TypeScript projects.
 
 ## Usage
 
-For further information read the linked docs at the top of this readme.
+The @git.zone/tsdoc module provides a very rich and interactive CLI interface together with a set of programmable classes that let you integrate documentation generation into your build processes or workflows. This section will walk you through every aspect of the module—from its basic CLI commands to its advanced internal API usage. All examples provided below use ESM syntax with TypeScript and are designed to be comprehensive. Every code snippet is written so you can easily copy, paste, and adapt to your project. The following guide is divided into several sections covering every major feature, tool integration, and customization options available in the module.
 
-> MIT licensed | **&copy;** [Lossless GmbH](https://lossless.gmbh)
+-------------------------------------------------------------------
-| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy.html)
+### Overview and Core Concepts
 
-[![repo-footer](https://gitzone.gitlab.io/assets/repo-footer.svg)](https://maintainedby.lossless.com)
+At its heart, @git.zone/tsdoc is a CLI tool that blends classic documentation generation (using libraries such as TypeDoc) with AI-enhanced techniques. The tool reads your project files, uses a context builder to optimize file content based on token usage and configurable trimming strategies, and then leverages an AI engine to generate enhanced documentation. This complete solution is designed to integrate smoothly into your project pipeline.
+
+Key features include:
+- **Auto-detection of documentation format**: The CLI attempts to determine the best documentation strategy for your project.
+- **Support for TypeDoc generation**: Build TypeDoc-compatible documentation directly.
+- **AI-Enhanced Documentation (AiDoc)**: Generate a README and project description using artificial intelligence that analyzes your project’s code and context.
+- **Plugin Integration**: The module leverages a variety of plugins (smartfile, smartgit, smartcli, smartai, etc.) to streamline tasks such as file manipulation, CLI interaction, shell command execution, and logging.
+- **Context Trimming and Optimization**: To manage token usage (especially for AI input), the module includes advanced context-building strategies that trim and summarize code files intelligently.
+- **Robust Internal API**: While the primary user interface is through the CLI, the underlying classes (AiDoc, TypeDoc, Readme, etc.) can be used to build custom integrations or extend the tool’s functionality.
+
+Below, you will find detailed explanations along with ESM/TypeScript code examples for all core use cases.
+
+-------------------------------------------------------------------
+### Command-Line Interface (CLI) Usage
+
+The most common way to interact with @git.zone/tsdoc is via its command-line interface (CLI). The CLI is designed to auto-detect your project’s context and trigger the appropriate commands based on your needs. Below is a guide on how to use the CLI commands.
+
+#### Basic Invocation
+
+When you run the command without any arguments, the tool attempts to determine the appropriate documentation generation mode:
+
+```bash
+tsdoc
+```
+
+This will scan the project directory and attempt to detect whether your project follows a TypeDoc convention or if it would benefit from an AI-enhanced documentation build. The auto-detection logic uses the project context (for example, the presence of a ts directory or specific configuration files) to decide the best course of action.
+
+##### Example Scenario
+
+Imagine you have a TypeScript project with the following structure:
+
+├── package.json  
+├── ts/  
+│   └── index.ts  
+└── readme.hints.md  
+
+When you execute:
+
+```bash
+tsdoc
+```
+
+The tool will analyze the project directory, recognizing the ts/ folder, and it will route the command to the appropriate documentation generator, such as the TypeDoc generator if it detects valid structure.
+
+#### TypeDoc Command
+
+For projects that require a traditional documentation format, you can explicitly generate documentation using the TypeDoc integration:
+
+```bash
+tsdoc typedoc --publicSubdir docs
+```
+
+This command instructs the module to generate HTML documentation using TypeDoc, placing the output into a public directory (or a custom subdirectory as specified).
+
+**Inside a TypeScript file, this command can be mirrored by calling the TypeDoc class directly:**
+
+```typescript
+import { TypeDoc } from '@git.zone/tsdoc';
+import * as path from 'path';
+
+const cwd = process.cwd();
+const typeDocInstance = new TypeDoc(cwd);
+
+const compileDocumentation = async (): Promise<void> => {
+  try {
+    // Specify the output subdirectory for documentation
+    await typeDocInstance.compile({
+      publicSubdir: 'docs'
+    });
+    console.log('Documentation successfully generated using TypeDoc.');
+  } catch (error) {
+    console.error('Error generating documentation with TypeDoc:', error);
+  }
+};
+
+compileDocumentation();
+```
+
+In this example, the script creates an instance of the TypeDoc class passing the current working directory. The compile method is then called with an options object, indicating that the public subdirectory should be named “docs.” The method spawns a shell command using the smart shell plugin to execute the TypeDoc binary.
+
+#### AI-Enhanced Documentation Command
+
+One of the standout features of this module is its AI-enhanced documentation capabilities. The `aidoc` command integrates with an OpenAI interface to produce a more contextual and detailed README and project description. This is particularly useful when your project codebase has evolved and requires documentation updates based on the current source code.
+
+To run the AI-enhanced documentation generation:
+
+```bash
+tsdoc aidoc
+```
+
+In an ESM/TypeScript project, you can use the AiDoc class to programmatically run the same functionality:
+
+```typescript
+import { AiDoc } from '@git.zone/tsdoc';
+
+const buildEnhancedDocs = async (): Promise<void> => {
+  const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
+  try {
+    // Start the AI interface; this internally checks if the token is valid and persists it
+    await aiDoc.start();
+
+    // Build the README file for the project directory
+    console.log('Generating README file using AI...');
+    await aiDoc.buildReadme(process.cwd());
+
+    // Build a new project description based on the codebase
+    console.log('Generating updated project description...');
+    await aiDoc.buildDescription(process.cwd());
+
+    console.log('AI-enhanced documentation generated successfully.');
+  } catch (error) {
+    console.error('Failed to generate AI-enhanced documentation:', error);
+  }
+};
+
+buildEnhancedDocs();
+```
+
+In the above snippet, we import the AiDoc class and create an instance with an OpenAI token. The methods start(), buildReadme(), and buildDescription() streamline the process of generating enhanced documentation by leveraging the underlying AI engine. This code example should serve as a blueprint for those wishing to integrate AI-driven documentation updates as part of their CI/CD pipelines.
+
+#### Testing Your Documentation Setup
+
+Before you commit changes to your project documentation, it is often worthwhile to run tests to ensure that your documentation generation process is behaving as expected. The module includes a `test` command:
+
+```bash
+tsdoc test
+```
+
+This command verifies that all components (CLI commands, TypeDoc compilation, AI integration, etc.) are properly configured.
+
+Here is an example test script written in TypeScript using a test bundle:
+
+```typescript
+import { expect, tap } from '@push.rocks/tapbundle';
+import { AiDoc } from '@git.zone/tsdoc';
+
+tap.test('AiDoc instance creation', async () => {
+  const aidoc = new AiDoc({ OPENAI_TOKEN: 'dummy-token' });
+  expect(aidoc).toBeInstanceOf(AiDoc);
+});
+
+tap.test('Running AI documentation generation', async () => {
+  const aidoc = new AiDoc({ OPENAI_TOKEN: 'dummy-token' });
+  await aidoc.start();
+  
+  // Attempt buildReadme and buildDescription synchronously for test coverage
+  await aidoc.buildReadme(process.cwd());
+  await aidoc.buildDescription(process.cwd());
+  
+  // If no errors are thrown, we assume the process works as expected
+  expect(true).toBe(true);
+});
+
+tap.start();
+```
+
+This test script demonstrates how to automate the validation process by using the provided AiDoc class. Using a testing framework like tap ensures that your documentation generation remains robust even as new features are added or as the project evolves.
+
+-------------------------------------------------------------------
+### Advanced Usage Scenarios
+
+Beyond using the CLI, @git.zone/tsdoc provides various classes and plugins that allow you to deeply integrate documentation generation within your project. The following sections document advanced usage scenarios where you programmatically interact with different components.
+
+#### 1. Deep Dive into AiDoc Functionality
+
+The AiDoc class is the core of the AI-enhanced documentation generation. It manages interactions with the OpenAI API, handles token validations, and integrates with project-specific configurations.
+
+Consider the following advanced usage example:
+
+```typescript
+import { AiDoc } from '@git.zone/tsdoc';
+import * as path from 'path';
+
+const generateProjectDocs = async () => {
+  // Create an instance of the AiDoc class with a configuration object 
+  // that includes your OpenAI token. This token will be used to query the AI.
+  const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
+
+  // Initialize the AI documentation system.
+  await aiDoc.start();
+
+  // Build the README file for the current project.
+  console.log('Building README for the project...');
+  await aiDoc.buildReadme(process.cwd());
+
+  // Build an updated project description based on the analysis of the source files.
+  console.log('Building project description...');
+  await aiDoc.buildDescription(process.cwd());
+
+  // You can also generate a commit message based on code changes by using the next commit object generation.
+  try {
+    console.log('Generating commit message based on your project changes...');
+    const nextCommit = await aiDoc.buildNextCommitObject(process.cwd());
+    console.log('Next commit message object:', nextCommit);
+  } catch (error) {
+    console.error('Error generating commit message:', error);
+  }
+};
+
+generateProjectDocs();
+```
+
+In this example, the AiDoc class handles multiple tasks:
+
+- It starts by validating and printing the sanitized token.
+- It generates and writes the README file based on dynamic analysis.
+- It updates the project description stored in your configuration files.
+- It even integrates with Git to produce a suggested commit message that factors in the current state of the project directory.
+
+Internally, methods such as buildReadme() interact with the ProjectContext class to gather files and determine the relevant context. This context is trimmed and processed based on token budgets, thus ensuring that the AI interface only receives the information it can effectively process.
+
+#### 2. Interacting with the TypeDoc Class Programmatically
+
+The TypeDoc class does not merely wrap the standard TypeDoc tool. It adds a layer of automation by preparing the TypeScript environment, generating a temporary tsconfig file, and invoking TypeDoc with the proper configuration. You can use this functionality to conditionally generate documentation or integrate it into your build steps.
+
+Below is another example demonstrating the integration:
+
+```typescript
+import { TypeDoc } from '@git.zone/tsdoc';
+import * as path from 'path';
+
+const generateTypeDocDocs = async () => {
+  // Assume you are in the root directory of your TypeScript project
+  const cwd = process.cwd();
+
+  // Create an instance of the TypeDoc class
+  const typeDocInstance = new TypeDoc(cwd);
+
+  // Prepare additional options if necessary (e.g., setting a public subdirectory for docs)
+  const options = { publicSubdir: 'documentation' };
+
+  // Compile your TypeScript project documentation.
+  // The compile method handles creating the tsconfig file, running the shell command, and cleaning up afterward.
+  try {
+    console.log('Compiling TypeScript documentation using TypeDoc...');
+    await typeDocInstance.compile(options);
+    console.log('Documentation generated at:', path.join(cwd, 'public', 'documentation'));
+  } catch (error) {
+    console.error('Error compiling TypeDoc documentation:', error);
+  }
+};
+
+generateTypeDocDocs();
+```
+
+This script clearly demonstrates how TypeDoc automation is structured inside the module. By invoking the compile() method, the class takes care of setting directory paths, preparing command arguments, and executing the underlying TypeDoc binary using the smart shell plugin.
+
+#### 3. Customizing Context Building
+
+One of the critical functionalities within @git.zone/tsdoc is its ability to build a smart context for documentation generation. The module not only collects file content from your project (like package.json, readme.hints.md, and other source files) but also intelligently trims and summarizes these contents to fit within token limits for AI processing.
+
+Consider the following deep-dive example into the context building process:
+
+```typescript
+import { EnhancedContext } from '@git.zone/tsdoc';
+import { ConfigManager } from '@git.zone/tsdoc/dist/ts/context/config-manager.js';
+
+const buildProjectContext = async () => {
+  const projectDir = process.cwd();
+
+  // Create an instance of the EnhancedContext class to optimize file content for AI use.
+  const enhancedContext = new EnhancedContext(projectDir);
+
+  // Initialize the context builder. This ensures that configuration (e.g., token budgets, trimming options) is loaded.
+  await enhancedContext.initialize();
+
+  // Optionally, you can choose to set a custom token budget and context mode.
+  enhancedContext.setTokenBudget(100000); // for example, limit tokens to 100K
+  enhancedContext.setContextMode('trimmed');
+
+  // Build the context string from selected files in your project.
+  const contextResult = await enhancedContext.buildContext('readme');
+
+  console.log('Context generated with token count:', contextResult.tokenCount);
+  console.log('Token savings due to trimming:', contextResult.tokenSavings);
+
+  // The context string includes file boundaries and token information.
+  console.log('Generated Context:', contextResult.context);
+};
+
+buildProjectContext();
+```
+
+In this example:
+- The EnhancedContext class is initialized with the project directory.
+- Configuration is loaded via the ConfigManager, which reads parameters from npmextra.json.
+- The context builder then gathers files (such as package.json, readme hints, TypeScript sources, etc.), trims unnecessary content, and builds a context string.
+- Finally, it prints the overall token counts and savings, giving you valuable feedback on how the context was optimized for the AI input.
+
+This detailed context-building mechanism is essential for managing large TypeScript projects. It ensures that the AI engine can process relevant code data without being overwhelmed by too many tokens.
+
+#### 4. Working with the Readme Class for Automatic README Generation
+
+The Readme class in @git.zone/tsdoc takes the AI-enhanced documentation further by not only generating a project-level README but also iterating over submodules within your project. This ensures that every published module has its own complete, AI-generated README.
+
+Here is an advanced example demonstrating how to trigger README generation for the main project and its submodules:
+
+```typescript
+import { AiDoc } from '@git.zone/tsdoc';
+import * as path from 'path';
+import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
+import { readFileSync } from 'fs';
+
+const buildProjectReadme = async () => {
+  const projectDir = process.cwd();
+
+  // Create an instance of the AiDoc class to handle AI-enhanced docs generation
+  const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
+
+  // Start the AI interface
+  await aiDoc.start();
+
+  // Build the primary README for the project directory
+  console.log('Generating primary README...');
+  await aiDoc.buildReadme(projectDir);
+
+  // Logging function to verify submodule processing
+  logger.log('info', `Primary README generated in ${projectDir}`);
+
+  // Assume that submodules are organized in distinct directories. 
+  // Here we simulate the process of scanning subdirectories and triggering README generation for each.
+  const subModules = ['submodule1', 'submodule2'];
+
+  // Loop through each submodule directory to generate its README.
+  for (const subModule of subModules) {
+    const subModuleDir = path.join(projectDir, subModule);
+    logger.log('info', `Generating README for submodule: ${subModule}`);
+
+    // Each submodule README is generated independently.
+    await aiDoc.buildReadme(subModuleDir);
+
+    // Optionally, read the generated README content for verification.
+    const readmePath = path.join(subModuleDir, 'readme.md');
+    try {
+      const readmeContent = readFileSync(readmePath, 'utf8');
+      logger.log('info', `Generated README for ${subModule}:\n${readmeContent.substring(0, 200)}...`);
+    } catch (error) {
+      logger.log('error', `Failed to read README for ${subModule}: ${error}`);
+    }
+  }
+};
+
+buildProjectReadme();
+```
+
+In this example, the script:
+- Starts by building the AI-enhanced README for the entire project.
+- Then iterates over a list of submodule directories and generates READMEs for each.
+- Uses the logging utility to provide immediate feedback on the generation process.
+- Optionally, reads back a snippet of the generated file to verify successful documentation generation.
+
+This approach ensures that projects with multiple submodules or packages maintain a consistent and high-quality documentation standard across every component.
+
+-------------------------------------------------------------------
+### Plugin-Based Architecture and Integrations
+
+Under the hood, @git.zone/tsdoc leverages a number of smaller, focused plugins that extend its functionality. These plugins facilitate file system operations, shell command execution, environment variable management, and logging. The modular design makes it easy to extend or customize the tool according to your needs.
+
+The relevant plugins include:
+- smartai: Provides the API integration with OpenAI.
+- smartcli: Handles CLI input parsing and command setup.
+- smartdelay: Manages asynchronous delays and debouncing.
+- smartfile: Offers an abstraction over file I/O operations.
+- smartgit: Facilitates integration with git repositories (e.g., retrieving diffs, commit status).
+- smartinteract: Eases interaction with the user (prompting for tokens, confirming actions).
+- smartlog and smartlogDestinationLocal: Provide comprehensive logging mechanisms.
+- smartpath, smartshell, and smarttime: Manage file paths, execute shell commands, and process time data respectively.
+
+Below is a sample snippet illustrating how you might directly interact with a few of these plugins to, for example, run a custom shell command or log events:
+
+```typescript
+import * as plugins from '@git.zone/tsdoc/dist/ts/plugins.js';
+import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
+
+const runCustomCommand = async () => {
+  // Create an instance of the smart shell utility
+  const smartshellInstance = new plugins.smartshell.Smartshell({
+    executor: 'bash',
+    pathDirectories: [plugins.smartpath.join(process.cwd(), 'node_modules/.bin')]
+  });
+
+  try {
+    // Execute a sample shell command, e.g., listing files in the current directory
+    const output = await smartshellInstance.exec('ls -la');
+    logger.log('info', `Shell command output:\n${output}`);
+  } catch (error) {
+    logger.log('error', 'Error executing custom shell command:', error);
+  }
+};
+
+runCustomCommand();
+```
+
+This example shows how to:
+- Import the necessary plugins.
+- Set up a smartshell instance by specifying the shell executor and the directories where executables are located.
+- Execute a command (in this case, listing directory contents) and log the results using the provided logging plugin.
+
+Such examples demonstrate the flexibility provided by the module’s internal API. They also illustrate that even if you choose not to use the CLI, you can still leverage the @git.zone/tsdoc functionality programmatically in a highly integrated fashion.
+
+-------------------------------------------------------------------
+### Handling Git Commit Messages with AI
+
+A unique feature of this tool is its capacity to assist with creating smart commit messages based on code changes. The Commit class (found within the aidocs_classes directory) ties together output from smartgit and AiDoc to suggest commit messages that are both descriptive and formatted according to conventional commit guidelines.
+
+Consider this example where you generate a commit message based on the diff from your git repository:
+
+```typescript
+import { AiDoc } from '@git.zone/tsdoc';
+
+const generateCommitMessage = async () => {
+  // Create an instance of AiDoc
+  const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
+
+  // Initialize the AI service
+  await aiDoc.start();
+
+  try {
+    // Generate and retrieve the next commit object based on the uncommitted changes in the repository.
+    const commitObject = await aiDoc.buildNextCommitObject(process.cwd());
+    console.log('Recommended commit object:', commitObject);
+
+    // The commit object is structured with the following fields:
+    // - recommendedNextVersionLevel: Indicates whether the commit is a fix, feature, or breaking change.
+    // - recommendedNextVersionScope: The scope of changes.
+    // - recommendedNextVersionMessage: A short commit message.
+    // - recommendedNextVersionDetails: A list of details explaining the changes.
+    // - recommendedNextVersion: A computed version string.
+  } catch (error) {
+    console.error('Error generating commit message:', error);
+  }
+};
+
+generateCommitMessage();
+```
+
+The process works as follows:
+1. The AiDoc instance is created and started.
+2. The tool uses the smartgit plugin to fetch uncommitted changes from the repository.
+3. It then builds a context string incorporating file diffs and project metadata.
+4. Finally, the OpenAI API is queried to produce a commit message formatted as JSON. This JSON object is parsed and can be used directly in your git workflow.
+
+This advanced integration assists teams in maintaining consistent commit message standards while reducing the manual burden of summarizing code changes.
+
+-------------------------------------------------------------------
+### Detailed Explanation of Internal Mechanics
+
+If you are curious about the intricate inner workings of @git.zone/tsdoc and wish to extend or debug its behavior, here is an in-depth explanation of some internal mechanisms.
+
+#### Context Trimming Strategy
+
+Managing token count is critical when interfacing with APIs that have strict limits. The module uses a multi-step process:
+- It gathers various files (such as package.json, ts files, readme hints).
+- It sorts the files and calculates the token count using the GPT tokenizer.
+- It applies trimming strategies such as removing function implementations or comments in TypeScript files, based on a configurable set of parameters.
+- Finally, it constructs a unified context string that includes file boundaries for clarity.
+
+For example, the ContextTrimmer class carries out these transformations:
+
+```typescript
+import { ContextTrimmer } from '@git.zone/tsdoc/dist/ts/context/context-trimmer.js';
+
+const trimFileContent = (filePath: string, content: string): string => {
+  // Create an instance of ContextTrimmer with default configuration
+  const trimmer = new ContextTrimmer({
+    removeImplementations: true,
+    preserveJSDoc: true,
+    maxFunctionLines: 5,
+    removeComments: true,
+    removeBlankLines: true
+  });
+  
+  // Trim the content based on the file type and configured options
+  const trimmedContent = trimmer.trimFile(filePath, content, 'trimmed');
+  return trimmedContent;
+};
+
+// Example usage with a TypeScript file
+const tsFileContent = `
+/**
+ * This function calculates the sum of two numbers.
+ */
+export const add = (a: number, b: number): number => {
+  // Calculation logic
+  return a + b;
+};
+`;
+
+const trimmedTSContent = trimFileContent('src/math.ts', tsFileContent);
+console.log('Trimmed TypeScript File Content:\n', trimmedTSContent);
+```
+
+This process helps in reducing the number of tokens before sending the data to the AI API while preserving the essential context needed for documentation generation.
+
+#### Dynamic Configuration Management
+
+The module’s configuration is stored in the npmextra.json file and includes settings for context building, trimming strategies, and task-specific options. The ConfigManager class reads these settings and merges them with default values. This dynamic configuration system ensures that the behavior of the documentation tool can be easily adjusted without altering the source code.
+
+```typescript
+import { ConfigManager } from '@git.zone/tsdoc/dist/ts/context/config-manager.js';
+
+const updateDocumentationConfig = async () => {
+  const projectDir = process.cwd();
+  const configManager = ConfigManager.getInstance();
+  
+  // Initialize the configuration manager with the project directory
+  await configManager.initialize(projectDir);
+  
+  // Retrieve the current configuration
+  let currentConfig = configManager.getConfig();
+  console.log('Current context configuration:', currentConfig);
+  
+  // If you want to change some parameters (e.g., maxTokens), update and then save the new configuration
+  const newConfig = { maxTokens: 150000 };
+  await configManager.updateConfig(newConfig);
+  
+  console.log('Configuration updated successfully.');
+};
+
+updateDocumentationConfig();
+```
+
+In this snippet, the ConfigManager:
+- Loads current configuration from npmextra.json.
+- Allows updates to specific keys (such as token limits).
+- Persists these changes back to the file system using the smartfile plugin.
+
+#### Logging and Diagnostic Output
+
+Throughout its execution, @git.zone/tsdoc logs important information such as token counts, file statistics, and shell command outputs. This logging is accomplished through a combination of the smartlog and smartlogDestinationLocal plugins. The following example illustrates how logging can help diagnose execution issues:
+
+```typescript
+import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
+
+const logDiagnosticInfo = () => {
+  logger.log('info', 'Starting documentation generation process...');
+  
+  // Log additional contextual information
+  logger.log('debug', 'Project directory:', process.cwd());
+  logger.log('debug', 'Token budget set for context building:', 150000);
+  
+  // Simulate a long-running process
+  setTimeout(() => {
+    logger.log('info', 'Documentation generation process completed successfully.');
+  }, 2000);
+};
+
+logDiagnosticInfo();
+```
+
+Using comprehensive logging, the tool provides feedback not only during normal execution but also in error scenarios, allowing developers to troubleshoot and optimize their documentation generation workflow.
+
+-------------------------------------------------------------------
+### Integrating @git.zone/tsdoc into a Continuous Integration Pipeline
+
+For teams looking to integrate documentation generation into their CI processes, @git.zone/tsdoc can be harnessed by scripting the CLI commands or by embedding the class-based API directly into your build scripts. Here’s an example of a CI script written in TypeScript that runs as part of a GitHub Action or similar workflow:
+
+```typescript
+import { runCli } from '@git.zone/tsdoc';
+import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
+
+const runDocumentationPipeline = async () => {
+  try {
+    logger.log('info', 'Starting the documentation pipeline...');
+    
+    // Run the CLI which automatically detects the project context and generates docs.
+    await runCli();
+    
+    logger.log('info', 'Documentation pipeline completed successfully.');
+  } catch (error) {
+    logger.log('error', 'Documentation pipeline encountered an error:', error);
+    process.exit(1);
+  }
+};
+
+runDocumentationPipeline();
+```
+
+In a CI environment, you can invoke this script to ensure that documentation is generated or updated as part of your deployment process. The process includes building the README, updating project descriptions, and generating TypeDoc documentation if the project structure warrants it.
+
+-------------------------------------------------------------------
+### Comprehensive Workflow Example
+
+Below is a full-fledged example that combines many of the above functionalities into a single workflow. This script is intended to be run as part of your build process or as a standalone command, and it demonstrates how to initialize all parts of the module, generate documentation for the main project and its submodules, update configuration, and log key diagnostics.
+
+```typescript
+import { AiDoc } from '@git.zone/tsdoc';
+import { TypeDoc } from '@git.zone/tsdoc';
+import { ConfigManager } from '@git.zone/tsdoc/dist/ts/context/config-manager.js';
+import { EnhancedContext } from '@git.zone/tsdoc/dist/ts/context/enhanced-context.js';
+import * as path from 'path';
+import { logger } from '@git.zone/tsdoc/dist/ts/logging.js';
+
+const runFullDocumentationWorkflow = async () => {
+  const projectDir = process.cwd();
+  
+  // Initialize configuration management
+  const configManager = ConfigManager.getInstance();
+  await configManager.initialize(projectDir);
+  logger.log('info', `Loaded configuration for project at ${projectDir}`);
+  
+  // Step 1: Generate conventional TypeDoc documentation
+  const typeDocInstance = new TypeDoc(projectDir);
+  try {
+    logger.log('info', 'Starting TypeDoc documentation generation...');
+    await typeDocInstance.compile({ publicSubdir: 'docs' });
+    logger.log('info', `TypeDoc documentation generated in ${path.join(projectDir, 'public', 'docs')}`);
+  } catch (error) {
+    logger.log('error', 'Error during TypeDoc generation:', error);
+  }
+  
+  // Step 2: Run AI-enhanced documentation generation
+  const aiDoc = new AiDoc({ OPENAI_TOKEN: 'your-openai-token' });
+  await aiDoc.start();
+  
+  // Generate main README and updated project description
+  try {
+    logger.log('info', 'Generating main README via AI-enhanced documentation...');
+    await aiDoc.buildReadme(projectDir);
+    logger.log('info', 'Main README generated successfully.');
+
+    logger.log('info', 'Generating updated project description...');
+    await aiDoc.buildDescription(projectDir);
+    logger.log('info', 'Project description updated successfully.');
+  } catch (error) {
+    logger.log('error', 'Error generating AI-enhanced documentation:', error);
+  }
+  
+  // Step 3: Generate contextual data using EnhancedContext
+  const enhancedContext = new EnhancedContext(projectDir);
+  await enhancedContext.initialize();
+  enhancedContext.setContextMode('trimmed');
+  enhancedContext.setTokenBudget(150000);
+
+  const contextResult = await enhancedContext.buildContext('readme');
+  logger.log('info', `Context built successfully. Total tokens: ${contextResult.tokenCount}. Savings: ${contextResult.tokenSavings}`);
+
+  // Step 4: Process submodules (if any) and generate READMEs
+  const subModules = ['submodule1', 'submodule2'];
+  for (const subModule of subModules) {
+    const subModuleDir = path.join(projectDir, subModule);
+    logger.log('info', `Processing submodule: ${subModule}`);
+    try {
+      await aiDoc.buildReadme(subModuleDir);
+      logger.log('info', `Submodule README generated for ${subModule}`);
+    } catch (error) {
+      logger.log('error', `Failed to generate README for ${subModule}:`, error);
+    }
+  }
+  
+  // Optional: Generate a commit message suggestion based on current changes
+  try {
+    logger.log('info', 'Generating commit message suggestion...');
+    const commitObject = await aiDoc.buildNextCommitObject(projectDir);
+    logger.log('info', 'Suggested commit message object:', commitObject);
+  } catch (error) {
+    logger.log('error', 'Error generating commit message suggestion:', error);
+  }
+  
+  logger.log('info', 'Full documentation workflow completed successfully.');
+};
+
+runFullDocumentationWorkflow();
+```
+
+This comprehensive workflow showcases the integration of various facets of the @git.zone/tsdoc module:
+- Loading and updating configuration via the ConfigManager.
+- Generating static documentation using TypeDoc.
+- Enhancing documentation with AI through the AiDoc class.
+- Optimizing project context with the EnhancedContext class.
+- Iterating over submodules to ensure all parts of your project are documented.
+- Providing useful diagnostic logging for every step.
+
+-------------------------------------------------------------------
+### Wrapping Up the Usage Guide
+
+The examples provided above demonstrate that @git.zone/tsdoc is not simply a CLI tool—it is a complete documentation framework designed to adapt to your workflow. Whether you are a developer looking to automate documentation updates in your CI pipeline or a team seeking an AI-powered enhancement for your project metadata, this module offers a wide range of interfaces and hooks for you to leverage.
+
+Key takeaways:
+- The CLI handles most routine tasks automatically while also exposing commands for specific documentation generation strategies.
+- Programmatic usage allows deep integration with your project’s build and commit processes.
+- The internal architecture—built on plugins, context optimization, and extensive logging—ensures that the tool can scale with project complexity.
+- Advanced users can customize context trimming, file inclusion rules, and even modify AI queries to better suit their project’s needs.
+
+Each code example provided here is written using modern ESM syntax and TypeScript to ensure compatibility with current development practices. Since the module is designed with extensibility in mind, developers are encouraged to explore the source code (especially the classes in the ts/ and ts/aidocs_classes directories) for further customization opportunities.
+
+By integrating @git.zone/tsdoc into your workflow, you ensure that your project documentation remains accurate, comprehensive, and reflective of your latest code changes—whether you are generating a simple README or a complex API documentation set enhanced by AI insights.
+
+Happy documenting!
+
+## License and Legal Information
+
+This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. 
+
+**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
+
+### Trademarks
+
+This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
+
+### Company Information
+
+Task Venture Capital GmbH  
+Registered at District court Bremen HRB 35230 HB, Germany
+
+For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
+
+By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

readme.plan.md

@@ -0,0 +1,314 @@
+# TSDocs Context Optimization Plan
+
+## Problem Statement
+
+For large TypeScript projects, the context generated for AI-based documentation creation becomes too large, potentially exceeding even o4-mini's 200K token limit. This affects the ability to effectively generate:
+
+- Project documentation (README.md)
+- API descriptions and keywords
+- Commit messages and changelogs
+
+Current implementation simply includes all TypeScript files and key project files, but lacks intelligent selection, prioritization, or content reduction mechanisms.
+
+## Analysis of Approaches
+
+### 1. Smart Content Selection
+
+**Description:** Intelligently select only files that are necessary for the specific task being performed, using heuristic rules.
+
+**Advantages:**
+- Simple to implement
+- Predictable behavior
+- Can be fine-tuned for different operations
+
+**Disadvantages:**
+- Requires manual tuning of rules
+- May miss important context in complex projects
+- Static approach lacks adaptability
+
+**Implementation Complexity:** Medium
+
+### 2. File Prioritization
+
+**Description:** Rank files by relevance using git history, file size, import/export analysis, and relationship to the current task.
+
+**Advantages:**
+- Adaptively includes the most relevant files first
+- Maintains context for frequently changed or central files
+- Can leverage git history for additional signals
+
+**Disadvantages:**
+- Complexity in determining accurate relevance scores
+- Requires analyzing project structure
+- May require scanning imports/exports for dependency analysis
+
+**Implementation Complexity:** High
+
+### 3. Chunking Strategy
+
+**Description:** Process the project in logical segments, generating intermediate results that are then combined to create the final output.
+
+**Advantages:**
+- Can handle projects of any size
+- Focused context for each specific part
+- May improve quality by focusing on specific areas deeply
+
+**Disadvantages:**
+- Complex orchestration of multiple AI calls
+- Challenge in maintaining consistency across chunks
+- May increase time and cost for processing
+
+**Implementation Complexity:** High
+
+### 4. Dynamic Context Trimming
+
+**Description:** Automatically reduce context by removing non-essential code while preserving structure. Techniques include:
+- Removing implementation details but keeping interfaces and type definitions
+- Truncating large functions while keeping signatures
+- Removing comments and whitespace (except JSDoc)
+- Keeping only imports/exports for context files
+
+**Advantages:**
+- Preserves full project structure
+- Flexible token usage based on importance
+- Good balance between completeness and token efficiency
+
+**Disadvantages:**
+- Potential to remove important implementation details
+- Risk of missing context needed for specific tasks
+- Complex rules for what to trim vs keep
+
+**Implementation Complexity:** Medium
+
+### 5. Embeddings-Based Retrieval
+
+**Description:** Create vector embeddings of project files and retrieve only the most relevant ones for a specific task using semantic similarity.
+
+**Advantages:**
+- Highly adaptive to different types of requests
+- Leverages semantic understanding of content
+- Can scale to extremely large projects
+
+**Disadvantages:**
+- Requires setting up and managing embeddings database
+- Added complexity of running vector similarity searches
+- Higher resource requirements for maintaining embeddings
+
+**Implementation Complexity:** Very High
+
+### 6. Task-Specific Contexts
+
+**Description:** Create separate optimized contexts for different tasks (readme, commit messages, etc.) with distinct file selection and processing strategies.
+
+**Advantages:**
+- Highly optimized for each specific task
+- Efficient token usage for each operation
+- Improved quality through task-focused contexts
+
+**Disadvantages:**
+- Maintenance of multiple context building strategies
+- More complex configuration
+- Potential duplication in implementation
+
+**Implementation Complexity:** Medium
+
+### 7. Recursive Summarization
+
+**Description:** Summarize larger files first, then include these summaries in the final context along with smaller files included in full.
+
+**Advantages:**
+- Can handle arbitrary project sizes
+- Preserves essential information from all files
+- Balanced approach to token usage
+
+**Disadvantages:**
+- Quality loss from summarization
+- Increased processing time from multiple AI calls
+- Complex orchestration logic
+
+**Implementation Complexity:** High
+
+## Implementation Strategy
+
+We propose a phased implementation approach, starting with the most impactful and straightforward approaches, then building toward more complex solutions as needed:
+
+### Phase 1: Foundation (1-2 weeks)
+
+1. **Implement Dynamic Context Trimming**
+   - Create a `ContextProcessor` class that takes SmartFile objects and applies trimming rules
+   - Implement configurable trimming rules (remove implementations, keep signatures)
+   - Add a configuration option to control trimming aggressiveness
+   - Support preserving JSDoc comments while removing other comments
+
+2. **Enhance Token Monitoring**
+   - Track token usage per file to identify problematic files
+   - Implement token budgeting to stay within limits
+   - Add detailed token reporting for optimization
+
+### Phase 2: Smart Selection (2-3 weeks)
+
+3. **Implement Task-Specific Contexts**
+   - Create specialized context builders for readme, commit messages, and descriptions
+   - Customize file selection rules for each task
+   - Add configuration options for task-specific settings
+
+4. **Add Smart Content Selection**
+   - Implement heuristic rules for file importance
+   - Create configuration for inclusion/exclusion patterns
+   - Add ability to focus on specific directories or modules
+
+### Phase 3: Advanced Techniques (3-4 weeks)
+
+5. **Implement File Prioritization**
+   - Add git history analysis to identify frequently changed files
+   - Implement dependency analysis to identify central files
+   - Create a scoring system for file relevance
+
+6. **Add Optional Recursive Summarization**
+   - Implement file summarization for large files
+   - Create a hybrid approach that mixes full files and summaries
+   - Add configuration to control summarization thresholds
+
+### Phase 4: Research-Based Approaches (Future Consideration)
+
+7. **Research and Evaluate Embeddings-Based Retrieval**
+   - Prototype embeddings creation for TypeScript files
+   - Evaluate performance and accuracy
+   - Implement if benefits justify the complexity
+
+8. **Explore Chunking Strategies**
+   - Research effective chunking approaches for documentation
+   - Prototype and evaluate performance
+   - Implement if benefits justify the complexity
+
+## Technical Design
+
+### Core Components
+
+1. **ContextBuilder** - Enhanced version of current ProjectContext
+   ```typescript
+   interface IContextBuilder {
+     buildContext(): Promise<string>;
+     getTokenCount(): number;
+     setContextMode(mode: 'normal' | 'trimmed' | 'summarized'): void;
+     setTokenBudget(maxTokens: number): void;
+     setPrioritizationStrategy(strategy: IPrioritizationStrategy): void;
+   }
+   ```
+
+2. **FileProcessor** - Handles per-file processing and trimming
+   ```typescript
+   interface IFileProcessor {
+     processFile(file: SmartFile): Promise<string>;
+     setProcessingMode(mode: 'full' | 'trim' | 'summarize'): void;
+     getTokenCount(): number;
+   }
+   ```
+
+3. **PrioritizationStrategy** - Ranks files by importance
+   ```typescript
+   interface IPrioritizationStrategy {
+     rankFiles(files: SmartFile[], context: string): Promise<SmartFile[]>;
+     setImportanceMetrics(metrics: IImportanceMetrics): void;
+   }
+   ```
+
+4. **TaskContextFactory** - Creates optimized contexts for specific tasks
+   ```typescript
+   interface ITaskContextFactory {
+     createContextForReadme(projectDir: string): Promise<string>;
+     createContextForCommit(projectDir: string, diff: string): Promise<string>;
+     createContextForDescription(projectDir: string): Promise<string>;
+   }
+   ```
+
+### Configuration Options
+
+The system will support configuration via a new section in `npmextra.json`:
+
+```json
+{
+  "tsdoc": {
+    "context": {
+      "maxTokens": 190000,
+      "defaultMode": "dynamic",
+      "taskSpecificSettings": {
+        "readme": {
+          "mode": "full",
+          "includePaths": ["src/", "lib/"],
+          "excludePaths": ["test/", "examples/"]
+        },
+        "commit": {
+          "mode": "trimmed",
+          "focusOnChangedFiles": true
+        },
+        "description": {
+          "mode": "summarized",
+          "includePackageInfo": true
+        }
+      },
+      "trimming": {
+        "removeImplementations": true,
+        "preserveInterfaces": true,
+        "preserveTypeDefs": true,
+        "preserveJSDoc": true,
+        "maxFunctionLines": 5
+      }
+    }
+  }
+}
+```
+
+## Cost-Benefit Analysis
+
+### Cost Considerations
+
+1. **Development costs**
+   - Initial implementation of foundational components (~30-40 hours)
+   - Testing and validation across different project sizes (~10-15 hours)
+   - Documentation and configuration examples (~5 hours)
+
+2. **Operational costs**
+   - Potential increased processing time for context preparation
+   - Additional API calls for summarization or embeddings approaches
+   - Monitoring and maintenance of the system
+
+### Benefits
+
+1. **Scalability**
+   - Support for projects of any size, up to and beyond o4-mini's 200K token limit
+   - Future-proof design that can adapt to different models and token limits
+
+2. **Quality improvements**
+   - More focused contexts lead to better AI outputs
+   - Task-specific optimization improves relevance
+   - Consistent performance regardless of project size
+
+3. **User experience**
+   - Predictable behavior for all project sizes
+   - Transparent token usage reporting
+   - Configuration options for different usage patterns
+
+## First Deliverable
+
+For immediate improvements, we recommend implementing Dynamic Context Trimming and Task-Specific Contexts first, as these offer the best balance of impact and implementation complexity.
+
+### Implementation Plan for Dynamic Context Trimming
+
+1. Create a basic `ContextTrimmer` class that processes TypeScript files:
+   - Remove function bodies but keep signatures
+   - Preserve interface and type definitions
+   - Keep imports and exports
+   - Preserve JSDoc comments
+
+2. Integrate with the existing ProjectContext class:
+   - Add a trimming mode option
+   - Apply trimming during the context building process
+   - Track and report token savings
+
+3. Modify the CLI to support trimming options:
+   - Add a `--trim` flag to enable trimming
+   - Add a `--trim-level` option for controlling aggressiveness
+   - Show token usage with and without trimming
+
+This approach could reduce token usage by 40-70% while preserving the essential structure of the codebase, making it suitable for large projects while maintaining high-quality AI outputs.

test/test.aidoc.nonci.ts

@@ -0,0 +1,39 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import * as qenv from '@push.rocks/qenv';
+let testQenv = new qenv.Qenv('./', '.nogit/');
+
+import * as tsdocs from '../ts/index.js';
+
+let aidocs: tsdocs.AiDoc;
+
+tap.test('should create an AIdocs class', async () => {
+  aidocs = new tsdocs.AiDoc({
+    OPENAI_TOKEN: await testQenv.getEnvVarOnDemand('OPENAI_TOKEN'),
+  });
+  expect(aidocs).toBeInstanceOf(tsdocs.AiDoc);
+});
+
+tap.test('should start AIdocs', async () => {
+  await aidocs.start();
+});
+
+tap.skip.test('should start AIdocs', async () => {
+  await aidocs.buildReadme('./');
+});
+
+tap.skip.test('should start AIdocs', async () => {
+  await aidocs.buildDescription('./');
+});
+
+tap.test('should build commit object', async () => {
+  const commitObject = await aidocs.buildNextCommitObject('./');
+  console.log(commitObject);
+  expect(commitObject).not.toBeUndefined();
+  expect(commitObject).toHaveProperty('recommendedNextVersion');
+  expect(commitObject).toHaveProperty('recommendedNextVersionLevel');
+  expect(commitObject).toHaveProperty('recommendedNextVersionScope');
+  expect(commitObject).toHaveProperty('recommendedNextVersionMessage');
+
+})
+
+tap.start();

test/test.ts

@@ -1,5 +1,5 @@
-import { expect, tap } from '@pushrocks/tapbundle';
+import { expect, tap } from '@push.rocks/tapbundle';
-import * as tsdoc from '../ts/index';
+import * as tsdoc from '../ts/index.js';
 
 tap.test('first test', async () => {
   console.log('test');

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @push.rocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@git.zone/tsdoc',
+  version: '1.5.0',
+  description: 'A comprehensive TypeScript documentation tool that leverages AI to generate and enhance project documentation, including dynamic README creation, API docs via TypeDoc, and smart commit message generation.'
+}

ts/aidocs_classes/commit.ts

@@ -0,0 +1,146 @@
+import * as plugins from '../plugins.js';
+import { AiDoc } from '../classes.aidoc.js';
+import { ProjectContext } from './projectcontext.js';
+
+export interface INextCommitObject {
+  recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project
+  recommendedNextVersionScope: string; // the recommended scope name of the next version, like "core" or "cli", or specific class names.
+  recommendedNextVersionMessage: string; // the commit message. Don't put fix() feat() or BREAKING CHANGE in the message. Please just the message itself.
+  recommendedNextVersionDetails: string[]; // detailed bullet points for the changelog
+  recommendedNextVersion: string; // the recommended next version of the project, x.x.x
+  changelog?: string; // the changelog for the next version
+}
+
+export class Commit {
+  private aiDocsRef: AiDoc;
+  private projectDir: string;
+
+  constructor(aiDocsRef: AiDoc, projectDirArg: string) {
+    this.aiDocsRef = aiDocsRef;
+    this.projectDir = projectDirArg;
+  }
+
+  public async buildNextCommitObject(): Promise<INextCommitObject> {
+    const smartgitInstance = new plugins.smartgit.Smartgit();
+    await smartgitInstance.init();
+    const gitRepo = await plugins.smartgit.GitRepo.fromOpeningRepoDir(
+      smartgitInstance,
+      this.projectDir
+    );
+    const diffStringArray = await gitRepo.getUncommittedDiff([
+      'pnpm-lock.yaml',
+      'package-lock.json',
+    ]);
+    // Use the new TaskContextFactory for optimized context
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    await taskContextFactory.initialize();
+    
+    // Generate context specifically for commit task
+    const contextResult = await taskContextFactory.createContextForCommit(
+      diffStringArray[0] ? diffStringArray.join('\n\n') : 'No changes.'
+    );
+    
+    // Get the optimized context string
+    let contextString = contextResult.context;
+    
+    // Log token usage statistics
+    console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
+    
+    // Check for token overflow against model limits
+    const MODEL_TOKEN_LIMIT = 200000; // o4-mini
+    if (contextResult.tokenCount > MODEL_TOKEN_LIMIT * 0.9) {
+      console.log(`⚠️ Warning: Context size (${contextResult.tokenCount} tokens) is close to or exceeds model limit (${MODEL_TOKEN_LIMIT} tokens).`);
+      console.log(`The model may not be able to process all information effectively.`);
+    }
+
+    let result = await this.aiDocsRef.openaiInstance.chat({
+      systemMessage: `
+You create a commit message for a git commit.
+The commit message should be based on the files in the project.
+You should not include any licensing information.
+You should not include any personal information.
+
+Important: Answer only in valid JSON.
+
+Your answer should be parseable with JSON.parse() without modifying anything.
+
+Here is the structure of the JSON you should return:
+
+interface {
+  recommendedNextVersionLevel: 'fix' | 'feat' | 'BREAKING CHANGE'; // the recommended next version level of the project
+  recommendedNextVersionScope: string; // the recommended scope name of the next version, like "core" or "cli", or specific class names.
+  recommendedNextVersionMessage: string; // the commit message. Don't put fix() feat() or BREAKING CHANGE in the message. Please just the message itself.
+  recommendedNextVersionDetails: string[]; // detailed bullet points for the changelog
+  recommendedNextVersion: string; // the recommended next version of the project, x.x.x
+}
+
+For the recommendedNextVersionDetails, please only add a detail entries to the array if it has an obvious value to the reader.
+
+You are being given the files of the project. You should use them to create the commit message.
+Also you are given a diff
+
+`,
+      messageHistory: [],
+      userMessage: contextString,
+    });
+
+    // console.log(result.message);
+    const resultObject: INextCommitObject = JSON.parse(
+      result.message.replace('```json', '').replace('```', '')
+    );
+
+    const previousChangelogPath = plugins.path.join(this.projectDir, 'changelog.md');
+    let previousChangelog: plugins.smartfile.SmartFile;
+    if (await plugins.smartfile.fs.fileExists(previousChangelogPath)) {
+      previousChangelog = await plugins.smartfile.SmartFile.fromFilePath(previousChangelogPath);
+    }
+
+    if (!previousChangelog) {
+      // lets build the changelog based on that
+      const commitMessages = await gitRepo.getAllCommitMessages();
+      console.log(JSON.stringify(commitMessages, null, 2));
+      let result2 = await this.aiDocsRef.openaiInstance.chat({
+        messageHistory: [],
+        systemMessage: `
+You are building a changelog.md file for the project.
+Omit commits and versions that lack relevant changes, but make sure to mention them as a range with a summarizing message instead.
+
+A changelog entry should look like this:
+
+    ## yyyy-mm-dd - x.x.x - scope here
+    main descriptiom here
+
+    - detailed bullet points follow
+
+You are given:
+* the commit messages of the project
+
+Only return the changelog file, so it can be written directly to changelog.md`,
+        userMessage: `
+Here are the commit messages:
+
+${JSON.stringify(commitMessages, null, 2)}
+  `,
+      });
+
+      previousChangelog = await plugins.smartfile.SmartFile.fromString(
+        previousChangelogPath,
+        result2.message.replaceAll('```markdown', '').replaceAll('```', ''),
+        'utf8'
+      );
+    }
+
+    let oldChangelog = previousChangelog.contents.toString().replace('# Changelog\n\n', '');
+    if (oldChangelog.startsWith('\n')) {
+      oldChangelog = oldChangelog.replace('\n', '');
+    }
+    let newDateString = new plugins.smarttime.ExtendedDate().exportToHyphedSortableDate();
+    let newChangelog = `# Changelog\n\n${`## ${newDateString} - {{nextVersion}} - {{nextVersionScope}}
+{{nextVersionMessage}}
+
+{{nextVersionDetails}}`}\n\n${oldChangelog}`;
+    resultObject.changelog = newChangelog;
+
+    return resultObject;
+  }
+}

ts/aidocs_classes/description.ts

@@ -0,0 +1,84 @@
+import type { AiDoc } from '../classes.aidoc.js';
+import * as plugins from '../plugins.js';
+import { ProjectContext } from './projectcontext.js';
+
+interface IDescriptionInterface {
+  description: string;
+  keywords: string[];
+}
+
+export class Description {
+  // INSTANCE
+  private aiDocsRef: AiDoc;
+  private projectDir: string;
+
+  constructor(aiDocsRef: AiDoc, projectDirArg: string) {
+    this.aiDocsRef = aiDocsRef;
+    this.projectDir = projectDirArg;
+  }
+
+  public async build() {
+    // Use the new TaskContextFactory for optimized context
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    await taskContextFactory.initialize();
+    
+    // Generate context specifically for description task
+    const contextResult = await taskContextFactory.createContextForDescription();
+    const contextString = contextResult.context;
+    
+    // Log token usage statistics
+    console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
+
+    let result = await this.aiDocsRef.openaiInstance.chat({
+      systemMessage: `
+You create a json adhering the following interface:
+{
+  description: string; // a sensible short, one sentence description of the project
+  keywords: string[]; // an array of tags that describe the project
+}
+
+The description should be based on what you understand from the project's files.
+The keywords should be based on use cases you see from the files.
+Don't be cheap about the way you think.
+
+Important: Answer only in valid JSON.
+You answer should be parseable with JSON.parse() without modifying anything.
+
+Don't wrap the JSON in three ticks json!!!
+    `,
+      messageHistory: [],
+      userMessage: contextString,
+    });
+
+    console.log(result.message);
+    const resultObject: IDescriptionInterface = JSON.parse(
+      result.message.replace('```json', '').replace('```', ''),
+    );
+
+    // Create a standard ProjectContext instance for file operations
+    const projectContext = new ProjectContext(this.projectDir);
+    const files = await projectContext.gatherFiles();
+    
+    const npmextraJson = files.smartfilesNpmextraJSON;
+    const npmextraJsonContent = JSON.parse(npmextraJson.contents.toString());
+
+    npmextraJsonContent.gitzone.module.description = resultObject.description;
+    npmextraJsonContent.gitzone.module.keywords = resultObject.keywords;
+
+    npmextraJson.contents = Buffer.from(JSON.stringify(npmextraJsonContent, null, 2));
+    await npmextraJson.write();
+
+    // do the same with packageJson
+    const packageJson = files.smartfilePackageJSON;
+    const packageJsonContent = JSON.parse(packageJson.contents.toString());
+    packageJsonContent.description = resultObject.description;
+    packageJsonContent.keywords = resultObject.keywords;
+    packageJson.contents = Buffer.from(JSON.stringify(packageJsonContent, null, 2));
+    await packageJson.write();
+
+    console.log(`\n======================\n`);
+    console.log(JSON.stringify(resultObject, null, 2));
+    console.log(`\n======================\n`);
+    return result.message;
+  }
+}

ts/aidocs_classes/index.ts

@@ -0,0 +1,4 @@
+export * from './commit.js';
+export * from './description.js';
+export * from './projectcontext.js';
+export * from './readme.js';

ts/aidocs_classes/projectcontext.ts

@@ -0,0 +1,127 @@
+import * as plugins from '../plugins.js';
+
+export class ProjectContext {
+  public static async fromDir(dirArg: string) {}
+
+  // INSTANCE
+  public projectDir: string;
+  private tokenCount: number = 0;
+  private contextString: string = '';
+
+  constructor(projectDirArg: string) {
+    this.projectDir = projectDirArg;
+  }
+
+  public async gatherFiles() {
+    const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(
+      plugins.path.join(this.projectDir, 'package.json'),
+      this.projectDir,
+    );
+    const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(
+      plugins.path.join(this.projectDir, 'readme.md'),
+      this.projectDir,
+    );
+
+    const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(
+      plugins.path.join(this.projectDir, 'readme.hints.md'),
+      this.projectDir,
+    );
+    const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(
+      plugins.path.join(this.projectDir, 'npmextra.json'),
+      this.projectDir,
+    );
+    const smartfilesMod = await plugins.smartfile.fs.fileTreeToObject(
+      this.projectDir,
+      'ts*/**/*.ts',
+    );
+    const smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(
+      this.projectDir,
+      'test/**/*.ts',
+    );
+    return {
+      smartfilePackageJSON,
+      smartfilesReadme,
+      smartfilesReadmeHints,
+      smartfilesNpmextraJSON,
+      smartfilesMod,
+      smartfilesTest,
+    };
+  }
+
+  public async convertFilesToContext(filesArg: plugins.smartfile.SmartFile[]) {
+    filesArg.map((fileArg) => {
+      // console.log(`  -> ${fileArg.relative}`);
+    });
+    return filesArg
+      .map((smartfile) => {
+        return `
+====== START OF FILE ${smartfile.relative} ======
+  
+${smartfile.contents.toString()}
+  
+====== END OF FILE ${smartfile.relative} ======
+        `;
+      })
+      .join('\n');
+  }
+
+  /**
+   * Calculate the token count for a string using the GPT tokenizer
+   * @param text The text to count tokens for
+   * @param model The model to use for token counting (default: gpt-3.5-turbo)
+   * @returns The number of tokens in the text
+   */
+  public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
+    try {
+      // Use the gpt-tokenizer library to count tokens
+      const tokens = plugins.gptTokenizer.encode(text);
+      return tokens.length;
+    } catch (error) {
+      console.error('Error counting tokens:', error);
+      // Provide a rough estimate (4 chars per token) if tokenization fails
+      return Math.ceil(text.length / 4);
+    }
+  }
+
+  private async buildContext(dirArg: string) {
+    const files = await this.gatherFiles();
+    let context = await this.convertFilesToContext([
+      files.smartfilePackageJSON,
+      files.smartfilesReadme,
+      files.smartfilesReadmeHints,
+      files.smartfilesNpmextraJSON,
+      ...files.smartfilesMod,
+      ...files.smartfilesTest,
+    ]);
+    // Count tokens in the context
+    this.contextString = context;
+    this.tokenCount = this.countTokens(context);
+    
+    // console.log(context);
+    return context;
+  }
+
+  /**
+   * Get the token count for the current context
+   * @returns The number of tokens in the context
+   */
+  public getTokenCount(): number {
+    return this.tokenCount;
+  }
+
+  /**
+   * Get both the context string and its token count
+   * @returns An object containing the context string and token count
+   */
+  public getContextWithTokenCount(): { context: string; tokenCount: number } {
+    return {
+      context: this.contextString,
+      tokenCount: this.tokenCount
+    };
+  }
+
+  public async update() {
+    const result = await this.buildContext(this.projectDir);
+    return result;
+  }
+}

ts/aidocs_classes/readme.ts

@@ -0,0 +1,152 @@
+import type { AiDoc } from '../classes.aidoc.js';
+import * as plugins from '../plugins.js';
+import * as paths from '../paths.js';
+import { ProjectContext } from './projectcontext.js';
+import { logger } from '../logging.js';
+
+export class Readme {
+  // INSTANCE
+  private aiDocsRef: AiDoc;
+  private projectDir: string;
+
+  constructor(aiDocsRef: AiDoc, projectDirArg: string) {
+    this.aiDocsRef = aiDocsRef;
+    this.projectDir = projectDirArg;
+  }
+
+  public async build() {
+    let finalReadmeString = ``;
+
+    // Use the new TaskContextFactory for optimized context
+    const taskContextFactory = new (await import('../context/index.js')).TaskContextFactory(this.projectDir);
+    await taskContextFactory.initialize();
+    
+    // Generate context specifically for readme task
+    const contextResult = await taskContextFactory.createContextForReadme();
+    const contextString = contextResult.context;
+    
+    // Log token usage statistics
+    console.log(`Token usage - Context: ${contextResult.tokenCount}, Files: ${contextResult.includedFiles.length + contextResult.trimmedFiles.length}, Savings: ${contextResult.tokenSavings}`);
+
+    // lets first check legal before introducung any cost
+    const projectContext = new ProjectContext(this.projectDir);
+    const npmExtraJson = JSON.parse(
+      (await projectContext.gatherFiles()).smartfilesNpmextraJSON.contents.toString()
+    );
+    const legalInfo = npmExtraJson?.tsdoc?.legal;
+    if (!legalInfo) {
+      const error = new Error(`No legal information found in npmextra.json`);
+      console.log(error);
+    }
+
+    let result = await this.aiDocsRef.openaiInstance.chat({
+      systemMessage: `
+You create markdown readmes for npm projects. You only output the markdown readme.
+
+The Readme should follow the following template:
+
+# Project Name
+[
+  The name is the module name of package.json
+  The description is in the description field of package.json
+]
+
+## Install
+[
+  Write a short text on how to install the project
+]
+
+## Usage
+[ 
+  Give code examples here.
+  Construct sensible scenarios for the user.
+  Make sure to show a complete set of features of the module.
+  Don't omit use cases.
+  It does not matter how much time you need.
+  ALWAYS USE ESM SYNTAX AND TYPESCRIPT.
+  DON'T CHICKEN OUT. Write at least 4000 words. More if necessary.
+  If there is already a readme, take the Usage section as base. Remove outdated content, and expand and improve upon the valid parts.
+  Super important: Check for completenes.
+  Don't include any licensing information. This will be added in a later step.
+  Avoid "in conclusions".
+
+  Good to know:
+  * npmextra.json contains overall module information.
+  * readme.hints.md provides valuable hints about module ideas.
+]
+            `,
+      messageHistory: [],
+      userMessage: contextString,
+    });
+
+    finalReadmeString += result.message + '\n' + legalInfo;
+
+    console.log(`\n======================\n`);
+    console.log(result.message);
+    console.log(`\n======================\n`);
+
+    const readme = (await projectContext.gatherFiles()).smartfilesReadme;
+    readme.contents = Buffer.from(finalReadmeString);
+    await readme.write();
+
+    // lets care about monorepo aspects
+    const tsPublishInstance = new plugins.tspublish.TsPublish();
+    const subModules = await tsPublishInstance.getModuleSubDirs(paths.cwd);
+    logger.log('info', `Found ${Object.keys(subModules).length} sub modules`);
+    for (const subModule of Object.keys(subModules)) {
+      logger.log('info', `Building readme for ${subModule}`);
+      const subModuleContextString = await projectContext.update();
+      let result = await this.aiDocsRef.openaiInstance.chat({
+        systemMessage: `
+  You create markdown readmes for npm projects. You only output the markdown readme.
+
+  IMPORTANT: YOU ARE NOW CREATING THE README FOR THE FOLLOWING SUB MODULE: ${subModule} !!!!!!!!!!!
+  The Sub Module will be published with the following data:
+  ${JSON.stringify(plugins.smartfile.fs.toStringSync(plugins.path.join(paths.cwd, subModule, 'tspublish.json')), null, 2)}
+
+  
+  The Readme should follow the following template:
+  
+  # Project Name
+  [
+    The name is the module name of package.json
+    The description is in the description field of package.json
+  ]
+  
+  ## Install
+  [
+    Write a short text on how to install the project
+  ]
+  
+  ## Usage
+  [ 
+    Give code examples here.
+    Construct sensible scenarios for the user.
+    Make sure to show a complete set of features of the module.
+    Don't omit use cases.
+    It does not matter how much time you need.
+    ALWAYS USE ESM SYNTAX AND TYPESCRIPT.
+    DON'T CHICKEN OUT. Write at least 4000 words. More if necessary.
+    If there is already a readme, take the Usage section as base. Remove outdated content, and expand and improve upon the valid parts.
+    Super important: Check for completenes.
+    Don't include any licensing information. This will be added in a later step.
+    Avoid "in conclusions".
+  
+    Good to know:
+    * npmextra.json contains overall module information.
+    * readme.hints.md provides valuable hints about module ideas.
+    * Your output lands directly in the readme.md file.
+    * Don't use \`\`\` at the beginning or the end. It'll cause problems. Only use it for codeblocks. You are directly writing markdown. No need to introduce it weirdly.
+  ]
+              `,
+        messageHistory: [],
+        userMessage: subModuleContextString,
+      });
+
+      const subModuleReadmeString = result.message + '\n' + legalInfo;
+      await plugins.smartfile.memory.toFs(subModuleReadmeString, plugins.path.join(paths.cwd, subModule, 'readme.md'));
+      logger.log('success', `Built readme for ${subModule}`);
+    }
+    return result.message;
+  }
+}

ts/classes.aidoc.ts

@@ -0,0 +1,130 @@
+import * as plugins from './plugins.js';
+
+import * as aiDocsClasses from './aidocs_classes/index.js';
+
+export class AiDoc {
+  private openaiToken: string;
+
+  public npmextraKV: plugins.npmextra.KeyValueStore;
+  public qenvInstance: plugins.qenv.Qenv;
+  public aidocInteract: plugins.smartinteract.SmartInteract;
+  public openaiInstance: plugins.smartai.OpenAiProvider;
+
+  argvArg: any;
+
+  constructor(argvArg?: any) {
+    this.argvArg = argvArg;
+  }
+
+  private printSanitizedToken() {
+    // Check if the token length is greater than the sum of startLength and endLength
+    let printToken: string;
+    if (this.openaiToken.length > 6) {
+      // Extract the beginning and end parts of the token
+      const start = this.openaiToken.substring(0, 3);
+      const end = this.openaiToken.substring(this.openaiToken.length - 3);
+      printToken = `${start}...${end}`;
+    } else {
+      // If the token is not long enough, return it as is
+      printToken = this.openaiToken;
+    }
+    console.log(`OpenAI Token on record: ${printToken}`);
+  }
+
+  public async start() {
+    // lets care about prerequisites
+    this.aidocInteract = new plugins.smartinteract.SmartInteract();
+    this.qenvInstance = new plugins.qenv.Qenv();
+    if (!(await this.qenvInstance.getEnvVarOnDemand('OPENAI_TOKEN'))) {
+      this.npmextraKV = new plugins.npmextra.KeyValueStore({
+        typeArg: 'userHomeDir',
+        identityArg: 'tsdoc',
+        mandatoryKeys: ['OPENAI_TOKEN'],
+      });
+
+      const missingKeys = await this.npmextraKV.getMissingMandatoryKeys();
+      if (missingKeys.length > 0) {
+        // lets try argv
+        if (this.argvArg?.OPENAI_TOKEN) {
+          this.openaiToken = this.argvArg.OPENAI_TOKEN;
+        } else {
+          // lets try smartinteract
+          // wait for a second until OpenAI fixes punycode problem...
+          await plugins.smartdelay.delayFor(1000);
+          const answerObject = await this.aidocInteract.askQuestion({
+            type: 'input',
+            message: `Please provide your OpenAI token. This will be persisted in your home directory.`,
+            name: 'OPENAI_TOKEN',
+            default: '',
+          });
+          this.openaiToken = answerObject.value;
+        }
+
+        this.printSanitizedToken();
+        await this.npmextraKV.writeKey('OPENAI_TOKEN', this.openaiToken);
+      }
+    }
+    if (!this.openaiToken) {
+      this.openaiToken = await this.npmextraKV.readKey('OPENAI_TOKEN');
+    }
+
+    // lets assume we have an OPENAI_Token now
+    this.openaiInstance = new plugins.smartai.OpenAiProvider({
+      openaiToken: this.openaiToken,
+    });
+    await this.openaiInstance.start();
+  }
+
+  public async buildReadme(projectDirArg: string) {
+    const readmeInstance = new aiDocsClasses.Readme(this, projectDirArg);
+    return await readmeInstance.build();
+  }
+
+  public async buildDescription(projectDirArg: string) {
+    const descriptionInstance = new aiDocsClasses.Description(this, projectDirArg);
+    return await descriptionInstance.build();
+  }
+
+  public async buildNextCommitObject(projectDirArg: string) {
+    const commitInstance = new aiDocsClasses.Commit(this, projectDirArg);
+    return await commitInstance.buildNextCommitObject();
+  }
+
+  public async getProjectContext(projectDirArg: string) {
+    const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
+    return await projectContextInstance.gatherFiles();
+  }
+  
+  /**
+   * Get the context with token count information
+   * @param projectDirArg The path to the project directory
+   * @returns An object containing the context string and its token count
+   */
+  public async getProjectContextWithTokenCount(projectDirArg: string) {
+    const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
+    await projectContextInstance.update();
+    return projectContextInstance.getContextWithTokenCount();
+  }
+  
+  /**
+   * Get just the token count for a project's context
+   * @param projectDirArg The path to the project directory
+   * @returns The number of tokens in the project context
+   */
+  public async getProjectContextTokenCount(projectDirArg: string) {
+    const projectContextInstance = new aiDocsClasses.ProjectContext(projectDirArg);
+    await projectContextInstance.update();
+    return projectContextInstance.getTokenCount();
+  }
+  
+  /**
+   * Count tokens in a text string using GPT tokenizer
+   * @param text The text to count tokens for
+   * @param model The model to use for tokenization (default: gpt-3.5-turbo)
+   * @returns The number of tokens in the text
+   */
+  public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
+    const projectContextInstance = new aiDocsClasses.ProjectContext('');
+    return projectContextInstance.countTokens(text, model);
+  }
+}

ts/classes.typedoc.ts

@@ -0,0 +1,58 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+
+export class TypeDoc {
+  public smartshellInstance = new plugins.smartshell.Smartshell({
+    executor: 'bash',
+    pathDirectories: [paths.binDir],
+  });
+
+  // Static
+  public static async isTypeDocDir(dirPathArg: string): Promise<boolean> {
+    return true;
+  }
+
+  // Instance
+  public typedocDirectory: string;
+  constructor(dirPathArg) {
+    this.typedocDirectory = dirPathArg;
+  }
+
+  public async compile(options?: { publicSubdir?: string }) {
+    const data = {
+      compilerOptions: {
+        experimentalDecorators: true,
+        useDefineForClassFields: false,
+        target: 'ES2022',
+        module: 'NodeNext',
+        moduleResolution: 'NodeNext',
+        esModuleInterop: true,
+        verbatimModuleSyntax: true,
+        skipLibCheck: true,
+      },
+      include: [],
+    };
+    let startDirectory = '';
+    if (plugins.smartfile.fs.isDirectory(plugins.path.join(paths.cwd, './ts'))) {
+      data.include.push(plugins.path.join(paths.cwd, './ts/**/*'));
+      startDirectory = 'ts';
+    }
+
+    if (plugins.smartfile.fs.isDirectory(plugins.path.join(paths.cwd, './ts_web'))) {
+      data.include.push(plugins.path.join(paths.cwd, './ts_web/**/*'));
+      if (!startDirectory) {
+        startDirectory = 'ts_web';
+      }
+    }
+
+    await plugins.smartfile.memory.toFs(JSON.stringify(data), paths.tsconfigFile);
+    let targetDir = paths.publicDir;
+    if (options?.publicSubdir) {
+      targetDir = plugins.path.join(targetDir, options.publicSubdir);
+    }
+    await this.smartshellInstance.exec(
+      `typedoc --tsconfig ${paths.tsconfigFile} --out ${targetDir} ${startDirectory}/index.ts`,
+    );
+    plugins.smartfile.fs.remove(paths.tsconfigFile);
+  }
+}

ts/cli.ts

@@ -0,0 +1,177 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import { logger } from './logging.js';
+
+import { TypeDoc } from './classes.typedoc.js';
+import { AiDoc } from './classes.aidoc.js';
+import * as context from './context/index.js';
+
+export const run = async () => {
+  const tsdocCli = new plugins.smartcli.Smartcli();
+
+  tsdocCli.standardCommand().subscribe(async (argvArg) => {
+    logger.log('warn', `Auto detecting environment!`);
+    switch (true) {
+      case await TypeDoc.isTypeDocDir(paths.cwd):
+        logger.log('ok', `Detected TypeDoc compliant directory at ${paths.cwd}`);
+        tsdocCli.triggerCommand('typedoc', argvArg);
+        break;
+      default:
+        logger.log('error', `Cannot determine docs format at ${paths.cwd}`);
+    }
+  });
+
+  tsdocCli.addCommand('typedoc').subscribe(async (argvArg) => {
+    const typeDocInstance = new TypeDoc(paths.cwd);
+    await typeDocInstance.compile({
+      publicSubdir: argvArg.publicSubdir,
+    });
+  });
+
+  tsdocCli.addCommand('aidoc').subscribe(async (argvArg) => {
+    const aidocInstance = new AiDoc();
+    await aidocInstance.start();
+    
+    // Get context token count if requested
+    if (argvArg.tokens || argvArg.showTokens) {
+      logger.log('info', `Calculating context token count...`);
+      const tokenCount = await aidocInstance.getProjectContextTokenCount(paths.cwd);
+      logger.log('ok', `Total context token count: ${tokenCount}`);
+      
+      if (argvArg.tokensOnly) {
+        return; // Exit early if we only want token count
+      }
+    }
+    
+    logger.log('info', `Generating new readme...`);
+    logger.log('info', `This may take some time...`);
+    await aidocInstance.buildReadme(paths.cwd);
+    logger.log('info', `Generating new keywords...`);
+    logger.log('info', `This may take some time...`);
+    await aidocInstance.buildDescription(paths.cwd);
+  });
+
+  tsdocCli.addCommand('tokens').subscribe(async (argvArg) => {
+    const aidocInstance = new AiDoc();
+    await aidocInstance.start();
+    
+    logger.log('info', `Calculating context token count...`);
+    
+    // Determine context mode based on args
+    let contextMode: context.ContextMode = 'full';
+    if (argvArg.trim || argvArg.trimmed) {
+      contextMode = 'trimmed';
+    } else if (argvArg.summarize || argvArg.summarized) {
+      contextMode = 'summarized';
+    }
+    
+    // Get task type if specified
+    let taskType: context.TaskType | undefined = undefined;
+    if (argvArg.task) {
+      if (['readme', 'commit', 'description'].includes(argvArg.task)) {
+        taskType = argvArg.task as context.TaskType;
+      } else {
+        logger.log('warn', `Unknown task type: ${argvArg.task}. Using default context.`);
+      }
+    }
+    
+    // Use enhanced context
+    const taskFactory = new context.TaskContextFactory(paths.cwd);
+    await taskFactory.initialize();
+    
+    let contextResult: context.IContextResult;
+    
+    if (argvArg.all) {
+      // Show stats for all task types
+      const stats = await taskFactory.getTokenStats();
+      
+      logger.log('ok', 'Token statistics by task:');
+      for (const [task, data] of Object.entries(stats)) {
+        logger.log('info', `\n${task.toUpperCase()}:`);
+        logger.log('info', `  Tokens: ${data.tokenCount}`);
+        logger.log('info', `  Token savings: ${data.savings}`);
+        logger.log('info', `  Files: ${data.includedFiles} included, ${data.trimmedFiles} trimmed, ${data.excludedFiles} excluded`);
+        
+        // Calculate percentage of model context
+        const o4MiniPercentage = (data.tokenCount / 200000 * 100).toFixed(2);
+        logger.log('info', `  Context usage: ${o4MiniPercentage}% of o4-mini (200K tokens)`);
+      }
+      
+      return;
+    }
+    
+    if (taskType) {
+      // Get context for specific task
+      contextResult = await taskFactory.createContextForTask(taskType);
+    } else {
+      // Get generic context with specified mode
+      const enhancedContext = new context.EnhancedContext(paths.cwd);
+      await enhancedContext.initialize();
+      enhancedContext.setContextMode(contextMode);
+      
+      if (argvArg.maxTokens) {
+        enhancedContext.setTokenBudget(parseInt(argvArg.maxTokens, 10));
+      }
+      
+      contextResult = await enhancedContext.buildContext();
+    }
+    
+    // Display results
+    logger.log('ok', `Total context token count: ${contextResult.tokenCount}`);
+    logger.log('info', `Files included: ${contextResult.includedFiles.length}`);
+    logger.log('info', `Files trimmed: ${contextResult.trimmedFiles.length}`);
+    logger.log('info', `Files excluded: ${contextResult.excludedFiles.length}`);
+    logger.log('info', `Token savings: ${contextResult.tokenSavings}`);
+    
+    if (argvArg.detailed) {
+      // Show more detailed info about the context and token usage
+      const o4MiniPercentage = (contextResult.tokenCount / 200000 * 100).toFixed(2);
+      logger.log('info', `Token usage: ${o4MiniPercentage}% of o4-mini 200K token context window`);
+      
+      if (argvArg.model) {
+        // Show percentages for different models
+        if (argvArg.model === 'gpt4') {
+          const gpt4Percentage = (contextResult.tokenCount / 8192 * 100).toFixed(2);
+          logger.log('info', `Token usage (GPT-4): ${gpt4Percentage}% of 8192 token context window`);
+        } else if (argvArg.model === 'gpt35') {
+          const gpt35Percentage = (contextResult.tokenCount / 4096 * 100).toFixed(2);
+          logger.log('info', `Token usage (GPT-3.5): ${gpt35Percentage}% of 4096 token context window`);
+        }
+      }
+      
+      // Estimate cost (approximate values)
+      const o4MiniInputCost = 0.00005; // per 1K tokens for o4-mini
+      const estimatedCost = (contextResult.tokenCount / 1000 * o4MiniInputCost).toFixed(6);
+      logger.log('info', `Estimated input cost: $${estimatedCost} (o4-mini)`);
+      
+      if (argvArg.listFiles) {
+        // List files included in context
+        logger.log('info', '\nIncluded files:');
+        contextResult.includedFiles.forEach(file => {
+          logger.log('info', `  ${file.relativePath} (${file.tokenCount} tokens)`);
+        });
+        
+        logger.log('info', '\nTrimmed files:');
+        contextResult.trimmedFiles.forEach(file => {
+          logger.log('info', `  ${file.relativePath} (${file.tokenCount} tokens)`);
+        });
+        
+        if (contextResult.excludedFiles.length > 0) {
+          logger.log('info', '\nExcluded files:');
+          contextResult.excludedFiles.forEach(file => {
+            logger.log('info', `  ${file.relativePath} (${file.tokenCount} tokens)`);
+          });
+        }
+      }
+    }
+  });
+
+  tsdocCli.addCommand('test').subscribe((argvArg) => {
+    tsdocCli.triggerCommand('typedoc', argvArg);
+    process.on('exit', async () => {
+      await plugins.smartfile.fs.remove(paths.publicDir);
+    });
+  });
+
+  tsdocCli.startParse();
+};

ts/context/config-manager.ts

@@ -0,0 +1,209 @@
+import * as plugins from '../plugins.js';
+import type { IContextConfig, ITrimConfig, ITaskConfig, TaskType, ContextMode } from './types.js';
+
+/**
+ * Manages configuration for context building
+ */
+export class ConfigManager {
+  private static instance: ConfigManager;
+  private config: IContextConfig;
+  private projectDir: string = '';
+  
+  /**
+   * Get the singleton instance of ConfigManager
+   */
+  public static getInstance(): ConfigManager {
+    if (!ConfigManager.instance) {
+      ConfigManager.instance = new ConfigManager();
+    }
+    return ConfigManager.instance;
+  }
+  
+  /**
+   * Private constructor for singleton pattern
+   */
+  private constructor() {
+    this.config = this.getDefaultConfig();
+  }
+  
+  /**
+   * Initialize the config manager with a project directory
+   * @param projectDir The project directory
+   */
+  public async initialize(projectDir: string): Promise<void> {
+    this.projectDir = projectDir;
+    await this.loadConfig();
+  }
+  
+  /**
+   * Get the default configuration
+   */
+  private getDefaultConfig(): IContextConfig {
+    return {
+      maxTokens: 190000, // Default for o4-mini with some buffer
+      defaultMode: 'trimmed',
+      taskSpecificSettings: {
+        readme: {
+          mode: 'trimmed',
+          includePaths: ['ts/', 'src/'],
+          excludePaths: ['test/', 'node_modules/']
+        },
+        commit: {
+          mode: 'trimmed',
+          focusOnChangedFiles: true
+        },
+        description: {
+          mode: 'trimmed',
+          includePackageInfo: true
+        }
+      },
+      trimming: {
+        removeImplementations: true,
+        preserveInterfaces: true,
+        preserveTypeDefs: true,
+        preserveJSDoc: true,
+        maxFunctionLines: 5,
+        removeComments: true,
+        removeBlankLines: true
+      }
+    };
+  }
+  
+  /**
+   * Load configuration from npmextra.json
+   */
+  private async loadConfig(): Promise<void> {
+    try {
+      if (!this.projectDir) {
+        return;
+      }
+      
+      // Create KeyValueStore for this project
+      // We'll just use smartfile directly instead of KeyValueStore
+      
+      // Read the npmextra.json file
+      const npmextraJsonFile = await plugins.smartfile.SmartFile.fromFilePath(
+        plugins.path.join(this.projectDir, 'npmextra.json')
+      );
+      const npmextraContent = JSON.parse(npmextraJsonFile.contents.toString());
+      
+      // Check for tsdoc context configuration
+      if (npmextraContent?.tsdoc?.context) {
+        // Merge with default config
+        this.config = this.mergeConfigs(this.config, npmextraContent.tsdoc.context);
+      }
+    } catch (error) {
+      console.error('Error loading context configuration:', error);
+    }
+  }
+  
+  /**
+   * Merge configurations, with userConfig taking precedence
+   * @param defaultConfig The default configuration
+   * @param userConfig The user configuration
+   */
+  private mergeConfigs(defaultConfig: IContextConfig, userConfig: Partial<IContextConfig>): IContextConfig {
+    const result: IContextConfig = { ...defaultConfig };
+    
+    // Merge top-level properties
+    if (userConfig.maxTokens !== undefined) result.maxTokens = userConfig.maxTokens;
+    if (userConfig.defaultMode !== undefined) result.defaultMode = userConfig.defaultMode;
+    
+    // Merge task-specific settings
+    if (userConfig.taskSpecificSettings) {
+      result.taskSpecificSettings = result.taskSpecificSettings || {};
+      
+      // For each task type, merge settings
+      (['readme', 'commit', 'description'] as TaskType[]).forEach(taskType => {
+        if (userConfig.taskSpecificSettings?.[taskType]) {
+          result.taskSpecificSettings![taskType] = {
+            ...result.taskSpecificSettings![taskType],
+            ...userConfig.taskSpecificSettings[taskType]
+          };
+        }
+      });
+    }
+    
+    // Merge trimming configuration
+    if (userConfig.trimming) {
+      result.trimming = {
+        ...result.trimming,
+        ...userConfig.trimming
+      };
+    }
+    
+    return result;
+  }
+  
+  /**
+   * Get the complete configuration
+   */
+  public getConfig(): IContextConfig {
+    return this.config;
+  }
+  
+  /**
+   * Get the trimming configuration
+   */
+  public getTrimConfig(): ITrimConfig {
+    return this.config.trimming || {};
+  }
+  
+  /**
+   * Get configuration for a specific task
+   * @param taskType The type of task
+   */
+  public getTaskConfig(taskType: TaskType): ITaskConfig {
+    // Get task-specific config or empty object
+    const taskConfig = this.config.taskSpecificSettings?.[taskType] || {};
+    
+    // If mode is not specified, use default mode
+    if (!taskConfig.mode) {
+      taskConfig.mode = this.config.defaultMode;
+    }
+    
+    return taskConfig;
+  }
+  
+  /**
+   * Get the maximum tokens allowed for context
+   */
+  public getMaxTokens(): number {
+    return this.config.maxTokens || 190000;
+  }
+  
+  /**
+   * Update the configuration
+   * @param config The new configuration
+   */
+  public async updateConfig(config: Partial<IContextConfig>): Promise<void> {
+    // Merge with existing config
+    this.config = this.mergeConfigs(this.config, config);
+    
+    try {
+      if (!this.projectDir) {
+        return;
+      }
+      
+      // Read the existing npmextra.json file
+      const npmextraJsonPath = plugins.path.join(this.projectDir, 'npmextra.json');
+      let npmextraContent = {};
+      
+      if (await plugins.smartfile.fs.fileExists(npmextraJsonPath)) {
+        const npmextraJsonFile = await plugins.smartfile.SmartFile.fromFilePath(npmextraJsonPath);
+        npmextraContent = JSON.parse(npmextraJsonFile.contents.toString()) || {};
+      }
+      
+      // Update the tsdoc context configuration
+      const typedContent = npmextraContent as any;
+      if (!typedContent.tsdoc) typedContent.tsdoc = {};
+      typedContent.tsdoc.context = this.config;
+      
+      // Write back to npmextra.json
+      const updatedContent = JSON.stringify(npmextraContent, null, 2);
+      await plugins.smartfile.memory.toFs(updatedContent, npmextraJsonPath);
+    } catch (error) {
+      console.error('Error updating context configuration:', error);
+    }
+  }
+}

ts/context/context-trimmer.ts

@@ -0,0 +1,246 @@
+import * as plugins from '../plugins.js';
+import type { ITrimConfig, ContextMode } from './types.js';
+
+/**
+ * Class responsible for trimming file contents to reduce token usage
+ * while preserving important information for context
+ */
+export class ContextTrimmer {
+  private config: ITrimConfig;
+  
+  /**
+   * Create a new ContextTrimmer with the given configuration
+   * @param config The trimming configuration
+   */
+  constructor(config?: ITrimConfig) {
+    this.config = {
+      removeImplementations: true,
+      preserveInterfaces: true,
+      preserveTypeDefs: true,
+      preserveJSDoc: true,
+      maxFunctionLines: 5,
+      removeComments: true,
+      removeBlankLines: true,
+      ...config
+    };
+  }
+  
+  /**
+   * Trim a file's contents based on the configuration
+   * @param filePath The path to the file
+   * @param content The file's contents
+   * @param mode The context mode to use
+   * @returns The trimmed file contents
+   */
+  public trimFile(filePath: string, content: string, mode: ContextMode = 'trimmed'): string {
+    // If mode is 'full', return the original content
+    if (mode === 'full') {
+      return content;
+    }
+    
+    // Process based on file type
+    if (filePath.endsWith('.ts') || filePath.endsWith('.tsx')) {
+      return this.trimTypeScriptFile(content);
+    } else if (filePath.endsWith('.md')) {
+      return this.trimMarkdownFile(content);
+    } else if (filePath.endsWith('.json')) {
+      return this.trimJsonFile(content);
+    }
+    
+    // Default to returning the original content for unknown file types
+    return content;
+  }
+  
+  /**
+   * Trim a TypeScript file to reduce token usage
+   * @param content The TypeScript file contents
+   * @returns The trimmed file contents
+   */
+  private trimTypeScriptFile(content: string): string {
+    let result = content;
+    
+    // Step 1: Preserve JSDoc comments if configured
+    const jsDocComments: string[] = [];
+    if (this.config.preserveJSDoc) {
+      const jsDocRegex = /\/\*\*[\s\S]*?\*\//g;
+      const matches = result.match(jsDocRegex) || [];
+      jsDocComments.push(...matches);
+    }
+    
+    // Step 2: Remove comments if configured
+    if (this.config.removeComments) {
+      // Remove single-line comments
+      result = result.replace(/\/\/.*$/gm, '');
+      // Remove multi-line comments (except JSDoc if preserveJSDoc is true)
+      if (!this.config.preserveJSDoc) {
+        result = result.replace(/\/\*[\s\S]*?\*\//g, '');
+      } else {
+        // Only remove non-JSDoc comments
+        result = result.replace(/\/\*(?!\*)[\s\S]*?\*\//g, '');
+      }
+    }
+    
+    // Step 3: Remove function implementations if configured
+    if (this.config.removeImplementations) {
+      // Match function and method bodies
+      result = result.replace(
+        /(\b(function|constructor|async function)\s+[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
+        (match, start, funcType, body, end) => {
+          // Keep function signature and opening brace, replace body with comment
+          return `${start} /* implementation removed */ ${end}`;
+        }
+      );
+      
+      // Match arrow function bodies
+      result = result.replace(
+        /(\([^)]*\)\s*=>\s*{)([\s\S]*?)(})/g,
+        (match, start, body, end) => {
+          return `${start} /* implementation removed */ ${end}`;
+        }
+      );
+      
+      // Match method declarations
+      result = result.replace(
+        /(^\s*[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/gm,
+        (match, start, body, end) => {
+          return `${start} /* implementation removed */ ${end}`;
+        }
+      );
+      
+      // Match class methods
+      result = result.replace(
+        /(\b(public|private|protected|static|async)?\s+[\w$]+\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
+        (match, start, modifier, body, end) => {
+          return `${start} /* implementation removed */ ${end}`;
+        }
+      );
+    } else if (this.config.maxFunctionLines && this.config.maxFunctionLines > 0) {
+      // If not removing implementations completely, limit the number of lines
+      // Match function and method bodies
+      result = result.replace(
+        /(\b(function|constructor|async function)\s+[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
+        (match, start, funcType, body, end) => {
+          return this.limitFunctionBody(start, body, end);
+        }
+      );
+      
+      // Match arrow function bodies
+      result = result.replace(
+        /(\([^)]*\)\s*=>\s*{)([\s\S]*?)(})/g,
+        (match, start, body, end) => {
+          return this.limitFunctionBody(start, body, end);
+        }
+      );
+      
+      // Match method declarations
+      result = result.replace(
+        /(^\s*[\w$]*\s*\([^)]*\)\s*{)([\s\S]*?)(})/gm,
+        (match, start, body, end) => {
+          return this.limitFunctionBody(start, body, end);
+        }
+      );
+      
+      // Match class methods
+      result = result.replace(
+        /(\b(public|private|protected|static|async)?\s+[\w$]+\s*\([^)]*\)\s*{)([\s\S]*?)(})/g,
+        (match, start, modifier, body, end) => {
+          return this.limitFunctionBody(start, body, end);
+        }
+      );
+    }
+    
+    // Step 4: Remove blank lines if configured
+    if (this.config.removeBlankLines) {
+      result = result.replace(/^\s*[\r\n]/gm, '');
+    }
+    
+    // Step 5: Restore preserved JSDoc comments
+    if (this.config.preserveJSDoc && jsDocComments.length > 0) {
+      // This is a placeholder; we already preserved JSDoc comments in the regex steps
+    }
+    
+    return result;
+  }
+  
+  /**
+   * Limit a function body to a maximum number of lines
+   * @param start The function signature and opening brace
+   * @param body The function body
+   * @param end The closing brace
+   * @returns The limited function body
+   */
+  private limitFunctionBody(start: string, body: string, end: string): string {
+    const lines = body.split('\n');
+    if (lines.length > this.config.maxFunctionLines!) {
+      const limitedBody = lines.slice(0, this.config.maxFunctionLines!).join('\n');
+      return `${start}${limitedBody}\n  // ... (${lines.length - this.config.maxFunctionLines!} lines trimmed)\n${end}`;
+    }
+    return `${start}${body}${end}`;
+  }
+  
+  /**
+   * Trim a Markdown file to reduce token usage
+   * @param content The Markdown file contents
+   * @returns The trimmed file contents
+   */
+  private trimMarkdownFile(content: string): string {
+    // For markdown files, we generally want to keep most content
+    // but we can remove lengthy code blocks if needed
+    return content;
+  }
+  
+  /**
+   * Trim a JSON file to reduce token usage
+   * @param content The JSON file contents
+   * @returns The trimmed file contents
+   */
+  private trimJsonFile(content: string): string {
+    try {
+      // Parse the JSON
+      const json = JSON.parse(content);
+      
+      // For package.json, keep only essential information
+      if ('name' in json && 'version' in json && 'dependencies' in json) {
+        const essentialKeys = [
+          'name', 'version', 'description', 'author', 'license',
+          'main', 'types', 'exports', 'type'
+        ];
+        
+        const trimmedJson: any = {};
+        essentialKeys.forEach(key => {
+          if (key in json) {
+            trimmedJson[key] = json[key];
+          }
+        });
+        
+        // Add dependency information without versions
+        if ('dependencies' in json) {
+          trimmedJson.dependencies = Object.keys(json.dependencies).reduce((acc, dep) => {
+            acc[dep] = '*'; // Replace version with wildcard
+            return acc;
+          }, {} as Record<string, string>);
+        }
+        
+        // Return the trimmed JSON
+        return JSON.stringify(trimmedJson, null, 2);
+      }
+      
+      // For other JSON files, leave as is
+      return content;
+    } catch (error) {
+      // If there's an error parsing the JSON, return the original content
+      return content;
+    }
+  }
+  
+  /**
+   * Update the trimmer configuration
+   * @param config The new configuration to apply
+   */
+  public updateConfig(config: ITrimConfig): void {
+    this.config = {
+      ...this.config,
+      ...config
+    };
+  }
+}

ts/context/enhanced-context.ts

@@ -0,0 +1,343 @@
+import * as plugins from '../plugins.js';
+import type { ContextMode, IContextResult, IFileInfo, TaskType } from './types.js';
+import { ContextTrimmer } from './context-trimmer.js';
+import { ConfigManager } from './config-manager.js';
+
+/**
+ * Enhanced ProjectContext that supports context optimization strategies
+ */
+export class EnhancedContext {
+  private projectDir: string;
+  private trimmer: ContextTrimmer;
+  private configManager: ConfigManager;
+  private contextMode: ContextMode = 'trimmed';
+  private tokenBudget: number = 190000; // Default for o4-mini
+  private contextResult: IContextResult = {
+    context: '',
+    tokenCount: 0,
+    includedFiles: [],
+    trimmedFiles: [],
+    excludedFiles: [],
+    tokenSavings: 0
+  };
+  
+  /**
+   * Create a new EnhancedContext
+   * @param projectDirArg The project directory
+   */
+  constructor(projectDirArg: string) {
+    this.projectDir = projectDirArg;
+    this.configManager = ConfigManager.getInstance();
+    this.trimmer = new ContextTrimmer(this.configManager.getTrimConfig());
+  }
+  
+  /**
+   * Initialize the context builder
+   */
+  public async initialize(): Promise<void> {
+    await this.configManager.initialize(this.projectDir);
+    this.tokenBudget = this.configManager.getMaxTokens();
+    this.trimmer.updateConfig(this.configManager.getTrimConfig());
+  }
+  
+  /**
+   * Set the context mode
+   * @param mode The context mode to use
+   */
+  public setContextMode(mode: ContextMode): void {
+    this.contextMode = mode;
+  }
+  
+  /**
+   * Set the token budget
+   * @param maxTokens The maximum tokens to use
+   */
+  public setTokenBudget(maxTokens: number): void {
+    this.tokenBudget = maxTokens;
+  }
+  
+  /**
+   * Gather files from the project
+   * @param includePaths Optional paths to include
+   * @param excludePaths Optional paths to exclude
+   */
+  public async gatherFiles(includePaths?: string[], excludePaths?: string[]): Promise<Record<string, plugins.smartfile.SmartFile | plugins.smartfile.SmartFile[]>> {
+    const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(
+      plugins.path.join(this.projectDir, 'package.json'),
+      this.projectDir,
+    );
+    
+    const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(
+      plugins.path.join(this.projectDir, 'readme.md'),
+      this.projectDir,
+    );
+
+    const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(
+      plugins.path.join(this.projectDir, 'readme.hints.md'),
+      this.projectDir,
+    );
+    
+    const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(
+      plugins.path.join(this.projectDir, 'npmextra.json'),
+      this.projectDir,
+    );
+    
+    // Use provided include paths or default to all TypeScript files
+    const includeGlobs = includePaths?.map(path => `${path}/**/*.ts`) || ['ts*/**/*.ts'];
+    
+    // Get TypeScript files
+    const smartfilesModPromises = includeGlobs.map(glob => 
+      plugins.smartfile.fs.fileTreeToObject(this.projectDir, glob)
+    );
+    
+    const smartfilesModArrays = await Promise.all(smartfilesModPromises);
+    
+    // Flatten the arrays
+    const smartfilesMod: plugins.smartfile.SmartFile[] = [];
+    smartfilesModArrays.forEach(array => {
+      smartfilesMod.push(...array);
+    });
+    
+    // Get test files if not excluded
+    let smartfilesTest: plugins.smartfile.SmartFile[] = [];
+    if (!excludePaths?.includes('test/')) {
+      smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(
+        this.projectDir,
+        'test/**/*.ts',
+      );
+    }
+    
+    return {
+      smartfilePackageJSON,
+      smartfilesReadme,
+      smartfilesReadmeHints,
+      smartfilesNpmextraJSON,
+      smartfilesMod,
+      smartfilesTest,
+    };
+  }
+  
+  /**
+   * Convert files to context string
+   * @param files The files to convert
+   * @param mode The context mode to use
+   */
+  public async convertFilesToContext(
+    files: plugins.smartfile.SmartFile[],
+    mode: ContextMode = this.contextMode
+  ): Promise<string> {
+    // Reset context result
+    this.contextResult = {
+      context: '',
+      tokenCount: 0,
+      includedFiles: [],
+      trimmedFiles: [],
+      excludedFiles: [],
+      tokenSavings: 0
+    };
+    
+    let totalTokenCount = 0;
+    let totalOriginalTokens = 0;
+    
+    // Sort files by importance (for now just a simple alphabetical sort)
+    // Later this could be enhanced with more sophisticated prioritization
+    const sortedFiles = [...files].sort((a, b) => a.relative.localeCompare(b.relative));
+    
+    const processedFiles: string[] = [];
+    
+    for (const smartfile of sortedFiles) {
+      // Calculate original token count
+      const originalContent = smartfile.contents.toString();
+      const originalTokenCount = this.countTokens(originalContent);
+      totalOriginalTokens += originalTokenCount;
+      
+      // Apply trimming based on mode
+      let processedContent = originalContent;
+      
+      if (mode !== 'full') {
+        processedContent = this.trimmer.trimFile(
+          smartfile.relative,
+          originalContent,
+          mode
+        );
+      }
+      
+      // Calculate new token count
+      const processedTokenCount = this.countTokens(processedContent);
+      
+      // Check if we have budget for this file
+      if (totalTokenCount + processedTokenCount > this.tokenBudget) {
+        // We don't have budget for this file
+        this.contextResult.excludedFiles.push({
+          path: smartfile.path,
+          contents: originalContent,
+          relativePath: smartfile.relative,
+          tokenCount: originalTokenCount
+        });
+        continue;
+      }
+      
+      // Format the file for context
+      const formattedContent = `
+====== START OF FILE ${smartfile.relative} ======
+  
+${processedContent}
+  
+====== END OF FILE ${smartfile.relative} ======
+      `;
+      
+      processedFiles.push(formattedContent);
+      totalTokenCount += processedTokenCount;
+      
+      // Track file in appropriate list
+      const fileInfo: IFileInfo = {
+        path: smartfile.path,
+        contents: processedContent,
+        relativePath: smartfile.relative,
+        tokenCount: processedTokenCount
+      };
+      
+      if (mode === 'full' || processedContent === originalContent) {
+        this.contextResult.includedFiles.push(fileInfo);
+      } else {
+        this.contextResult.trimmedFiles.push(fileInfo);
+        this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
+      }
+    }
+    
+    // Join all processed files
+    const context = processedFiles.join('\n');
+    
+    // Update context result
+    this.contextResult.context = context;
+    this.contextResult.tokenCount = totalTokenCount;
+    
+    return context;
+  }
+  
+  /**
+   * Build context for the project
+   * @param taskType Optional task type for task-specific context
+   */
+  public async buildContext(taskType?: TaskType): Promise<IContextResult> {
+    // Initialize if needed
+    if (this.tokenBudget === 0) {
+      await this.initialize();
+    }
+    
+    // Get task-specific configuration if a task type is provided
+    if (taskType) {
+      const taskConfig = this.configManager.getTaskConfig(taskType);
+      if (taskConfig.mode) {
+        this.setContextMode(taskConfig.mode);
+      }
+    }
+    
+    // Gather files
+    const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
+    const files = await this.gatherFiles(
+      taskConfig?.includePaths,
+      taskConfig?.excludePaths
+    );
+    
+    // Convert files to context
+    // Create an array of all files to process
+    const allFiles: plugins.smartfile.SmartFile[] = [];
+    
+    // Add individual files
+    if (files.smartfilePackageJSON) allFiles.push(files.smartfilePackageJSON as plugins.smartfile.SmartFile);
+    if (files.smartfilesReadme) allFiles.push(files.smartfilesReadme as plugins.smartfile.SmartFile);
+    if (files.smartfilesReadmeHints) allFiles.push(files.smartfilesReadmeHints as plugins.smartfile.SmartFile);
+    if (files.smartfilesNpmextraJSON) allFiles.push(files.smartfilesNpmextraJSON as plugins.smartfile.SmartFile);
+    
+    // Add arrays of files
+    if (files.smartfilesMod) {
+      if (Array.isArray(files.smartfilesMod)) {
+        allFiles.push(...files.smartfilesMod);
+      } else {
+        allFiles.push(files.smartfilesMod);
+      }
+    }
+    
+    if (files.smartfilesTest) {
+      if (Array.isArray(files.smartfilesTest)) {
+        allFiles.push(...files.smartfilesTest);
+      } else {
+        allFiles.push(files.smartfilesTest);
+      }
+    }
+    
+    const context = await this.convertFilesToContext(allFiles);
+    
+    return this.contextResult;
+  }
+  
+  /**
+   * Update the context with git diff information for commit tasks
+   * @param gitDiff The git diff to include
+   */
+  public updateWithGitDiff(gitDiff: string): IContextResult {
+    // If we don't have a context yet, return empty result
+    if (!this.contextResult.context) {
+      return this.contextResult;
+    }
+    
+    // Add git diff to context
+    const diffSection = `
+====== GIT DIFF ======
+
+${gitDiff}
+
+====== END GIT DIFF ======
+    `;
+    
+    const diffTokenCount = this.countTokens(diffSection);
+    
+    // Update context and token count
+    this.contextResult.context += diffSection;
+    this.contextResult.tokenCount += diffTokenCount;
+    
+    return this.contextResult;
+  }
+  
+  /**
+   * Count tokens in a string
+   * @param text The text to count tokens for
+   * @param model The model to use for token counting
+   */
+  public countTokens(text: string, model: string = 'gpt-3.5-turbo'): number {
+    try {
+      // Use the gpt-tokenizer library to count tokens
+      const tokens = plugins.gptTokenizer.encode(text);
+      return tokens.length;
+    } catch (error) {
+      console.error('Error counting tokens:', error);
+      // Provide a rough estimate if tokenization fails
+      return Math.ceil(text.length / 4);
+    }
+  }
+  
+  /**
+   * Get the context result
+   */
+  public getContextResult(): IContextResult {
+    return this.contextResult;
+  }
+  
+  /**
+   * Get the token count for the current context
+   */
+  public getTokenCount(): number {
+    return this.contextResult.tokenCount;
+  }
+  
+  /**
+   * Get both the context string and its token count
+   */
+  public getContextWithTokenCount(): { context: string; tokenCount: number } {
+    return {
+      context: this.contextResult.context,
+      tokenCount: this.contextResult.tokenCount
+    };
+  }
+}

ts/context/index.ts

@@ -0,0 +1,32 @@
+import { EnhancedContext } from './enhanced-context.js';
+import { TaskContextFactory } from './task-context-factory.js';
+import { ConfigManager } from './config-manager.js';
+import { ContextTrimmer } from './context-trimmer.js';
+import type { 
+  ContextMode, 
+  IContextConfig, 
+  IContextResult,
+  IFileInfo,
+  ITrimConfig,
+  ITaskConfig,
+  TaskType
+} from './types.js';
+
+export {
+  // Classes
+  EnhancedContext,
+  TaskContextFactory,
+  ConfigManager,
+  ContextTrimmer,
+};
+
+// Types
+export type {
+  ContextMode,
+  IContextConfig,
+  IContextResult,
+  IFileInfo,
+  ITrimConfig,
+  ITaskConfig,
+  TaskType
+};

ts/context/task-context-factory.ts

@@ -0,0 +1,138 @@
+import * as plugins from '../plugins.js';
+import { EnhancedContext } from './enhanced-context.js';
+import { ConfigManager } from './config-manager.js';
+import type { IContextResult, TaskType } from './types.js';
+
+/**
+ * Factory class for creating task-specific context
+ */
+export class TaskContextFactory {
+  private projectDir: string;
+  private configManager: ConfigManager;
+  
+  /**
+   * Create a new TaskContextFactory
+   * @param projectDirArg The project directory
+   */
+  constructor(projectDirArg: string) {
+    this.projectDir = projectDirArg;
+    this.configManager = ConfigManager.getInstance();
+  }
+  
+  /**
+   * Initialize the factory
+   */
+  public async initialize(): Promise<void> {
+    await this.configManager.initialize(this.projectDir);
+  }
+  
+  /**
+   * Create context for README generation
+   */
+  public async createContextForReadme(): Promise<IContextResult> {
+    const contextBuilder = new EnhancedContext(this.projectDir);
+    await contextBuilder.initialize();
+    
+    // Get README-specific configuration
+    const taskConfig = this.configManager.getTaskConfig('readme');
+    if (taskConfig.mode) {
+      contextBuilder.setContextMode(taskConfig.mode);
+    }
+    
+    // Build the context for README task
+    return await contextBuilder.buildContext('readme');
+  }
+  
+  /**
+   * Create context for description generation
+   */
+  public async createContextForDescription(): Promise<IContextResult> {
+    const contextBuilder = new EnhancedContext(this.projectDir);
+    await contextBuilder.initialize();
+    
+    // Get description-specific configuration
+    const taskConfig = this.configManager.getTaskConfig('description');
+    if (taskConfig.mode) {
+      contextBuilder.setContextMode(taskConfig.mode);
+    }
+    
+    // Build the context for description task
+    return await contextBuilder.buildContext('description');
+  }
+  
+  /**
+   * Create context for commit message generation
+   * @param gitDiff Optional git diff to include
+   */
+  public async createContextForCommit(gitDiff?: string): Promise<IContextResult> {
+    const contextBuilder = new EnhancedContext(this.projectDir);
+    await contextBuilder.initialize();
+    
+    // Get commit-specific configuration
+    const taskConfig = this.configManager.getTaskConfig('commit');
+    if (taskConfig.mode) {
+      contextBuilder.setContextMode(taskConfig.mode);
+    }
+    
+    // Build the context for commit task
+    const contextResult = await contextBuilder.buildContext('commit');
+    
+    // If git diff is provided, add it to the context
+    if (gitDiff) {
+      contextBuilder.updateWithGitDiff(gitDiff);
+    }
+    
+    return contextBuilder.getContextResult();
+  }
+  
+  /**
+   * Create context for any task type
+   * @param taskType The task type to create context for
+   * @param additionalContent Optional additional content to include
+   */
+  public async createContextForTask(
+    taskType: TaskType,
+    additionalContent?: string
+  ): Promise<IContextResult> {
+    switch (taskType) {
+      case 'readme':
+        return this.createContextForReadme();
+      case 'description':
+        return this.createContextForDescription();
+      case 'commit':
+        return this.createContextForCommit(additionalContent);
+      default:
+        // Generic context for unknown task types
+        const contextBuilder = new EnhancedContext(this.projectDir);
+        await contextBuilder.initialize();
+        return await contextBuilder.buildContext();
+    }
+  }
+  
+  /**
+   * Get token stats for all task types
+   */
+  public async getTokenStats(): Promise<Record<TaskType, {
+    tokenCount: number;
+    savings: number;
+    includedFiles: number;
+    trimmedFiles: number;
+    excludedFiles: number;
+  }>> {
+    const taskTypes: TaskType[] = ['readme', 'description', 'commit'];
+    const stats: Record<TaskType, any> = {} as any;
+    
+    for (const taskType of taskTypes) {
+      const result = await this.createContextForTask(taskType);
+      stats[taskType] = {
+        tokenCount: result.tokenCount,
+        savings: result.tokenSavings,
+        includedFiles: result.includedFiles.length,
+        trimmedFiles: result.trimmedFiles.length,
+        excludedFiles: result.excludedFiles.length
+      };
+    }
+    
+    return stats;
+  }
+}

ts/context/types.ts

@@ -0,0 +1,95 @@
+/**
+ * Context processing mode to control how context is built
+ */
+export type ContextMode = 'full' | 'trimmed' | 'summarized';
+
+/**
+ * Configuration for context trimming
+ */
+export interface ITrimConfig {
+  /** Whether to remove function implementations */
+  removeImplementations?: boolean;
+  /** Whether to preserve interface definitions */
+  preserveInterfaces?: boolean;
+  /** Whether to preserve type definitions */
+  preserveTypeDefs?: boolean;
+  /** Whether to preserve JSDoc comments */
+  preserveJSDoc?: boolean;
+  /** Maximum lines to keep for function bodies (if not removing completely) */
+  maxFunctionLines?: number;
+  /** Whether to remove normal comments (non-JSDoc) */
+  removeComments?: boolean;
+  /** Whether to remove blank lines */
+  removeBlankLines?: boolean;
+}
+
+/**
+ * Task types that require different context optimization
+ */
+export type TaskType = 'readme' | 'commit' | 'description';
+
+/**
+ * Configuration for different tasks
+ */
+export interface ITaskConfig {
+  /** The context mode to use for this task */
+  mode?: ContextMode;
+  /** File paths to include for this task */
+  includePaths?: string[];
+  /** File paths to exclude for this task */
+  excludePaths?: string[];
+  /** For commit tasks, whether to focus on changed files */
+  focusOnChangedFiles?: boolean;
+  /** For description tasks, whether to include package info */
+  includePackageInfo?: boolean;
+}
+
+/**
+ * Complete context configuration
+ */
+export interface IContextConfig {
+  /** Maximum tokens to use for context */
+  maxTokens?: number;
+  /** Default context mode */
+  defaultMode?: ContextMode;
+  /** Task-specific settings */
+  taskSpecificSettings?: {
+    [key in TaskType]?: ITaskConfig;
+  };
+  /** Trimming configuration */
+  trimming?: ITrimConfig;
+}
+
+/**
+ * Basic file information interface
+ */
+export interface IFileInfo {
+  /** The file path */
+  path: string;
+  /** The file contents */
+  contents: string;
+  /** The file's relative path from the project root */
+  relativePath: string;
+  /** The estimated token count of the file */
+  tokenCount?: number;
+  /** The file's importance score (higher is more important) */
+  importanceScore?: number;
+}
+
+/**
+ * Result of context building
+ */
+export interface IContextResult {
+  /** The generated context string */
+  context: string;
+  /** The total token count of the context */
+  tokenCount: number;
+  /** Files included in the context */
+  includedFiles: IFileInfo[];
+  /** Files that were trimmed */
+  trimmedFiles: IFileInfo[];
+  /** Files that were excluded */
+  excludedFiles: IFileInfo[];
+  /** Token savings from trimming */
+  tokenSavings: number;
+}

ts/index.ts

@@ -1,6 +1,12 @@
-import * as early from '@pushrocks/early';
+import * as early from '@push.rocks/early';
 early.start('tsdoc');
-import * as plugins from './tsdoc.plugins';
+import * as plugins from './plugins.js';
-import * as cli from './tsdoc.cli';
+import * as cli from './cli.js';
 early.stop();
-cli.run();
+
+export const runCli = async () => {
+  await cli.run();
+};
+
+// exports
+export * from './classes.aidoc.js';

ts/logging.ts

@@ -0,0 +1,6 @@
+import { commitinfo } from './00_commitinfo_data.js';
+import * as plugins from './plugins.js';
+
+export const logger = plugins.smartlog.Smartlog.createForCommitinfo(commitinfo);
+
+logger.addLogDestination(new plugins.smartlogDestinationLocal.DestinationLocal());

ts/paths.ts

@@ -0,0 +1,16 @@
+import * as plugins from './plugins.js';
+
+// dirs
+export const packageDir = plugins.path.join(
+  plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
+  '../',
+);
+export const cwd = process.cwd();
+export const binDir = plugins.path.join(packageDir, './node_modules/.bin');
+export const assetsDir = plugins.path.join(packageDir, './assets');
+export const publicDir = plugins.path.join(cwd, './public');
+export const tsDir = plugins.path.join(cwd, './ts');
+
+// files
+export const tsconfigFile = plugins.path.join(assetsDir, './tsconfig.json');
+export const typedocOptionsFile = plugins.path.join(assetsDir, './typedoc.json');

ts/plugins.ts

@@ -0,0 +1,46 @@
+// node native
+import * as path from 'path';
+
+export { path };
+
+// pushrocks scope
+import * as npmextra from '@push.rocks/npmextra';
+import * as qenv from '@push.rocks/qenv';
+import * as smartai from '@push.rocks/smartai';
+import * as smartcli from '@push.rocks/smartcli';
+import * as smartdelay from '@push.rocks/smartdelay';
+import * as smartfile from '@push.rocks/smartfile';
+import * as smartgit from '@push.rocks/smartgit';
+import * as smartinteract from '@push.rocks/smartinteract';
+import * as smartlog from '@push.rocks/smartlog';
+import * as smartlogDestinationLocal from '@push.rocks/smartlog-destination-local';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartshell from '@push.rocks/smartshell';
+import * as smarttime from '@push.rocks/smarttime';
+
+export {
+  npmextra,
+  qenv,
+  smartai,
+  smartcli,
+  smartdelay,
+  smartfile,
+  smartgit,
+  smartinteract,
+  smartlog,
+  smartlogDestinationLocal,
+  smartpath,
+  smartshell,
+  smarttime,
+};
+
+// @git.zone scope
+import * as tspublish from '@git.zone/tspublish';
+
+export { tspublish };
+
+// third party scope
+import * as typedoc from 'typedoc';
+import * as gptTokenizer from 'gpt-tokenizer';
+
+export { typedoc, gptTokenizer };

ts/tsdoc.classes.mkdocs.ts

@@ -1,67 +0,0 @@
-import * as plugins from './tsdoc.plugins';
-import * as paths from './tsdoc.paths';
-
-export class MkDocs {
-  public smartshellInstance = new plugins.smartshell.Smartshell({
-    executor: 'bash',
-    pathDirectories: [paths.binDir]
-  });
-
-  public static async isMkDocsDir(dirPathArg: string): Promise<boolean> {
-    const result = await plugins.smartfile.fs.fileExists(
-      plugins.path.join(dirPathArg, 'mkdocs.yml')
-    );
-    return result;
-  }
-
-  public static async handleCommand(argvArg) {
-    const mkdocsInstance = new MkDocs(paths.cwd);
-    switch (true) {
-      case argvArg.serve:
-        await mkdocsInstance.serve();
-        break;
-      case argvArg.publish:
-        await mkdocsInstance.publish();
-        break;
-      default:
-        await mkdocsInstance.compile();
-        break;
-    }
-  }
-
-  // Instance
-  public typedocDirectory: string;
-  constructor(dirPathArg) {
-    this.typedocDirectory = dirPathArg;
-  }
-
-  public async update() {
-    await this.smartshellInstance.exec(
-      `docker pull registry.gitlab.com/hosttoday/ht-docker-mkdocs`
-    );
-  }
-
-  public async compile() {
-    await this.update();
-    await this.smartshellInstance.exec(`rm -rf public/`);
-    await this.smartshellInstance.exec(
-      `docker run --rm -p 8000:8000 -v ${
-        paths.cwd
-      }:/docs registry.gitlab.com/hosttoday/ht-docker-mkdocs build`
-    );
-  }
-
-  public async serve() {
-    await this.update();
-    await this.smartshellInstance.exec(
-      `docker run --rm -p 8000:8000 -v ${
-        paths.cwd
-      }:/docs registry.gitlab.com/hosttoday/ht-docker-mkdocs`
-    );
-  }
-
-  public async publish() {
-    await this.compile();
-    await this.smartshellInstance.exec(`gitzone commit`);
-  }
-}

ts/tsdoc.classes.typedoc.ts

@@ -1,29 +0,0 @@
-import * as plugins from './tsdoc.plugins';
-import * as paths from './tsdoc.paths';
-
-export class TypeDoc {
-  public smartshellInstance = new plugins.smartshell.Smartshell({
-    executor: 'bash',
-    pathDirectories: [paths.binDir]
-  });
-
-  // Static
-  public static async isTypeDocDir(dirPathArg: string): Promise<boolean> {
-    const result = await plugins.smartfile.fs.fileExists(
-      plugins.path.join(dirPathArg, 'mkdocs.yml')
-    );
-    return !result;
-  }
-
-  // Instance
-  public typedocDirectory: string;
-  constructor(dirPathArg) {
-    this.typedocDirectory = dirPathArg;
-  }
-
-  public async compile() {
-    await this.smartshellInstance.exec(
-      `typedoc --module "commonjs" --target "ES2016" --out public/ ts/`
-    );
-  }
-}

ts/tsdoc.cli.ts

@@ -1,36 +0,0 @@
-import * as plugins from './tsdoc.plugins';
-import * as paths from './tsdoc.paths';
-import { logger } from './tsdoc.logging';
-
-import { TypeDoc } from './tsdoc.classes.typedoc';
-import { MkDocs } from './tsdoc.classes.mkdocs';
-
-export const run = async () => {
-  const tsdocCli = new plugins.smartcli.Smartcli();
-  tsdocCli.addCommand('typedoc').subscribe(async argvArg => {
-    const typeDocInstance = new TypeDoc(paths.cwd);
-    await typeDocInstance.compile();
-  });
-
-  tsdocCli.addCommand('mkdocs').subscribe(async argvArg => {
-    await MkDocs.handleCommand(argvArg);
-  });
-
-  tsdocCli.standardTask().subscribe(async argvArg => {
-    logger.log('warn', `Auto detecting environment!`);
-    switch (true) {
-      case await TypeDoc.isTypeDocDir(paths.cwd):
-        logger.log('ok', `Detected TypeDoc compliant directory at ${paths.cwd}`);
-        tsdocCli.trigger('typedoc');
-        break;
-      case await MkDocs.isMkDocsDir(paths.cwd):
-        logger.log('ok', `Detected MkDocs compliant directory at ${paths.cwd}`);
-        tsdocCli.trigger('mkdocs');
-        break;
-      default:
-        logger.log('error', `Cannot determine docs format at ${paths.cwd}`);
-    }
-  });
-
-  tsdocCli.startParse();
-};

ts/tsdoc.logging.ts

@@ -1,15 +0,0 @@
-import * as plugins from './tsdoc.plugins';
-
-export const logger = new plugins.smartlog.Smartlog({
-  logContext: {
-    company: 'Some Company',
-    companyunit: 'Some CompanyUnit',
-    containerName: 'Some Containername',
-    environment: 'local',
-    runtime: 'node',
-    zone: 'gitzone'
-  },
-  minimumLogLevel: 'silly'
-});
-
-logger.addLogDestination(new plugins.smartlogDestinationLocal.DestinationLocal());

ts/tsdoc.paths.ts

@@ -1,5 +0,0 @@
-import * as plugins from './tsdoc.plugins';
-
-export const packageDir = plugins.path.join(__dirname, '../');
-export const cwd = process.cwd();
-export const binDir = plugins.path.join(packageDir, './node_modules/.bin');

ts/tsdoc.plugins.ts

@@ -1,18 +0,0 @@
-// node native
-import * as path from 'path';
-
-export { path };
-
-// pushrocks scope
-import * as smartcli from '@pushrocks/smartcli';
-import * as smartfile from '@pushrocks/smartfile';
-import * as smartlog from '@pushrocks/smartlog';
-import * as smartlogDestinationLocal from '@pushrocks/smartlog-destination-local';
-import * as smartshell from '@pushrocks/smartshell';
-
-export { smartcli, smartfile, smartlog, smartlogDestinationLocal, smartshell };
-
-// third party scope
-import * as typedoc from 'typedoc';
-
-export { typedoc };

tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "useDefineForClassFields": false,
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true
+  },
+  "exclude": [
+    "dist_*/**/*.d.ts"
+  ]
+}

tslint.json

@@ -1,17 +0,0 @@
-{
-  "extends": ["tslint:latest", "tslint-config-prettier"],
-  "rules": {
-    "semicolon": [true, "always"],
-    "no-console": false,
-    "ordered-imports": false,
-    "object-literal-sort-keys": false,
-    "member-ordering": {
-      "options":{
-        "order": [
-          "static-method"
-        ]
-      }
-    }
-  },
-  "defaultSeverity": "warning"
-}