feat(cache,build,docs): switch cache storage to SmartMongo and align build configuration with updated dependencies

This commit is contained in:
2026-03-28 07:21:29 +00:00
parent 6260e90b09
commit 7e567d78da
13 changed files with 116 additions and 63 deletions

View File

@@ -1,5 +1,13 @@
# Changelog
## 2026-03-28 - 2.13.0 - feat(cache,build,docs)
switch cache storage to SmartMongo and align build configuration with updated dependencies
- replaces LocalTsmDb usage with SmartMongo in the cache layer and related tests
- moves tsbundle and tswatch configuration from npmextra.json to .smartconfig.json
- updates Deno, frontend, and provider dependencies and adds @std/assert imports for tests
- refreshes README to document managed secrets, sync, action log, and the expanded handler and view set
## 2026-03-02 - 2.12.0 - feat(pipelines)
add pipelines view modes, time-range filtering, group aggregation, sorting, and job log polling

View File

@@ -12,15 +12,16 @@
"@std/fs": "jsr:@std/fs@^1.0.19",
"@std/encoding": "jsr:@std/encoding@^1.0.10",
"@api.global/typedrequest-interfaces": "npm:@api.global/typedrequest-interfaces@^3.0.19",
"@api.global/typedrequest": "npm:@api.global/typedrequest@^3.2.6",
"@api.global/typedserver": "npm:@api.global/typedserver@^8.3.1",
"@api.global/typedrequest": "npm:@api.global/typedrequest@^3.3.0",
"@api.global/typedserver": "npm:@api.global/typedserver@^8.4.6",
"@push.rocks/smartguard": "npm:@push.rocks/smartguard@^3.1.0",
"@push.rocks/smartjwt": "npm:@push.rocks/smartjwt@^2.2.1",
"@apiclient.xyz/gitea": "npm:@apiclient.xyz/gitea@^1.2.0",
"@apiclient.xyz/gitlab": "npm:@apiclient.xyz/gitlab@^2.2.0",
"@push.rocks/smartmongo": "npm:@push.rocks/smartmongo@^5.1.0",
"@push.rocks/smartdata": "npm:@push.rocks/smartdata@^7.0.15",
"@push.rocks/smartsecret": "npm:@push.rocks/smartsecret@^1.0.1"
"@apiclient.xyz/gitea": "npm:@apiclient.xyz/gitea@^1.5.0",
"@apiclient.xyz/gitlab": "npm:@apiclient.xyz/gitlab@^2.6.0",
"@push.rocks/smartmongo": "npm:@push.rocks/smartmongo@^7.0.0",
"@push.rocks/smartdata": "npm:@push.rocks/smartdata@^7.1.3",
"@push.rocks/smartsecret": "npm:@push.rocks/smartsecret@^1.0.2",
"@std/assert": "jsr:@std/assert@^1.0.0"
},
"compilerOptions": {
"lib": [

21
license Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) Task Venture Capital GmbH
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -14,15 +14,20 @@
"license": "MIT",
"dependencies": {
"@api.global/typedrequest-interfaces": "^3.0.19",
"@api.global/typedserver": "8.4.0",
"@api.global/typedsocket": "^4.1.0",
"@apiclient.xyz/gitea": "1.5.0",
"@apiclient.xyz/gitlab": "2.6.0",
"@design.estate/dees-catalog": "^3.43.3",
"@design.estate/dees-element": "^2.1.6"
"@api.global/typedserver": "^8.4.6",
"@api.global/typedsocket": "^4.1.2",
"@apiclient.xyz/gitea": "^1.5.0",
"@apiclient.xyz/gitlab": "^2.6.0",
"@design.estate/dees-catalog": "^3.49.0",
"@design.estate/dees-element": "^2.2.4"
},
"devDependencies": {
"@git.zone/tsbundle": "^2.9.0",
"@git.zone/tswatch": "^3.2.0"
"@git.zone/tsbundle": "^2.10.0",
"@git.zone/tswatch": "^3.3.2"
},
"pnpm": {
"onlyBuiltDependencies": [
"esbuild"
]
}
}

View File

@@ -1,18 +1,21 @@
# @serve.zone/gitops
A unified dashboard for managing Gitea and GitLab instances — browse projects, manage secrets, monitor CI/CD pipelines, stream build logs, and receive webhook notifications, all from a single app.
A unified dashboard for managing Gitea and GitLab instances — browse projects, manage secrets, monitor CI/CD pipelines, stream build logs, sync configurations, and receive webhook notifications, all from a single app. 🚀
## Issue Reporting and Security
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
## 🚀 Features
## Features
- **Multi-Provider** — Connect to Gitea and GitLab simultaneously via a unified provider abstraction
- **Secrets Management** — View, create, update, and delete CI/CD secrets across projects and groups
- **Pipeline Monitoring** — Browse pipelines, view jobs, retry failed builds, cancel running ones
- **Build Log Streaming** — Fetch and display raw job logs with monospace rendering
- **Webhook Integration** — Receive push/PR/pipeline events via `POST /webhook/:connectionId` and broadcast to all connected clients in real-time via WebSocket
- **Managed Secrets** — Define secrets once and push them to multiple providers/scopes automatically
- **Pipeline Monitoring** — Browse pipelines with time-range filtering, view modes, group aggregation, and sorting; view jobs, retry failed builds, cancel running ones
- **Build Log Streaming** — Fetch and display raw job logs with monospace rendering and live polling
- **Sync Configurations** — Define repo sync rules across providers with status tracking
- **Action Log** — Global audit trail of all operations across the system
- **Webhook Integration** — Receive push/PR/pipeline events via `POST /webhook/:connectionId` and broadcast to all connected WebSocket clients in real-time
- **Secrets Cache & Scanning** — Background scan service fetches and caches all secrets every 24h with upsert-based deduplication
- **Secure Token Storage** — Connection tokens stored in OS keychain via `@push.rocks/smartsecret` (encrypted file fallback), never in plaintext on disk
- **Auto-Refresh** — Frontend polls for updates every 30s, with manual refresh available on every view
@@ -24,7 +27,7 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community
- [Deno](https://deno.land/) v2+
- [pnpm](https://pnpm.io/) (for frontend deps and bundling)
- MongoDB-compatible database (auto-provisioned via `@push.rocks/smartmongo` / `LocalTsmDb`)
- MongoDB-compatible database (auto-provisioned via `@push.rocks/smartmongo`)
### Setup
@@ -74,7 +77,7 @@ Data is stored at `~/.serve.zone/gitops/`:
│ (HTTP/WS)│ (Providers) │ (24h background scan) │
├──────────┤ ├───────────────────────────┤
│ Handlers │ GiteaProvider│ CacheDb │
(9 total)│ GitLabProvider│ (LocalTsmDb + SmartdataDb)│
(12 total)│ GitLabProvider│ (SmartMongo + SmartdataDb)│
├──────────┴───────────────┴───────────────────────────┤
│ StorageManager │
│ (filesystem key-value store) │
@@ -87,7 +90,7 @@ Data is stored at `~/.serve.zone/gitops/`:
│ Frontend SPA │
│ Lit + dees-catalog + smartstate │
├──────────────────────────────────────────────────────┤
Dashboard │ 8 Views │ WebSocket Client │ Auto-Refresh│
│ Dashboard │ 11 Views │ WebSocket Client │ Auto-Refresh│
└──────────────────────────────────────────────────────┘
```
@@ -96,30 +99,33 @@ Data is stored at `~/.serve.zone/gitops/`:
- **`GitopsApp`** — Main orchestrator. Owns all subsystems, handles startup/shutdown lifecycle.
- **`ConnectionManager`** — CRUD for provider connections. Tokens secured in OS keychain. Background health checks on startup.
- **`BaseProvider`** → **`GiteaProvider`** / **`GitLabProvider`** — Unified interface over both APIs (projects, groups, secrets, pipelines, jobs, logs).
- **`OpsServer`** — TypedServer-based HTTP/WebSocket server with 9 handler modules:
- **`OpsServer`** — TypedServer-based HTTP/WebSocket server with 12 handler modules:
- `AdminHandler` — JWT-based auth (login/logout/verify)
- `ConnectionsHandler` — Connection CRUD + test
- `ProjectsHandler` / `GroupsHandler` — Browse repos and orgs
- `SecretsHandler` — Cache-first secret CRUD
- `PipelinesHandler` — Pipeline list/jobs/retry/cancel
- `ManagedSecretsHandler` — Managed secret definitions and push operations
- `PipelinesHandler` — Pipeline list/jobs/retry/cancel with filtering and aggregation
- `LogsHandler` — Job log fetch
- `WebhookHandler` — Custom HTTP route for incoming webhooks
- `ActionsHandler` — Force scan / scan status
- `ActionLogHandler` — Global audit trail queries
- `SyncHandler` — Repo sync configuration and status
- **`SecretsScanService`** — Background scanner with upsert-based deduplication. Runs on startup and every 24h.
- **`CacheDb`** — Embedded MongoDB via `LocalTsmDb` + `SmartdataDb`. TTL-based expiration with periodic cleanup.
- **`CacheDb`** — Embedded MongoDB via `SmartMongo` + `SmartdataDb`. TTL-based expiration with periodic cleanup.
- **`StorageManager`** — Filesystem-backed key-value store with atomic writes.
### Frontend (`ts_web/`)
- Built with [Lit](https://lit.dev/) web components and [@design.estate/dees-catalog](https://code.foss.global/design.estate/dees-catalog) UI library
- Reactive state management via `smartstate` (4 state parts: login, connections, data, UI)
- 8 tabbed views: Overview, Connections, Projects, Groups, Secrets, Pipelines, Build Log, Actions
- Built with [Lit](https://lit.dev/) web components using TC39 standard decorators and [@design.estate/dees-catalog](https://code.foss.global/design.estate/dees-catalog) UI library
- Reactive state management via `smartstate` (login, connections, data, UI state parts)
- 11 tabbed views: Overview, Connections, Projects, Groups, Secrets, Managed Secrets, Pipelines, Build Log, Actions, Action Log, Sync
- WebSocket client for real-time webhook push notifications
- Bundled to `ts_bundled/bundle.ts` via `@git.zone/tsbundle` (base64-encoded, committed to git)
### Shared Types (`ts_interfaces/`)
- `data/` — Data models (`IProject`, `ISecret`, `IPipeline`, `IIdentity`, etc.)
- `data/` — Data models (`IProject`, `ISecret`, `IPipeline`, `IIdentity`, `IConnection`, `ISyncConfig`, `IManagedSecret`, `IActionLogEntry`, etc.)
- `requests/` — TypedRequest interfaces for all RPC endpoints
## 🔌 API
@@ -156,17 +162,33 @@ All endpoints use [TypedRequest](https://code.foss.global/api.global/typedreques
| `getAllSecrets` | Get all secrets for a connection+scope (cache-first) |
| `getSecrets` | Get secrets for a specific entity (cache-first) |
| `createSecret` / `updateSecret` / `deleteSecret` | Secret CRUD |
| `getPipelines` | List pipelines for a project |
| `getPipelines` | List pipelines for a project (with time-range filtering) |
| `getPipelineJobs` | List jobs for a pipeline |
| `retryPipeline` / `cancelPipeline` | Pipeline actions |
| `getJobLog` | Fetch raw build log for a job |
### Managed Secrets
| Method | Description |
|---|---|
| `getManagedSecrets` | List managed secret definitions |
| `createManagedSecret` / `updateManagedSecret` / `deleteManagedSecret` | Managed secret CRUD |
### Sync
| Method | Description |
|---|---|
| `getSyncConfigs` | List sync configurations |
| `createSyncConfig` / `updateSyncConfig` / `deleteSyncConfig` | Sync config CRUD |
| `getRepoSyncStatus` | Get sync status for repos |
### Actions
| Method | Description |
|---|---|
| `forceScanSecrets` | Trigger immediate full secrets scan |
| `getScanStatus` | Get scan status, last result, timestamp |
| `getActionLog` | Query global audit trail |
### Webhooks
@@ -200,16 +222,16 @@ gitops/
├── mod.ts # Entry point
├── deno.json # Deno config + import map
├── package.json # npm metadata + scripts
├── npmextra.json # tsbundle + tswatch config
├── .smartconfig.json # tsbundle + tswatch config
├── html/index.html # HTML shell
├── ts/ # Backend
│ ├── classes/ # GitopsApp, ConnectionManager
│ ├── classes/ # GitopsApp, ConnectionManager, SyncManager, ActionLog
│ ├── providers/ # BaseProvider, GiteaProvider, GitLabProvider
│ ├── storage/ # StorageManager
│ ├── cache/ # CacheDb, CacheCleaner, SecretsScanService
│ │ └── documents/ # CachedProject, CachedSecret
│ └── opsserver/ # OpsServer + 9 handlers
│ ├── handlers/ # AdminHandler, SecretsHandler, etc.
│ └── opsserver/ # OpsServer + 12 handlers
│ ├── handlers/ # AdminHandler, SecretsHandler, SyncHandler, etc.
│ └── helpers/ # Guards (JWT verification)
├── ts_interfaces/ # Shared TypeScript types
│ ├── data/ # IProject, ISecret, IPipeline, etc.
@@ -217,14 +239,14 @@ gitops/
├── ts_web/ # Frontend SPA
│ ├── appstate.ts # Smartstate store + actions
│ └── elements/ # Lit web components
│ └── views/ # 8 view components
│ └── views/ # 11 view components
├── ts_bundled/bundle.ts # Embedded frontend (base64, committed)
└── test/ # Deno tests
```
## License and Legal Information
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file.
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [license](./license) file.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.

View File

@@ -1,4 +1,4 @@
import { assertEquals, assertExists } from 'https://deno.land/std@0.208.0/assert/mod.ts';
import { assertEquals, assertExists } from '@std/assert';
import { BaseProvider, GiteaProvider, GitLabProvider } from '../ts/providers/index.ts';
import { ConnectionManager } from '../ts/classes/connectionmanager.ts';
import { GitopsApp } from '../ts/classes/gitopsapp.ts';

View File

@@ -1,4 +1,4 @@
import { assertEquals, assertExists } from 'https://deno.land/std@0.208.0/assert/mod.ts';
import { assertEquals, assertExists } from '@std/assert';
import { StorageManager } from '../ts/storage/index.ts';
import * as smartsecret from '@push.rocks/smartsecret';

View File

@@ -1,22 +1,20 @@
import { assertEquals, assertExists } from 'https://deno.land/std@0.208.0/assert/mod.ts';
import { LocalTsmDb } from '@push.rocks/smartmongo';
import { assertEquals, assertExists } from '@std/assert';
import { SmartMongo } from '@push.rocks/smartmongo';
import { SmartdataDb, SmartDataDbDoc, Collection, svDb, unI } from '@push.rocks/smartdata';
Deno.test({
name: 'TsmDb spike: LocalTsmDb + SmartdataDb roundtrip',
name: 'TsmDb spike: SmartMongo + SmartdataDb roundtrip',
sanitizeOps: false,
sanitizeResources: false,
fn: async () => {
const tmpDir = await Deno.makeTempDir();
// 1. Start local MongoDB-compatible server
const localDb = new LocalTsmDb({ folderPath: tmpDir });
const { connectionUri } = await localDb.start();
assertExists(connectionUri);
const mongo = await SmartMongo.createAndStart();
const mongoDescriptor = await mongo.getMongoDescriptor();
assertExists(mongoDescriptor.mongoDbUrl);
// 2. Connect smartdata
const smartDb = new SmartdataDb({
mongoDbUrl: connectionUri,
mongoDbUrl: mongoDescriptor.mongoDbUrl,
mongoDbName: 'gitops_spike_test',
});
await smartDb.init();
@@ -52,8 +50,8 @@ Deno.test({
assertEquals(found.label, 'spike');
assertEquals(found.value, 42);
// 6. Cleanup — smartDb closes; localDb.stop() hangs under Deno, so fire-and-forget
// 6. Cleanup — smartDb closes; mongo.stop() may hang under Deno, so fire-and-forget
await smartDb.close();
localDb.stop().catch(() => {});
mongo.stop().catch(() => {});
},
});

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/gitops',
version: '2.12.0',
version: '2.13.0',
description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs'
}

View File

@@ -14,7 +14,7 @@ export interface ICacheDbOptions {
export class CacheDb {
private static instance: CacheDb | null = null;
private localTsmDb: InstanceType<typeof plugins.smartmongo.LocalTsmDb> | null = null;
private smartMongo: InstanceType<typeof plugins.smartmongo.SmartMongo> | null = null;
private smartdataDb: InstanceType<typeof plugins.smartdata.SmartdataDb> | null = null;
private options: Required<ICacheDbOptions>;
@@ -39,13 +39,11 @@ export class CacheDb {
async start(): Promise<void> {
logger.info('Starting CacheDb...');
this.localTsmDb = new plugins.smartmongo.LocalTsmDb({
folderPath: this.options.storagePath,
});
const { connectionUri } = await this.localTsmDb.start();
this.smartMongo = await plugins.smartmongo.SmartMongo.createAndStart();
const mongoDescriptor = await this.smartMongo.getMongoDescriptor();
this.smartdataDb = new plugins.smartdata.SmartdataDb({
mongoDbUrl: connectionUri,
mongoDbUrl: mongoDescriptor.mongoDbUrl,
mongoDbName: this.options.dbName,
});
await this.smartdataDb.init();
@@ -58,9 +56,9 @@ export class CacheDb {
await this.smartdataDb.close();
this.smartdataDb = null;
}
if (this.localTsmDb) {
// localDb.stop() may hang under Deno — fire-and-forget with timeout
const stopPromise = this.localTsmDb.stop().catch(() => {});
if (this.smartMongo) {
// smartMongo.stop() may hang under Deno — fire-and-forget with timeout
const stopPromise = this.smartMongo.stop().catch(() => {});
await Promise.race([
stopPromise,
new Promise<void>((resolve) => {
@@ -68,7 +66,7 @@ export class CacheDb {
Deno.unrefTimer(id);
}),
]);
this.localTsmDb = null;
this.smartMongo = null;
}
logger.success('CacheDb stopped');
}

File diff suppressed because one or more lines are too long

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/gitops',
version: '2.12.0',
version: '2.13.0',
description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs'
}