9 Commits

29 changed files with 1224 additions and 350 deletions

View File

@@ -1,5 +1,40 @@
# Changelog # Changelog
## 2026-03-28 - 2.13.0 - feat(cache,build,docs)
switch cache storage to SmartMongo and align build configuration with updated dependencies
- replaces LocalTsmDb usage with SmartMongo in the cache layer and related tests
- moves tsbundle and tswatch configuration from npmextra.json to .smartconfig.json
- updates Deno, frontend, and provider dependencies and adds @std/assert imports for tests
- refreshes README to document managed secrets, sync, action log, and the expanded handler and view set
## 2026-03-02 - 2.12.0 - feat(pipelines)
add pipelines view modes, time-range filtering, group aggregation, sorting, and job log polling
- Add view modes for pipelines: 'current', 'project', 'group', and 'error'; support timeRange and sortBy parameters on getPipelines requests and in the UI.
- Implement aggregated pipeline fetching across projects with batching, deduplication, and active-run prioritization (fetchCurrentPipelines, fetchAggregatedPipelines, fetchGroupPipelines, fetchErrorPipelines).
- Add filtering by time ranges (1h, 6h, 1d, 3d, 7d, 30d) and sorting options (created, duration, status) with status priority ordering.
- Extend BaseProvider API: add IPipelineListOptions (status, ref, source), add getGroupProjects, and update Gitea/GitLab providers to honor new options and include projectName mapping.
- Enhance web UI: new controls/state for viewMode, timeRange, sortBy, group selection, plus job log polling with auto-scroll and cleanup on disconnect.
- Bump dependencies: @apiclient.xyz/gitea 1.3.0 -> 1.4.0 and @apiclient.xyz/gitlab 2.4.0 -> 2.5.0.
## 2026-03-02 - 2.11.1 - fix(meta)
update repository metadata (non-functional change)
- Change was metadata-only (+1 -1) with no source code changes
- Current package.json version: 2.11.0 — recommend patch bump to 2.11.1
## 2026-03-02 - 2.11.0 - feat(sync)
add branch & tag listing support and improve sync mirroring and sync log routing
- Bump @apiclient.xyz/gitea to 1.3.0 and @apiclient.xyz/gitlab to 2.4.0
- Add IBranch and ITag interfaces and export them from ts_interfaces
- Add getBranches/getTags to BaseProvider and implement paginated branch/tag listing for Gitea and GitLab providers
- SyncManager now creates a temporary mirrors directory (RAM-backed), auto-cleans it on shutdown, and no longer requires a configured syncMirrorsPath (removed from paths and gitopsapp)
- Add refsMatch in SyncManager to accurately compare local branches/tags with target refs (uses for-each-ref and ls-remote) to avoid unnecessary pushes
- Introduce avatarUploadCache and other internal sync manager improvements
- Change log channel/tagging: sync log messages use 'sync' (was 'git') and TypedSocket broadcasts use a new 'syncLogClient' tag; web client now sets that tag when creating the socket
## 2026-02-28 - 2.10.0 - feat(managed-secrets) ## 2026-02-28 - 2.10.0 - feat(managed-secrets)
add centrally managed secrets with GITOPS_ prefix pushed to multiple targets add centrally managed secrets with GITOPS_ prefix pushed to multiple targets

View File

@@ -1,6 +1,6 @@
{ {
"name": "@serve.zone/gitops", "name": "@serve.zone/gitops",
"version": "2.8.0", "version": "2.13.0",
"exports": "./mod.ts", "exports": "./mod.ts",
"nodeModulesDir": "auto", "nodeModulesDir": "auto",
"tasks": { "tasks": {
@@ -12,15 +12,16 @@
"@std/fs": "jsr:@std/fs@^1.0.19", "@std/fs": "jsr:@std/fs@^1.0.19",
"@std/encoding": "jsr:@std/encoding@^1.0.10", "@std/encoding": "jsr:@std/encoding@^1.0.10",
"@api.global/typedrequest-interfaces": "npm:@api.global/typedrequest-interfaces@^3.0.19", "@api.global/typedrequest-interfaces": "npm:@api.global/typedrequest-interfaces@^3.0.19",
"@api.global/typedrequest": "npm:@api.global/typedrequest@^3.2.6", "@api.global/typedrequest": "npm:@api.global/typedrequest@^3.3.0",
"@api.global/typedserver": "npm:@api.global/typedserver@^8.3.1", "@api.global/typedserver": "npm:@api.global/typedserver@^8.4.6",
"@push.rocks/smartguard": "npm:@push.rocks/smartguard@^3.1.0", "@push.rocks/smartguard": "npm:@push.rocks/smartguard@^3.1.0",
"@push.rocks/smartjwt": "npm:@push.rocks/smartjwt@^2.2.1", "@push.rocks/smartjwt": "npm:@push.rocks/smartjwt@^2.2.1",
"@apiclient.xyz/gitea": "npm:@apiclient.xyz/gitea@^1.2.0", "@apiclient.xyz/gitea": "npm:@apiclient.xyz/gitea@^1.5.0",
"@apiclient.xyz/gitlab": "npm:@apiclient.xyz/gitlab@^2.2.0", "@apiclient.xyz/gitlab": "npm:@apiclient.xyz/gitlab@^2.6.0",
"@push.rocks/smartmongo": "npm:@push.rocks/smartmongo@^5.1.0", "@push.rocks/smartmongo": "npm:@push.rocks/smartmongo@^7.0.0",
"@push.rocks/smartdata": "npm:@push.rocks/smartdata@^7.0.15", "@push.rocks/smartdata": "npm:@push.rocks/smartdata@^7.1.3",
"@push.rocks/smartsecret": "npm:@push.rocks/smartsecret@^1.0.1" "@push.rocks/smartsecret": "npm:@push.rocks/smartsecret@^1.0.2",
"@std/assert": "jsr:@std/assert@^1.0.0"
}, },
"compilerOptions": { "compilerOptions": {
"lib": [ "lib": [

21
license Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) Task Venture Capital GmbH
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -1,6 +1,6 @@
{ {
"name": "@serve.zone/gitops", "name": "@serve.zone/gitops",
"version": "2.10.0", "version": "2.13.0",
"description": "GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs", "description": "GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs",
"main": "mod.ts", "main": "mod.ts",
"type": "module", "type": "module",
@@ -14,15 +14,20 @@
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@api.global/typedrequest-interfaces": "^3.0.19", "@api.global/typedrequest-interfaces": "^3.0.19",
"@api.global/typedserver": "8.4.0", "@api.global/typedserver": "^8.4.6",
"@api.global/typedsocket": "^4.1.0", "@api.global/typedsocket": "^4.1.2",
"@apiclient.xyz/gitea": "1.2.0", "@apiclient.xyz/gitea": "^1.5.0",
"@apiclient.xyz/gitlab": "2.3.0", "@apiclient.xyz/gitlab": "^2.6.0",
"@design.estate/dees-catalog": "^3.43.3", "@design.estate/dees-catalog": "^3.49.0",
"@design.estate/dees-element": "^2.1.6" "@design.estate/dees-element": "^2.2.4"
}, },
"devDependencies": { "devDependencies": {
"@git.zone/tsbundle": "^2.9.0", "@git.zone/tsbundle": "^2.10.0",
"@git.zone/tswatch": "^3.2.0" "@git.zone/tswatch": "^3.3.2"
},
"pnpm": {
"onlyBuiltDependencies": [
"esbuild"
]
} }
} }

BIN
pipelines-current-mode.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

View File

@@ -1,18 +1,21 @@
# @serve.zone/gitops # @serve.zone/gitops
A unified dashboard for managing Gitea and GitLab instances — browse projects, manage secrets, monitor CI/CD pipelines, stream build logs, and receive webhook notifications, all from a single app. A unified dashboard for managing Gitea and GitLab instances — browse projects, manage secrets, monitor CI/CD pipelines, stream build logs, sync configurations, and receive webhook notifications, all from a single app. 🚀
## Issue Reporting and Security ## Issue Reporting and Security
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly. For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
## 🚀 Features ## Features
- **Multi-Provider** — Connect to Gitea and GitLab simultaneously via a unified provider abstraction - **Multi-Provider** — Connect to Gitea and GitLab simultaneously via a unified provider abstraction
- **Secrets Management** — View, create, update, and delete CI/CD secrets across projects and groups - **Secrets Management** — View, create, update, and delete CI/CD secrets across projects and groups
- **Pipeline Monitoring** — Browse pipelines, view jobs, retry failed builds, cancel running ones - **Managed Secrets** — Define secrets once and push them to multiple providers/scopes automatically
- **Build Log Streaming** — Fetch and display raw job logs with monospace rendering - **Pipeline Monitoring** — Browse pipelines with time-range filtering, view modes, group aggregation, and sorting; view jobs, retry failed builds, cancel running ones
- **Webhook Integration** — Receive push/PR/pipeline events via `POST /webhook/:connectionId` and broadcast to all connected clients in real-time via WebSocket - **Build Log Streaming** — Fetch and display raw job logs with monospace rendering and live polling
- **Sync Configurations** — Define repo sync rules across providers with status tracking
- **Action Log** — Global audit trail of all operations across the system
- **Webhook Integration** — Receive push/PR/pipeline events via `POST /webhook/:connectionId` and broadcast to all connected WebSocket clients in real-time
- **Secrets Cache & Scanning** — Background scan service fetches and caches all secrets every 24h with upsert-based deduplication - **Secrets Cache & Scanning** — Background scan service fetches and caches all secrets every 24h with upsert-based deduplication
- **Secure Token Storage** — Connection tokens stored in OS keychain via `@push.rocks/smartsecret` (encrypted file fallback), never in plaintext on disk - **Secure Token Storage** — Connection tokens stored in OS keychain via `@push.rocks/smartsecret` (encrypted file fallback), never in plaintext on disk
- **Auto-Refresh** — Frontend polls for updates every 30s, with manual refresh available on every view - **Auto-Refresh** — Frontend polls for updates every 30s, with manual refresh available on every view
@@ -24,7 +27,7 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community
- [Deno](https://deno.land/) v2+ - [Deno](https://deno.land/) v2+
- [pnpm](https://pnpm.io/) (for frontend deps and bundling) - [pnpm](https://pnpm.io/) (for frontend deps and bundling)
- MongoDB-compatible database (auto-provisioned via `@push.rocks/smartmongo` / `LocalTsmDb`) - MongoDB-compatible database (auto-provisioned via `@push.rocks/smartmongo`)
### Setup ### Setup
@@ -74,7 +77,7 @@ Data is stored at `~/.serve.zone/gitops/`:
│ (HTTP/WS)│ (Providers) │ (24h background scan) │ │ (HTTP/WS)│ (Providers) │ (24h background scan) │
├──────────┤ ├───────────────────────────┤ ├──────────┤ ├───────────────────────────┤
│ Handlers │ GiteaProvider│ CacheDb │ │ Handlers │ GiteaProvider│ CacheDb │
(9 total)│ GitLabProvider│ (LocalTsmDb + SmartdataDb)│ (12 total)│ GitLabProvider│ (SmartMongo + SmartdataDb)│
├──────────┴───────────────┴───────────────────────────┤ ├──────────┴───────────────┴───────────────────────────┤
│ StorageManager │ │ StorageManager │
│ (filesystem key-value store) │ │ (filesystem key-value store) │
@@ -87,7 +90,7 @@ Data is stored at `~/.serve.zone/gitops/`:
│ Frontend SPA │ │ Frontend SPA │
│ Lit + dees-catalog + smartstate │ │ Lit + dees-catalog + smartstate │
├──────────────────────────────────────────────────────┤ ├──────────────────────────────────────────────────────┤
Dashboard │ 8 Views │ WebSocket Client │ Auto-Refresh│ │ Dashboard │ 11 Views │ WebSocket Client │ Auto-Refresh│
└──────────────────────────────────────────────────────┘ └──────────────────────────────────────────────────────┘
``` ```
@@ -96,30 +99,33 @@ Data is stored at `~/.serve.zone/gitops/`:
- **`GitopsApp`** — Main orchestrator. Owns all subsystems, handles startup/shutdown lifecycle. - **`GitopsApp`** — Main orchestrator. Owns all subsystems, handles startup/shutdown lifecycle.
- **`ConnectionManager`** — CRUD for provider connections. Tokens secured in OS keychain. Background health checks on startup. - **`ConnectionManager`** — CRUD for provider connections. Tokens secured in OS keychain. Background health checks on startup.
- **`BaseProvider`** → **`GiteaProvider`** / **`GitLabProvider`** — Unified interface over both APIs (projects, groups, secrets, pipelines, jobs, logs). - **`BaseProvider`** → **`GiteaProvider`** / **`GitLabProvider`** — Unified interface over both APIs (projects, groups, secrets, pipelines, jobs, logs).
- **`OpsServer`** — TypedServer-based HTTP/WebSocket server with 9 handler modules: - **`OpsServer`** — TypedServer-based HTTP/WebSocket server with 12 handler modules:
- `AdminHandler` — JWT-based auth (login/logout/verify) - `AdminHandler` — JWT-based auth (login/logout/verify)
- `ConnectionsHandler` — Connection CRUD + test - `ConnectionsHandler` — Connection CRUD + test
- `ProjectsHandler` / `GroupsHandler` — Browse repos and orgs - `ProjectsHandler` / `GroupsHandler` — Browse repos and orgs
- `SecretsHandler` — Cache-first secret CRUD - `SecretsHandler` — Cache-first secret CRUD
- `PipelinesHandler` — Pipeline list/jobs/retry/cancel - `ManagedSecretsHandler` — Managed secret definitions and push operations
- `PipelinesHandler` — Pipeline list/jobs/retry/cancel with filtering and aggregation
- `LogsHandler` — Job log fetch - `LogsHandler` — Job log fetch
- `WebhookHandler` — Custom HTTP route for incoming webhooks - `WebhookHandler` — Custom HTTP route for incoming webhooks
- `ActionsHandler` — Force scan / scan status - `ActionsHandler` — Force scan / scan status
- `ActionLogHandler` — Global audit trail queries
- `SyncHandler` — Repo sync configuration and status
- **`SecretsScanService`** — Background scanner with upsert-based deduplication. Runs on startup and every 24h. - **`SecretsScanService`** — Background scanner with upsert-based deduplication. Runs on startup and every 24h.
- **`CacheDb`** — Embedded MongoDB via `LocalTsmDb` + `SmartdataDb`. TTL-based expiration with periodic cleanup. - **`CacheDb`** — Embedded MongoDB via `SmartMongo` + `SmartdataDb`. TTL-based expiration with periodic cleanup.
- **`StorageManager`** — Filesystem-backed key-value store with atomic writes. - **`StorageManager`** — Filesystem-backed key-value store with atomic writes.
### Frontend (`ts_web/`) ### Frontend (`ts_web/`)
- Built with [Lit](https://lit.dev/) web components and [@design.estate/dees-catalog](https://code.foss.global/design.estate/dees-catalog) UI library - Built with [Lit](https://lit.dev/) web components using TC39 standard decorators and [@design.estate/dees-catalog](https://code.foss.global/design.estate/dees-catalog) UI library
- Reactive state management via `smartstate` (4 state parts: login, connections, data, UI) - Reactive state management via `smartstate` (login, connections, data, UI state parts)
- 8 tabbed views: Overview, Connections, Projects, Groups, Secrets, Pipelines, Build Log, Actions - 11 tabbed views: Overview, Connections, Projects, Groups, Secrets, Managed Secrets, Pipelines, Build Log, Actions, Action Log, Sync
- WebSocket client for real-time webhook push notifications - WebSocket client for real-time webhook push notifications
- Bundled to `ts_bundled/bundle.ts` via `@git.zone/tsbundle` (base64-encoded, committed to git) - Bundled to `ts_bundled/bundle.ts` via `@git.zone/tsbundle` (base64-encoded, committed to git)
### Shared Types (`ts_interfaces/`) ### Shared Types (`ts_interfaces/`)
- `data/` — Data models (`IProject`, `ISecret`, `IPipeline`, `IIdentity`, etc.) - `data/` — Data models (`IProject`, `ISecret`, `IPipeline`, `IIdentity`, `IConnection`, `ISyncConfig`, `IManagedSecret`, `IActionLogEntry`, etc.)
- `requests/` — TypedRequest interfaces for all RPC endpoints - `requests/` — TypedRequest interfaces for all RPC endpoints
## 🔌 API ## 🔌 API
@@ -156,17 +162,33 @@ All endpoints use [TypedRequest](https://code.foss.global/api.global/typedreques
| `getAllSecrets` | Get all secrets for a connection+scope (cache-first) | | `getAllSecrets` | Get all secrets for a connection+scope (cache-first) |
| `getSecrets` | Get secrets for a specific entity (cache-first) | | `getSecrets` | Get secrets for a specific entity (cache-first) |
| `createSecret` / `updateSecret` / `deleteSecret` | Secret CRUD | | `createSecret` / `updateSecret` / `deleteSecret` | Secret CRUD |
| `getPipelines` | List pipelines for a project | | `getPipelines` | List pipelines for a project (with time-range filtering) |
| `getPipelineJobs` | List jobs for a pipeline | | `getPipelineJobs` | List jobs for a pipeline |
| `retryPipeline` / `cancelPipeline` | Pipeline actions | | `retryPipeline` / `cancelPipeline` | Pipeline actions |
| `getJobLog` | Fetch raw build log for a job | | `getJobLog` | Fetch raw build log for a job |
### Managed Secrets
| Method | Description |
|---|---|
| `getManagedSecrets` | List managed secret definitions |
| `createManagedSecret` / `updateManagedSecret` / `deleteManagedSecret` | Managed secret CRUD |
### Sync
| Method | Description |
|---|---|
| `getSyncConfigs` | List sync configurations |
| `createSyncConfig` / `updateSyncConfig` / `deleteSyncConfig` | Sync config CRUD |
| `getRepoSyncStatus` | Get sync status for repos |
### Actions ### Actions
| Method | Description | | Method | Description |
|---|---| |---|---|
| `forceScanSecrets` | Trigger immediate full secrets scan | | `forceScanSecrets` | Trigger immediate full secrets scan |
| `getScanStatus` | Get scan status, last result, timestamp | | `getScanStatus` | Get scan status, last result, timestamp |
| `getActionLog` | Query global audit trail |
### Webhooks ### Webhooks
@@ -200,16 +222,16 @@ gitops/
├── mod.ts # Entry point ├── mod.ts # Entry point
├── deno.json # Deno config + import map ├── deno.json # Deno config + import map
├── package.json # npm metadata + scripts ├── package.json # npm metadata + scripts
├── npmextra.json # tsbundle + tswatch config ├── .smartconfig.json # tsbundle + tswatch config
├── html/index.html # HTML shell ├── html/index.html # HTML shell
├── ts/ # Backend ├── ts/ # Backend
│ ├── classes/ # GitopsApp, ConnectionManager │ ├── classes/ # GitopsApp, ConnectionManager, SyncManager, ActionLog
│ ├── providers/ # BaseProvider, GiteaProvider, GitLabProvider │ ├── providers/ # BaseProvider, GiteaProvider, GitLabProvider
│ ├── storage/ # StorageManager │ ├── storage/ # StorageManager
│ ├── cache/ # CacheDb, CacheCleaner, SecretsScanService │ ├── cache/ # CacheDb, CacheCleaner, SecretsScanService
│ │ └── documents/ # CachedProject, CachedSecret │ │ └── documents/ # CachedProject, CachedSecret
│ └── opsserver/ # OpsServer + 9 handlers │ └── opsserver/ # OpsServer + 12 handlers
│ ├── handlers/ # AdminHandler, SecretsHandler, etc. │ ├── handlers/ # AdminHandler, SecretsHandler, SyncHandler, etc.
│ └── helpers/ # Guards (JWT verification) │ └── helpers/ # Guards (JWT verification)
├── ts_interfaces/ # Shared TypeScript types ├── ts_interfaces/ # Shared TypeScript types
│ ├── data/ # IProject, ISecret, IPipeline, etc. │ ├── data/ # IProject, ISecret, IPipeline, etc.
@@ -217,14 +239,14 @@ gitops/
├── ts_web/ # Frontend SPA ├── ts_web/ # Frontend SPA
│ ├── appstate.ts # Smartstate store + actions │ ├── appstate.ts # Smartstate store + actions
│ └── elements/ # Lit web components │ └── elements/ # Lit web components
│ └── views/ # 8 view components │ └── views/ # 11 view components
├── ts_bundled/bundle.ts # Embedded frontend (base64, committed) ├── ts_bundled/bundle.ts # Embedded frontend (base64, committed)
└── test/ # Deno tests └── test/ # Deno tests
``` ```
## License and Legal Information ## License and Legal Information
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file. This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [license](./license) file.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file. **Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.

View File

@@ -1,4 +1,4 @@
import { assertEquals, assertExists } from 'https://deno.land/std@0.208.0/assert/mod.ts'; import { assertEquals, assertExists } from '@std/assert';
import { BaseProvider, GiteaProvider, GitLabProvider } from '../ts/providers/index.ts'; import { BaseProvider, GiteaProvider, GitLabProvider } from '../ts/providers/index.ts';
import { ConnectionManager } from '../ts/classes/connectionmanager.ts'; import { ConnectionManager } from '../ts/classes/connectionmanager.ts';
import { GitopsApp } from '../ts/classes/gitopsapp.ts'; import { GitopsApp } from '../ts/classes/gitopsapp.ts';

View File

@@ -1,4 +1,4 @@
import { assertEquals, assertExists } from 'https://deno.land/std@0.208.0/assert/mod.ts'; import { assertEquals, assertExists } from '@std/assert';
import { StorageManager } from '../ts/storage/index.ts'; import { StorageManager } from '../ts/storage/index.ts';
import * as smartsecret from '@push.rocks/smartsecret'; import * as smartsecret from '@push.rocks/smartsecret';

View File

@@ -1,22 +1,20 @@
import { assertEquals, assertExists } from 'https://deno.land/std@0.208.0/assert/mod.ts'; import { assertEquals, assertExists } from '@std/assert';
import { LocalTsmDb } from '@push.rocks/smartmongo'; import { SmartMongo } from '@push.rocks/smartmongo';
import { SmartdataDb, SmartDataDbDoc, Collection, svDb, unI } from '@push.rocks/smartdata'; import { SmartdataDb, SmartDataDbDoc, Collection, svDb, unI } from '@push.rocks/smartdata';
Deno.test({ Deno.test({
name: 'TsmDb spike: LocalTsmDb + SmartdataDb roundtrip', name: 'TsmDb spike: SmartMongo + SmartdataDb roundtrip',
sanitizeOps: false, sanitizeOps: false,
sanitizeResources: false, sanitizeResources: false,
fn: async () => { fn: async () => {
const tmpDir = await Deno.makeTempDir();
// 1. Start local MongoDB-compatible server // 1. Start local MongoDB-compatible server
const localDb = new LocalTsmDb({ folderPath: tmpDir }); const mongo = await SmartMongo.createAndStart();
const { connectionUri } = await localDb.start(); const mongoDescriptor = await mongo.getMongoDescriptor();
assertExists(connectionUri); assertExists(mongoDescriptor.mongoDbUrl);
// 2. Connect smartdata // 2. Connect smartdata
const smartDb = new SmartdataDb({ const smartDb = new SmartdataDb({
mongoDbUrl: connectionUri, mongoDbUrl: mongoDescriptor.mongoDbUrl,
mongoDbName: 'gitops_spike_test', mongoDbName: 'gitops_spike_test',
}); });
await smartDb.init(); await smartDb.init();
@@ -52,8 +50,8 @@ Deno.test({
assertEquals(found.label, 'spike'); assertEquals(found.label, 'spike');
assertEquals(found.value, 42); assertEquals(found.value, 42);
// 6. Cleanup — smartDb closes; localDb.stop() hangs under Deno, so fire-and-forget // 6. Cleanup — smartDb closes; mongo.stop() may hang under Deno, so fire-and-forget
await smartDb.close(); await smartDb.close();
localDb.stop().catch(() => {}); mongo.stop().catch(() => {});
}, },
}); });

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@serve.zone/gitops', name: '@serve.zone/gitops',
version: '2.8.0', version: '2.13.0',
description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs' description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs'
} }

View File

@@ -14,7 +14,7 @@ export interface ICacheDbOptions {
export class CacheDb { export class CacheDb {
private static instance: CacheDb | null = null; private static instance: CacheDb | null = null;
private localTsmDb: InstanceType<typeof plugins.smartmongo.LocalTsmDb> | null = null; private smartMongo: InstanceType<typeof plugins.smartmongo.SmartMongo> | null = null;
private smartdataDb: InstanceType<typeof plugins.smartdata.SmartdataDb> | null = null; private smartdataDb: InstanceType<typeof plugins.smartdata.SmartdataDb> | null = null;
private options: Required<ICacheDbOptions>; private options: Required<ICacheDbOptions>;
@@ -39,13 +39,11 @@ export class CacheDb {
async start(): Promise<void> { async start(): Promise<void> {
logger.info('Starting CacheDb...'); logger.info('Starting CacheDb...');
this.localTsmDb = new plugins.smartmongo.LocalTsmDb({ this.smartMongo = await plugins.smartmongo.SmartMongo.createAndStart();
folderPath: this.options.storagePath, const mongoDescriptor = await this.smartMongo.getMongoDescriptor();
});
const { connectionUri } = await this.localTsmDb.start();
this.smartdataDb = new plugins.smartdata.SmartdataDb({ this.smartdataDb = new plugins.smartdata.SmartdataDb({
mongoDbUrl: connectionUri, mongoDbUrl: mongoDescriptor.mongoDbUrl,
mongoDbName: this.options.dbName, mongoDbName: this.options.dbName,
}); });
await this.smartdataDb.init(); await this.smartdataDb.init();
@@ -58,9 +56,9 @@ export class CacheDb {
await this.smartdataDb.close(); await this.smartdataDb.close();
this.smartdataDb = null; this.smartdataDb = null;
} }
if (this.localTsmDb) { if (this.smartMongo) {
// localDb.stop() may hang under Deno — fire-and-forget with timeout // smartMongo.stop() may hang under Deno — fire-and-forget with timeout
const stopPromise = this.localTsmDb.stop().catch(() => {}); const stopPromise = this.smartMongo.stop().catch(() => {});
await Promise.race([ await Promise.race([
stopPromise, stopPromise,
new Promise<void>((resolve) => { new Promise<void>((resolve) => {
@@ -68,7 +66,7 @@ export class CacheDb {
Deno.unrefTimer(id); Deno.unrefTimer(id);
}), }),
]); ]);
this.localTsmDb = null; this.smartMongo = null;
} }
logger.success('CacheDb stopped'); logger.success('CacheDb stopped');
} }

View File

@@ -230,7 +230,7 @@ export class ConnectionManager {
try { try {
if (conn.providerType === 'gitlab') { if (conn.providerType === 'gitlab') {
const gitlabClient = new plugins.gitlabClient.GitLabClient(conn.baseUrl, conn.token); const gitlabClient = new plugins.gitlabClient.GitLabClient(conn.baseUrl, conn.token);
const group = await gitlabClient.getGroupByPath(conn.groupFilter); const group = await gitlabClient.getGroup(conn.groupFilter);
conn.groupFilterId = String(group.id); conn.groupFilterId = String(group.id);
logger.info(`Resolved group filter "${conn.groupFilter}" to ID ${conn.groupFilterId}`); logger.info(`Resolved group filter "${conn.groupFilter}" to ID ${conn.groupFilterId}`);
} else { } else {

View File

@@ -70,7 +70,6 @@ export class GitopsApp {
this.storageManager, this.storageManager,
this.connectionManager, this.connectionManager,
this.actionLog, this.actionLog,
this.paths.syncMirrorsPath,
); );
await this.syncManager.init(); await this.syncManager.init();

View File

@@ -4,6 +4,7 @@ import type * as interfaces from '../../ts_interfaces/index.ts';
import type { ConnectionManager } from './connectionmanager.ts'; import type { ConnectionManager } from './connectionmanager.ts';
import type { ActionLog } from './actionlog.ts'; import type { ActionLog } from './actionlog.ts';
import type { StorageManager } from '../storage/index.ts'; import type { StorageManager } from '../storage/index.ts';
import type { BaseProvider } from '../providers/classes.baseprovider.ts';
const SYNC_PREFIX = '/sync/'; const SYNC_PREFIX = '/sync/';
const SYNC_STATUS_PREFIX = '/sync-status/'; const SYNC_STATUS_PREFIX = '/sync-status/';
@@ -19,15 +20,19 @@ export class SyncManager {
private runningSync: Set<string> = new Set(); private runningSync: Set<string> = new Set();
private syncedGroupMeta: Set<string> = new Set(); private syncedGroupMeta: Set<string> = new Set();
private currentSyncConfig: interfaces.data.ISyncConfig | null = null; private currentSyncConfig: interfaces.data.ISyncConfig | null = null;
private avatarUploadCache: Map<string, string> = new Map();
private mirrorsPath = '';
constructor( constructor(
private storageManager: StorageManager, private storageManager: StorageManager,
private connectionManager: ConnectionManager, private connectionManager: ConnectionManager,
private actionLog: ActionLog, private actionLog: ActionLog,
private mirrorsPath: string,
) {} ) {}
async init(): Promise<void> { async init(): Promise<void> {
// Create temp directory for mirrors (RAM-backed on most Linux systems via tmpfs)
this.mirrorsPath = await Deno.makeTempDir({ prefix: 'gitops-mirrors-' });
await this.loadConfigs(); await this.loadConfigs();
for (const config of this.configs) { for (const config of this.configs) {
if (config.status === 'active') { if (config.status === 'active') {
@@ -44,6 +49,12 @@ export class SyncManager {
clearInterval(timer); clearInterval(timer);
} }
this.timers.clear(); this.timers.clear();
// Clean up temp mirrors directory
if (this.mirrorsPath) {
try {
await Deno.remove(this.mirrorsPath, { recursive: true });
} catch { /* may already be gone */ }
}
} }
// ============================================================================ // ============================================================================
@@ -290,7 +301,7 @@ export class SyncManager {
const batch = projects.slice(i, i + CONCURRENCY); const batch = projects.slice(i, i + CONCURRENCY);
await Promise.all(batch.map(async (project) => { await Promise.all(batch.map(async (project) => {
try { try {
logger.syncLog('info', `Syncing ${project.fullPath}...`, 'git'); logger.syncLog('info', `Syncing ${project.fullPath}...`, 'sync');
await this.syncRepo(config, project, sourceConn, targetConn); await this.syncRepo(config, project, sourceConn, targetConn);
synced++; synced++;
await this.updateRepoStatus(config.id, project.fullPath, { await this.updateRepoStatus(config.id, project.fullPath, {
@@ -298,7 +309,7 @@ export class SyncManager {
lastSyncAt: Date.now(), lastSyncAt: Date.now(),
lastSyncError: undefined, lastSyncError: undefined,
}); });
logger.syncLog('success', `Synced ${project.fullPath}`, 'git'); logger.syncLog('success', `Synced ${project.fullPath}`, 'sync');
} catch (err) { } catch (err) {
const errMsg = err instanceof Error ? err.message : String(err); const errMsg = err instanceof Error ? err.message : String(err);
await this.updateRepoStatus(config.id, project.fullPath, { await this.updateRepoStatus(config.id, project.fullPath, {
@@ -306,7 +317,7 @@ export class SyncManager {
lastSyncError: errMsg, lastSyncError: errMsg,
lastSyncAt: Date.now(), lastSyncAt: Date.now(),
}); });
logger.syncLog('error', `Sync failed for ${project.fullPath}: ${errMsg}`, 'git'); logger.syncLog('error', `Sync failed for ${project.fullPath}: ${errMsg}`, 'sync');
} }
})); }));
} }
@@ -372,21 +383,38 @@ export class SyncManager {
// Ensure target group/project hierarchy exists // Ensure target group/project hierarchy exists
await this.ensureTargetExists(targetConn, targetFullPath, project, sourceConn, sourceConn.groupFilter, config.targetGroupOffset); await this.ensureTargetExists(targetConn, targetFullPath, project, sourceConn, sourceConn.groupFilter, config.targetGroupOffset);
// API-based ref comparison (fast path — avoids git clone when refs already match)
const sourceProvider = this.connectionManager.getProvider(sourceConn.id);
const targetProvider = this.connectionManager.getProvider(targetConn.id);
const apiRefsMatch = await this.refsMatchViaApi(
sourceProvider, targetProvider, project.fullPath, targetFullPath,
);
if (apiRefsMatch === true) {
logger.syncLog('info', `Refs match via API for ${project.fullPath}, skipping git`, 'api');
await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetFullPath);
return;
}
// Clone or fetch from source // Clone or fetch from source
try { try {
const exists = await this.dirExists(mirrorDir); const exists = await this.dirExists(mirrorDir);
if (!exists) { if (!exists) {
await Deno.mkdir(mirrorDir, { recursive: true }); await Deno.mkdir(mirrorDir, { recursive: true });
await this.runGit(['clone', '--bare', sourceUrl, '.'], mirrorDir); await this.runGit(['clone', '--bare', sourceUrl, '.'], mirrorDir);
} else {
// Update source remote URL in case it changed
try {
await this.runGit(['remote', 'set-url', 'origin', sourceUrl], mirrorDir);
} catch {
// Ignore errors
}
await this.runGit(['fetch', '--prune', 'origin'], mirrorDir);
} }
// Ensure fetch refspec is configured (bare clones don't set one by default,
// which prevents tracking branch renames like master -> main)
await this.runGit(
['config', 'remote.origin.fetch', '+refs/heads/*:refs/heads/*'], mirrorDir,
);
// Update source remote URL in case connection changed
try {
await this.runGit(['remote', 'set-url', 'origin', sourceUrl], mirrorDir);
} catch {
// Ignore errors
}
// Fetch latest refs from source (--prune removes branches deleted on remote)
await this.runGit(['fetch', '--prune', 'origin'], mirrorDir);
} catch (err: any) { } catch (err: any) {
const msg = err instanceof Error ? err.message : String(err); const msg = err instanceof Error ? err.message : String(err);
if (msg.includes("couldn't find remote ref HEAD")) { if (msg.includes("couldn't find remote ref HEAD")) {
@@ -421,26 +449,33 @@ export class SyncManager {
}); });
} }
// Phase 1: push all refs without pruning (ensures target has all source branches) // Compare refs to determine if push is needed
await this.runGit([ const refsAlreadyMatch = !isUnrelated && await this.refsMatch(mirrorDir);
'push', 'target',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*',
], mirrorDir);
// Phase 2: sync default_branch now that all branches exist on target if (refsAlreadyMatch) {
await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath, targetFullPath); logger.syncLog('info', `Refs already match for ${project.fullPath}, skipping push`, 'api');
} else {
// Phase 1: push all refs without pruning (ensures target has all source branches)
await this.runGit([
'push', 'target',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*',
], mirrorDir);
// Phase 2b: unprotect stale branches on target so --prune can delete them // Phase 2: sync default_branch now that all branches exist on target
await this.unprotectStaleBranches(targetConn, targetFullPath, mirrorDir); await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath, targetFullPath);
// Phase 3: push with --prune to remove stale branches (safe now that default_branch is correct) // Phase 2b: unprotect stale branches on target so --prune can delete them
await this.runGit([ await this.unprotectStaleBranches(targetConn, targetFullPath, mirrorDir);
'push', 'target',
'+refs/heads/*:refs/heads/*', // Phase 3: push with --prune to remove stale branches (safe now that default_branch is correct)
'+refs/tags/*:refs/tags/*', await this.runGit([
'--prune', 'push', 'target',
], mirrorDir); '+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*',
'--prune',
], mirrorDir);
}
// Sync project metadata (description, visibility, topics, default_branch, avatar) // Sync project metadata (description, visibility, topics, default_branch, avatar)
await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetFullPath); await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetFullPath);
@@ -487,7 +522,7 @@ export class SyncManager {
for (const segment of groupSegments) { for (const segment of groupSegments) {
currentPath = currentPath ? `${currentPath}/${segment}` : segment; currentPath = currentPath ? `${currentPath}/${segment}` : segment;
try { try {
const group = await client.getGroupByPath(currentPath); const group = await client.getGroup(currentPath);
parentId = group.id; parentId = group.id;
} catch { } catch {
// Group doesn't exist — create it // Group doesn't exist — create it
@@ -498,7 +533,7 @@ export class SyncManager {
} catch (createErr: any) { } catch (createErr: any) {
// 409 = already exists (race condition), try fetching again // 409 = already exists (race condition), try fetching again
if (String(createErr).includes('409') || String(createErr).includes('already')) { if (String(createErr).includes('409') || String(createErr).includes('already')) {
const group = await client.getGroupByPath(currentPath); const group = await client.getGroup(currentPath);
parentId = group.id; parentId = group.id;
} else { } else {
throw createErr; throw createErr;
@@ -523,7 +558,7 @@ export class SyncManager {
try { try {
// Check if project exists by path // Check if project exists by path
await client.getGroupByPath(projectPath); await client.getGroup(projectPath);
// If this succeeds, it's actually a group, not a project... unlikely but handle // If this succeeds, it's actually a group, not a project... unlikely but handle
} catch { } catch {
// Project doesn't exist as a group path; try creating it // Project doesn't exist as a group path; try creating it
@@ -1007,10 +1042,14 @@ export class SyncManager {
if (data[0] === 0x89 && data[1] === 0x50) return 'image/png'; if (data[0] === 0x89 && data[1] === 0x50) return 'image/png';
if (data[0] === 0xFF && data[1] === 0xD8) return 'image/jpeg'; if (data[0] === 0xFF && data[1] === 0xD8) return 'image/jpeg';
if (data[0] === 0x47 && data[1] === 0x49) return 'image/gif'; if (data[0] === 0x47 && data[1] === 0x49) return 'image/gif';
// SVG: text-based XML, no magic bytes — check content
const textStart = new TextDecoder().decode(data.slice(0, 200));
if (textStart.includes('<svg') || textStart.includes('<?xml')) return 'image/svg+xml';
// Fallback: check URL extension // Fallback: check URL extension
if (url.includes('.png')) return 'image/png'; if (url.includes('.png')) return 'image/png';
if (url.includes('.jpg') || url.includes('.jpeg')) return 'image/jpeg'; if (url.includes('.jpg') || url.includes('.jpeg')) return 'image/jpeg';
if (url.includes('.gif')) return 'image/gif'; if (url.includes('.gif')) return 'image/gif';
if (url.includes('.svg')) return 'image/svg+xml';
return 'image/png'; // default return 'image/png'; // default
} }
@@ -1068,7 +1107,7 @@ export class SyncManager {
if (!targetProject) return; if (!targetProject) return;
const client = new plugins.gitlabClient.GitLabClient(targetConn.baseUrl, targetConn.token); const client = new plugins.gitlabClient.GitLabClient(targetConn.baseUrl, targetConn.token);
const protectedBranches = await client.getProtectedBranches(targetProject.id); const protectedBranches = await client.requestGetProtectedBranches(targetProject.id);
if (protectedBranches.length === 0) return; if (protectedBranches.length === 0) return;
// Get list of branches in the local mirror (= source branches) // Get list of branches in the local mirror (= source branches)
@@ -1080,7 +1119,7 @@ export class SyncManager {
for (const pb of protectedBranches) { for (const pb of protectedBranches) {
if (!localBranches.has(pb.name)) { if (!localBranches.has(pb.name)) {
logger.syncLog('info', `Unprotecting stale branch "${pb.name}" on ${targetFullPath}`, 'api'); logger.syncLog('info', `Unprotecting stale branch "${pb.name}" on ${targetFullPath}`, 'api');
await client.unprotectBranch(targetProject.id, pb.name); await client.requestUnprotectBranch(targetProject.id, pb.name);
} }
} }
} catch (err) { } catch (err) {
@@ -1146,7 +1185,6 @@ export class SyncManager {
if (sourceGroup) { if (sourceGroup) {
const groupMeta = this.extractGroupMeta(sourceConn, sourceGroup); const groupMeta = this.extractGroupMeta(sourceConn, sourceGroup);
if (groupMeta.avatarUrl) { if (groupMeta.avatarUrl) {
logger.syncLog('info', `Applying group avatar to ${targetFullPath}`, 'api');
await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, groupMeta.avatarUrl, targetProject); await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, groupMeta.avatarUrl, targetProject);
groupAvatarApplied = true; groupAvatarApplied = true;
} }
@@ -1293,24 +1331,41 @@ export class SyncManager {
meta: { description: string; visibility: string; topics: string[]; defaultBranch: string }, meta: { description: string; visibility: string; topics: string[]; defaultBranch: string },
): Promise<void> { ): Promise<void> {
if (conn.providerType === 'gitlab') { if (conn.providerType === 'gitlab') {
// Update description, visibility, topics (always safe)
await this.rawApiCall(conn, 'PUT', `/api/v4/projects/${rawProject.id}`, { await this.rawApiCall(conn, 'PUT', `/api/v4/projects/${rawProject.id}`, {
description: meta.description, description: meta.description,
visibility: this.normalizeVisibility(meta.visibility), visibility: this.normalizeVisibility(meta.visibility),
topics: meta.topics, topics: meta.topics,
default_branch: meta.defaultBranch,
}); });
// Update default_branch separately — may fail if the branch doesn't exist in git
try {
await this.rawApiCall(conn, 'PUT', `/api/v4/projects/${rawProject.id}`, {
default_branch: meta.defaultBranch,
});
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
logger.syncLog('warn', `Could not set default_branch to "${meta.defaultBranch}" for ${fullPath}: ${errMsg}`, 'api');
}
} else { } else {
const segments = fullPath.split('/'); const segments = fullPath.split('/');
const repo = segments.pop()!; const repo = segments.pop()!;
const owner = segments[0] || ''; const owner = segments[0] || '';
const encodedOwner = encodeURIComponent(owner); const encodedOwner = encodeURIComponent(owner);
const encodedRepo = encodeURIComponent(repo); const encodedRepo = encodeURIComponent(repo);
// Update description, visibility, default_branch // Update description, visibility
await this.rawApiCall(conn, 'PATCH', `/api/v1/repos/${encodedOwner}/${encodedRepo}`, { await this.rawApiCall(conn, 'PATCH', `/api/v1/repos/${encodedOwner}/${encodedRepo}`, {
description: meta.description, description: meta.description,
private: this.normalizeVisibility(meta.visibility) === 'private', private: this.normalizeVisibility(meta.visibility) === 'private',
default_branch: meta.defaultBranch,
}); });
// Update default_branch separately — may fail if the branch doesn't exist in git
try {
await this.rawApiCall(conn, 'PATCH', `/api/v1/repos/${encodedOwner}/${encodedRepo}`, {
default_branch: meta.defaultBranch,
});
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
logger.syncLog('warn', `Could not set default_branch to "${meta.defaultBranch}" for ${fullPath}: ${errMsg}`, 'api');
}
// Topics are a separate endpoint in Gitea // Topics are a separate endpoint in Gitea
await this.rawApiCall(conn, 'PUT', `/api/v1/repos/${encodedOwner}/${encodedRepo}/topics`, { await this.rawApiCall(conn, 'PUT', `/api/v1/repos/${encodedOwner}/${encodedRepo}/topics`, {
topics: meta.topics, topics: meta.topics,
@@ -1355,25 +1410,55 @@ export class SyncManager {
private async syncProjectAvatar( private async syncProjectAvatar(
sourceConn: interfaces.data.IProviderConnection, sourceConn: interfaces.data.IProviderConnection,
targetConn: interfaces.data.IProviderConnection, targetConn: interfaces.data.IProviderConnection,
sourceFullPath: string, _sourceFullPath: string,
targetFullPath: string, targetFullPath: string,
sourceAvatarUrl: string, sourceAvatarUrl: string,
targetRawProject: any, targetRawProject: any,
): Promise<void> { ): Promise<void> {
// Resolve relative avatar URLs // Resolve relative avatar URLs
const resolvedUrl = sourceAvatarUrl.startsWith('http') const resolvedSourceUrl = sourceAvatarUrl.startsWith('http')
? sourceAvatarUrl ? sourceAvatarUrl
: `${sourceConn.baseUrl.replace(/\/+$/, '')}${sourceAvatarUrl}`; : `${sourceConn.baseUrl.replace(/\/+$/, '')}${sourceAvatarUrl}`;
const avatarData = await this.rawBinaryFetch(sourceConn, resolvedUrl); const sourceAvatarData = await this.rawBinaryFetch(sourceConn, resolvedSourceUrl);
if (!avatarData || avatarData.length === 0) return; if (!sourceAvatarData || sourceAvatarData.length === 0) return;
// Skip SVG avatars — not supported by GitLab project endpoints
const mimeType = this.guessAvatarMimeType(sourceAvatarData, resolvedSourceUrl);
if (mimeType === 'image/svg+xml') {
logger.syncLog('warn', `Skipping SVG avatar for ${targetFullPath} (not supported by target)`, 'api');
return;
}
// Check in-memory cache: skip if source hasn't changed since last upload
const sourceHash = await this.hashBytes(sourceAvatarData);
const cacheKey = `project:${targetFullPath}`;
if (this.avatarUploadCache.get(cacheKey) === sourceHash) {
return; // Source avatar unchanged since last upload
}
// Compare with target's current avatar to avoid unnecessary uploads
const targetMeta = this.extractProjectMeta(targetConn, targetRawProject);
if (targetMeta.avatarUrl) {
try {
const resolvedTargetUrl = targetMeta.avatarUrl.startsWith('http')
? targetMeta.avatarUrl
: `${targetConn.baseUrl.replace(/\/+$/, '')}${targetMeta.avatarUrl}`;
const targetAvatarData = await this.rawBinaryFetch(targetConn, resolvedTargetUrl);
if (targetAvatarData && this.binaryEqual(sourceAvatarData, targetAvatarData)) {
this.avatarUploadCache.set(cacheKey, sourceHash);
return; // Avatars are identical — skip upload
}
} catch {
// Failed to fetch target avatar — proceed with upload as safe fallback
}
}
logger.syncLog('info', `Syncing avatar for ${targetFullPath}...`, 'api'); logger.syncLog('info', `Syncing avatar for ${targetFullPath}...`, 'api');
if (targetConn.providerType === 'gitlab') { if (targetConn.providerType === 'gitlab') {
// GitLab: multipart upload // GitLab: multipart upload
const mimeType = this.guessAvatarMimeType(avatarData, resolvedUrl); const blob = new Blob([sourceAvatarData.buffer as ArrayBuffer], { type: mimeType });
const blob = new Blob([avatarData.buffer as ArrayBuffer], { type: mimeType });
const ext = mimeType.split('/')[1] || 'png'; const ext = mimeType.split('/')[1] || 'png';
const formData = new FormData(); const formData = new FormData();
formData.append('avatar', blob, `avatar.${ext}`); formData.append('avatar', blob, `avatar.${ext}`);
@@ -1387,13 +1472,15 @@ export class SyncManager {
const segments = targetFullPath.split('/'); const segments = targetFullPath.split('/');
const repo = segments.pop()!; const repo = segments.pop()!;
const owner = segments[0] || ''; const owner = segments[0] || '';
const base64Image = this.uint8ArrayToBase64(avatarData); const base64Image = this.uint8ArrayToBase64(sourceAvatarData);
await this.rawApiCall( await this.rawApiCall(
targetConn, 'POST', targetConn, 'POST',
`/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/avatar`, `/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/avatar`,
{ image: base64Image }, { image: base64Image },
); );
} }
this.avatarUploadCache.set(cacheKey, sourceHash);
} }
private async removeProjectAvatar( private async removeProjectAvatar(
@@ -1423,18 +1510,48 @@ export class SyncManager {
sourceAvatarUrl: string, sourceAvatarUrl: string,
targetRawGroup: any, targetRawGroup: any,
): Promise<void> { ): Promise<void> {
const resolvedUrl = sourceAvatarUrl.startsWith('http') const resolvedSourceUrl = sourceAvatarUrl.startsWith('http')
? sourceAvatarUrl ? sourceAvatarUrl
: `${sourceConn.baseUrl.replace(/\/+$/, '')}${sourceAvatarUrl}`; : `${sourceConn.baseUrl.replace(/\/+$/, '')}${sourceAvatarUrl}`;
const avatarData = await this.rawBinaryFetch(sourceConn, resolvedUrl); const sourceAvatarData = await this.rawBinaryFetch(sourceConn, resolvedSourceUrl);
if (!avatarData || avatarData.length === 0) return; if (!sourceAvatarData || sourceAvatarData.length === 0) return;
// Skip SVG avatars — not supported by GitLab project endpoints
const mimeType = this.guessAvatarMimeType(sourceAvatarData, resolvedSourceUrl);
if (mimeType === 'image/svg+xml') {
logger.syncLog('warn', `Skipping SVG avatar for group ${targetGroupPath} (not supported by target)`, 'api');
return;
}
// Check in-memory cache: skip if source hasn't changed since last upload
const sourceHash = await this.hashBytes(sourceAvatarData);
const cacheKey = `group:${targetGroupPath}`;
if (this.avatarUploadCache.get(cacheKey) === sourceHash) {
return; // Source avatar unchanged since last upload
}
// Compare with target's current avatar to avoid unnecessary uploads
const targetMeta = this.extractGroupMeta(targetConn, targetRawGroup);
if (targetMeta.avatarUrl) {
try {
const resolvedTargetUrl = targetMeta.avatarUrl.startsWith('http')
? targetMeta.avatarUrl
: `${targetConn.baseUrl.replace(/\/+$/, '')}${targetMeta.avatarUrl}`;
const targetAvatarData = await this.rawBinaryFetch(targetConn, resolvedTargetUrl);
if (targetAvatarData && this.binaryEqual(sourceAvatarData, targetAvatarData)) {
this.avatarUploadCache.set(cacheKey, sourceHash);
return; // Avatars are identical — skip upload
}
} catch {
// Failed to fetch target avatar — proceed with upload as safe fallback
}
}
logger.syncLog('info', `Syncing avatar for group ${targetGroupPath}...`, 'api'); logger.syncLog('info', `Syncing avatar for group ${targetGroupPath}...`, 'api');
if (targetConn.providerType === 'gitlab') { if (targetConn.providerType === 'gitlab') {
const mimeType = this.guessAvatarMimeType(avatarData, resolvedUrl); const blob = new Blob([sourceAvatarData.buffer as ArrayBuffer], { type: mimeType });
const blob = new Blob([avatarData.buffer as ArrayBuffer], { type: mimeType });
const ext = mimeType.split('/')[1] || 'png'; const ext = mimeType.split('/')[1] || 'png';
const formData = new FormData(); const formData = new FormData();
formData.append('avatar', blob, `avatar.${ext}`); formData.append('avatar', blob, `avatar.${ext}`);
@@ -1445,13 +1562,15 @@ export class SyncManager {
); );
} else { } else {
const orgName = targetGroupPath.split('/')[0] || targetGroupPath; const orgName = targetGroupPath.split('/')[0] || targetGroupPath;
const base64Image = this.uint8ArrayToBase64(avatarData); const base64Image = this.uint8ArrayToBase64(sourceAvatarData);
await this.rawApiCall( await this.rawApiCall(
targetConn, 'POST', targetConn, 'POST',
`/api/v1/orgs/${encodeURIComponent(orgName)}/avatar`, `/api/v1/orgs/${encodeURIComponent(orgName)}/avatar`,
{ image: base64Image }, { image: base64Image },
); );
} }
this.avatarUploadCache.set(cacheKey, sourceHash);
} }
private async removeGroupAvatar( private async removeGroupAvatar(
@@ -1471,6 +1590,19 @@ export class SyncManager {
} }
} }
private binaryEqual(a: Uint8Array, b: Uint8Array): boolean {
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) return false;
}
return true;
}
private async hashBytes(data: Uint8Array): Promise<string> {
const hashBuffer = await crypto.subtle.digest('SHA-256', data.buffer as ArrayBuffer);
return Array.from(new Uint8Array(hashBuffer)).map(b => b.toString(16).padStart(2, '0')).join('');
}
private uint8ArrayToBase64(bytes: Uint8Array): string { private uint8ArrayToBase64(bytes: Uint8Array): string {
let binary = ''; let binary = '';
for (let i = 0; i < bytes.length; i++) { for (let i = 0; i < bytes.length; i++) {
@@ -1534,14 +1666,14 @@ export class SyncManager {
// Walk the basePath to find the parent group, then create "obsolete" subgroup // Walk the basePath to find the parent group, then create "obsolete" subgroup
let parentId: number | undefined; let parentId: number | undefined;
if (basePath) { if (basePath) {
const parentGroup = await client.getGroupByPath(basePath); const parentGroup = await client.getGroup(basePath);
parentId = parentGroup.id; parentId = parentGroup.id;
} }
// Try to get existing obsolete group // Try to get existing obsolete group
const obsoletePath = basePath ? `${basePath}/obsolete` : 'obsolete'; const obsoletePath = basePath ? `${basePath}/obsolete` : 'obsolete';
try { try {
const group = await client.getGroupByPath(obsoletePath); const group = await client.getGroup(obsoletePath);
return { type: 'gitlab', groupId: group.id }; return { type: 'gitlab', groupId: group.id };
} catch { } catch {
// Doesn't exist — create it // Doesn't exist — create it
@@ -1552,7 +1684,7 @@ export class SyncManager {
return { type: 'gitlab', groupId: newGroup.id }; return { type: 'gitlab', groupId: newGroup.id };
} catch (createErr: any) { } catch (createErr: any) {
if (String(createErr).includes('409') || String(createErr).includes('already')) { if (String(createErr).includes('409') || String(createErr).includes('already')) {
const group = await client.getGroupByPath(obsoletePath); const group = await client.getGroup(obsoletePath);
return { type: 'gitlab', groupId: group.id }; return { type: 'gitlab', groupId: group.id };
} }
throw createErr; throw createErr;
@@ -1690,6 +1822,133 @@ export class SyncManager {
} }
} }
/**
* Fetch all branch and tag SHAs from a repo via provider API.
* Returns null on any error (safe fallback to git-based comparison).
*/
private async listRefsViaProvider(
provider: BaseProvider,
fullPath: string,
): Promise<{ branches: Map<string, string>; tags: Map<string, string> } | null> {
try {
const [branches, tags] = await Promise.all([
provider.getBranches(fullPath),
provider.getTags(fullPath),
]);
return {
branches: new Map(branches.map((b) => [b.name, b.commitSha])),
tags: new Map(tags.map((t) => [t.name, t.commitSha])),
};
} catch {
return null;
}
}
/**
* Compare refs between source and target via provider API (no git clone needed).
* Returns true (match), false (differ), or null (can't determine — fall through to git).
*/
private async refsMatchViaApi(
sourceProvider: BaseProvider,
targetProvider: BaseProvider,
sourceFullPath: string,
targetFullPath: string,
): Promise<boolean | null> {
const [sourceRefs, targetRefs] = await Promise.all([
this.listRefsViaProvider(sourceProvider, sourceFullPath),
this.listRefsViaProvider(targetProvider, targetFullPath),
]);
if (!sourceRefs || !targetRefs) return null;
// Compare branches
if (sourceRefs.branches.size !== targetRefs.branches.size) return false;
for (const [name, sha] of sourceRefs.branches) {
if (targetRefs.branches.get(name) !== sha) return false;
}
// Compare tags
if (sourceRefs.tags.size !== targetRefs.tags.size) return false;
for (const [name, sha] of sourceRefs.tags) {
if (targetRefs.tags.get(name) !== sha) return false;
}
return true;
}
/**
* Compare local refs (source) with target remote refs.
* Returns true when all branches and tags are identical — no push needed.
*/
private async refsMatch(mirrorDir: string): Promise<boolean> {
try {
// Local branches (source)
const localHeadsRaw = await this.runGit(
['for-each-ref', '--format=%(refname:strip=2) %(objectname)', 'refs/heads/'], mirrorDir,
);
// Target branches (fetched by checkUnrelatedHistory)
const targetHeadsRaw = await this.runGit(
['for-each-ref', '--format=%(refname:strip=3) %(objectname)', 'refs/remotes/target/'], mirrorDir,
);
// Local tags
const localTagsRaw = await this.runGit(
['for-each-ref', '--format=%(refname:strip=2) %(objectname)', 'refs/tags/'], mirrorDir,
);
// Target tags via ls-remote (avoids shared refs/tags/ namespace ambiguity in bare repos)
const targetTagsRaw = await this.runGit(['ls-remote', '--tags', 'target'], mirrorDir);
const parseRefLines = (raw: string): Map<string, string> => {
const map = new Map<string, string>();
for (const line of raw.trim().split('\n')) {
if (!line.trim()) continue;
const parts = line.trim().split(/\s+/);
if (parts.length >= 2) {
map.set(parts[0], parts[1]);
}
}
return map;
};
const parseLsRemoteTags = (raw: string): Map<string, string> => {
const map = new Map<string, string>();
for (const line of raw.trim().split('\n')) {
if (!line.trim()) continue;
// Skip ^{} dereference lines
if (line.includes('^{}')) continue;
const parts = line.trim().split(/\s+/);
if (parts.length >= 2) {
// parts[0] = sha, parts[1] = refs/tags/name
const tagName = parts[1].replace('refs/tags/', '');
map.set(tagName, parts[0]);
}
}
return map;
};
const localHeads = parseRefLines(localHeadsRaw);
const targetHeads = parseRefLines(targetHeadsRaw);
const localTags = parseRefLines(localTagsRaw);
const targetTags = parseLsRemoteTags(targetTagsRaw);
// Compare branches
if (localHeads.size !== targetHeads.size) return false;
for (const [name, sha] of localHeads) {
if (targetHeads.get(name) !== sha) return false;
}
// Compare tags
if (localTags.size !== targetTags.size) return false;
for (const [name, sha] of localTags) {
if (targetTags.get(name) !== sha) return false;
}
return true;
} catch {
// On any error, fall back to pushing (safe default)
return false;
}
}
private async runGit(args: string[], cwd?: string): Promise<string> { private async runGit(args: string[], cwd?: string): Promise<string> {
const cmd = new Deno.Command('git', { const cmd = new Deno.Command('git', {
args, args,

View File

@@ -2,6 +2,27 @@ import * as plugins from '../../plugins.ts';
import type { OpsServer } from '../classes.opsserver.ts'; import type { OpsServer } from '../classes.opsserver.ts';
import * as interfaces from '../../../ts_interfaces/index.ts'; import * as interfaces from '../../../ts_interfaces/index.ts';
import { requireValidIdentity } from '../helpers/guards.ts'; import { requireValidIdentity } from '../helpers/guards.ts';
import type { BaseProvider } from '../../providers/classes.baseprovider.ts';
const TIME_RANGE_MS: Record<string, number> = {
'1h': 60 * 60 * 1000,
'6h': 6 * 60 * 60 * 1000,
'1d': 24 * 60 * 60 * 1000,
'3d': 3 * 24 * 60 * 60 * 1000,
'7d': 7 * 24 * 60 * 60 * 1000,
'30d': 30 * 24 * 60 * 60 * 1000,
};
const STATUS_PRIORITY: Record<string, number> = {
running: 0,
pending: 1,
waiting: 2,
manual: 3,
failed: 4,
canceled: 5,
success: 6,
skipped: 7,
};
export class PipelinesHandler { export class PipelinesHandler {
public typedrouter = new plugins.typedrequest.TypedRouter(); public typedrouter = new plugins.typedrequest.TypedRouter();
@@ -16,7 +37,7 @@ export class PipelinesHandler {
} }
private registerHandlers(): void { private registerHandlers(): void {
// Get pipelines // Get pipelines — supports view modes
this.typedrouter.addTypedHandler( this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetPipelines>( new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetPipelines>(
'getPipelines', 'getPipelines',
@@ -25,10 +46,32 @@ export class PipelinesHandler {
const provider = this.opsServerRef.gitopsAppRef.connectionManager.getProvider( const provider = this.opsServerRef.gitopsAppRef.connectionManager.getProvider(
dataArg.connectionId, dataArg.connectionId,
); );
const pipelines = await provider.getPipelines(dataArg.projectId, {
page: dataArg.page, const viewMode = dataArg.viewMode || 'project';
}); const timeRange = dataArg.timeRange || '1d';
return { pipelines }; const sortBy = dataArg.sortBy || 'created';
let pipelines: interfaces.data.IPipeline[];
if (viewMode === 'project') {
if (!dataArg.projectId) return { pipelines: [] };
pipelines = await provider.getPipelines(dataArg.projectId, {
page: dataArg.page,
status: dataArg.status,
});
pipelines = this.filterByTimeRange(pipelines, timeRange);
} else if (viewMode === 'current') {
pipelines = await this.fetchCurrentPipelines(provider, timeRange);
} else if (viewMode === 'group') {
if (!dataArg.groupId) return { pipelines: [] };
pipelines = await this.fetchGroupPipelines(provider, dataArg.groupId, timeRange);
} else if (viewMode === 'error') {
pipelines = await this.fetchErrorPipelines(provider, timeRange);
} else {
pipelines = [];
}
return { pipelines: this.sortPipelines(pipelines, sortBy).slice(0, 200) };
}, },
), ),
); );
@@ -94,4 +137,145 @@ export class PipelinesHandler {
), ),
); );
} }
// ---------------------------------------------------------------------------
// View mode helpers
// ---------------------------------------------------------------------------
/**
* Current mode: running/pending always shown, plus recent pipelines within timeRange.
* Fetches a generous page of recent pipelines per project (100), then splits into
* active (running/pending/waiting) and the rest. Active are always shown; the rest
* is filtered by timeRange.
*/
private async fetchCurrentPipelines(
provider: BaseProvider,
timeRange: string,
): Promise<interfaces.data.IPipeline[]> {
const projects = await provider.getProjects();
const allPipelines = await this.fetchAggregatedPipelines(provider, projects, { perPage: 100 });
// Running/pending pipelines are always shown regardless of time
const active = allPipelines.filter(
(p) => p.status === 'running' || p.status === 'pending' || p.status === 'waiting',
);
const rest = allPipelines.filter(
(p) => p.status !== 'running' && p.status !== 'pending' && p.status !== 'waiting',
);
const filteredRest = this.filterByTimeRange(rest, timeRange);
// Deduplicate (active pipelines may also appear in filtered rest)
const activeIds = new Set(active.map((p) => `${p.connectionId}:${p.projectId}:${p.id}`));
const uniqueRest = filteredRest.filter(
(p) => !activeIds.has(`${p.connectionId}:${p.projectId}:${p.id}`),
);
return [...active, ...uniqueRest];
}
/**
* Group mode: pipelines from all projects in a group
*/
private async fetchGroupPipelines(
provider: BaseProvider,
groupId: string,
timeRange: string,
): Promise<interfaces.data.IPipeline[]> {
const projects = await provider.getGroupProjects(groupId);
const allPipelines = await this.fetchAggregatedPipelines(provider, projects);
return this.filterByTimeRange(allPipelines, timeRange);
}
/**
* Error mode: only failed pipelines
*/
private async fetchErrorPipelines(
provider: BaseProvider,
timeRange: string,
): Promise<interfaces.data.IPipeline[]> {
const projects = await provider.getProjects();
const allPipelines = await this.fetchAggregatedPipelines(provider, projects, { status: 'failed' });
return this.filterByTimeRange(allPipelines, timeRange);
}
/**
* Fetch pipelines from multiple projects in parallel (batched)
*/
private async fetchAggregatedPipelines(
provider: BaseProvider,
projects: interfaces.data.IProject[],
opts?: { status?: string; perPage?: number },
): Promise<interfaces.data.IPipeline[]> {
const BATCH_SIZE = 10;
const perPage = opts?.perPage || 50;
const allPipelines: interfaces.data.IPipeline[] = [];
for (let i = 0; i < projects.length; i += BATCH_SIZE) {
const batch = projects.slice(i, i + BATCH_SIZE);
const results = await Promise.allSettled(
batch.map(async (project) => {
const pipelines = await provider.getPipelines(project.id, {
perPage,
status: opts?.status,
});
// Enrich with proper project name
return pipelines.map((p) => ({
...p,
projectName: project.fullPath || project.name || p.projectId,
}));
}),
);
for (let j = 0; j < results.length; j++) {
const result = results[j];
if (result.status === 'fulfilled') {
allPipelines.push(...result.value);
} else {
const projectId = batch[j]?.id || 'unknown';
console.warn(`[PipelinesHandler] Failed to fetch pipelines for project ${projectId}: ${result.reason}`);
}
}
}
return allPipelines;
}
// ---------------------------------------------------------------------------
// Filtering and sorting
// ---------------------------------------------------------------------------
private filterByTimeRange(
pipelines: interfaces.data.IPipeline[],
timeRange: string,
): interfaces.data.IPipeline[] {
const cutoffMs = TIME_RANGE_MS[timeRange] || TIME_RANGE_MS['1d'];
const cutoff = Date.now() - cutoffMs;
return pipelines.filter((p) => {
if (!p.createdAt) return false;
return new Date(p.createdAt).getTime() >= cutoff;
});
}
private sortPipelines(
pipelines: interfaces.data.IPipeline[],
sortBy: string,
): interfaces.data.IPipeline[] {
const sorted = [...pipelines];
switch (sortBy) {
case 'duration':
sorted.sort((a, b) => (b.duration || 0) - (a.duration || 0));
break;
case 'status':
sorted.sort(
(a, b) => (STATUS_PRIORITY[a.status] ?? 99) - (STATUS_PRIORITY[b.status] ?? 99),
);
break;
case 'created':
default:
sorted.sort(
(a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),
);
break;
}
return sorted;
}
} }

View File

@@ -22,7 +22,7 @@ export class SyncHandler {
try { try {
const typedsocket = this.opsServerRef.server?.typedserver?.typedsocket; const typedsocket = this.opsServerRef.server?.typedserver?.typedsocket;
if (!typedsocket) return; if (!typedsocket) return;
typedsocket.findAllTargetConnectionsByTag('allClients').then((connections) => { typedsocket.findAllTargetConnectionsByTag('syncLogClient').then((connections) => {
for (const conn of connections) { for (const conn of connections) {
typedsocket typedsocket
.createTypedRequest<interfaces.requests.IReq_PushSyncLog>('pushSyncLog', conn) .createTypedRequest<interfaces.requests.IReq_PushSyncLog>('pushSyncLog', conn)

View File

@@ -31,7 +31,7 @@ export class WebhookHandler {
try { try {
const typedsocket = this.opsServerRef.server.typedserver.typedsocket; const typedsocket = this.opsServerRef.server.typedserver.typedsocket;
if (typedsocket) { if (typedsocket) {
const connections = await typedsocket.findAllTargetConnectionsByTag('allClients'); const connections = await typedsocket.findAllTargetConnectionsByTag('syncLogClient');
for (const conn of connections) { for (const conn of connections) {
const req = typedsocket.createTypedRequest<interfaces.requests.IReq_WebhookNotification>( const req = typedsocket.createTypedRequest<interfaces.requests.IReq_WebhookNotification>(
'webhookNotification', 'webhookNotification',

View File

@@ -4,7 +4,6 @@ export interface IGitopsPaths {
gitopsHomeDir: string; gitopsHomeDir: string;
defaultStoragePath: string; defaultStoragePath: string;
defaultTsmDbPath: string; defaultTsmDbPath: string;
syncMirrorsPath: string;
} }
/** /**
@@ -16,6 +15,5 @@ export function resolvePaths(baseDir?: string): IGitopsPaths {
gitopsHomeDir: home, gitopsHomeDir: home,
defaultStoragePath: path.join(home, 'storage'), defaultStoragePath: path.join(home, 'storage'),
defaultTsmDbPath: path.join(home, 'tsmdb'), defaultTsmDbPath: path.join(home, 'tsmdb'),
syncMirrorsPath: path.join(home, 'mirrors'),
}; };
} }

View File

@@ -11,6 +11,12 @@ export interface IListOptions {
perPage?: number; perPage?: number;
} }
export interface IPipelineListOptions extends IListOptions {
status?: string;
ref?: string;
source?: string;
}
/** /**
* Abstract base class for Git provider implementations. * Abstract base class for Git provider implementations.
* Subclasses implement Gitea API v1 or GitLab API v4. * Subclasses implement Gitea API v1 or GitLab API v4.
@@ -36,6 +42,9 @@ export abstract class BaseProvider {
// Groups / Orgs // Groups / Orgs
abstract getGroups(opts?: IListOptions): Promise<interfaces.data.IGroup[]>; abstract getGroups(opts?: IListOptions): Promise<interfaces.data.IGroup[]>;
// Group Projects
abstract getGroupProjects(groupId: string, opts?: IListOptions): Promise<interfaces.data.IProject[]>;
// Secrets — project scope // Secrets — project scope
abstract getProjectSecrets(projectId: string): Promise<interfaces.data.ISecret[]>; abstract getProjectSecrets(projectId: string): Promise<interfaces.data.ISecret[]>;
abstract createProjectSecret( abstract createProjectSecret(
@@ -64,10 +73,14 @@ export abstract class BaseProvider {
): Promise<interfaces.data.ISecret>; ): Promise<interfaces.data.ISecret>;
abstract deleteGroupSecret(groupId: string, key: string): Promise<void>; abstract deleteGroupSecret(groupId: string, key: string): Promise<void>;
// Branches / Tags
abstract getBranches(projectFullPath: string, opts?: IListOptions): Promise<interfaces.data.IBranch[]>;
abstract getTags(projectFullPath: string, opts?: IListOptions): Promise<interfaces.data.ITag[]>;
// Pipelines / CI // Pipelines / CI
abstract getPipelines( abstract getPipelines(
projectId: string, projectId: string,
opts?: IListOptions, opts?: IPipelineListOptions,
): Promise<interfaces.data.IPipeline[]>; ): Promise<interfaces.data.IPipeline[]>;
abstract getPipelineJobs( abstract getPipelineJobs(
projectId: string, projectId: string,

View File

@@ -1,6 +1,6 @@
import * as plugins from '../plugins.ts'; import * as plugins from '../plugins.ts';
import type * as interfaces from '../../ts_interfaces/index.ts'; import type * as interfaces from '../../ts_interfaces/index.ts';
import { BaseProvider, type ITestConnectionResult, type IListOptions } from './classes.baseprovider.ts'; import { BaseProvider, type ITestConnectionResult, type IListOptions, type IPipelineListOptions } from './classes.baseprovider.ts';
/** /**
* Gitea API v1 provider implementation * Gitea API v1 provider implementation
@@ -18,62 +18,46 @@ export class GiteaProvider extends BaseProvider {
} }
async getProjects(opts?: IListOptions): Promise<interfaces.data.IProject[]> { async getProjects(opts?: IListOptions): Promise<interfaces.data.IProject[]> {
// Use org-scoped listing when groupFilterId is set const repos = this.groupFilterId
const fetchFn = this.groupFilterId ? await (await this.client.getOrg(this.groupFilterId)).getRepos(opts)
? (o: IListOptions) => this.client.getOrgRepos(this.groupFilterId!, o) : await this.client.getRepos(opts);
: (o: IListOptions) => this.client.getRepos(o); return repos.map((r) => this.mapProject(r));
// If caller explicitly requests a specific page, respect it (no auto-pagination)
if (opts?.page) {
const repos = await fetchFn(opts);
return repos.map((r) => this.mapProject(r));
}
const allRepos: plugins.giteaClient.IGiteaRepository[] = [];
const perPage = opts?.perPage || 50;
let page = 1;
while (true) {
const repos = await fetchFn({ ...opts, page, perPage });
allRepos.push(...repos);
if (repos.length < perPage) break;
page++;
}
return allRepos.map((r) => this.mapProject(r));
} }
async getGroups(opts?: IListOptions): Promise<interfaces.data.IGroup[]> { async getGroups(opts?: IListOptions): Promise<interfaces.data.IGroup[]> {
// When groupFilterId is set, return only that single org
if (this.groupFilterId) { if (this.groupFilterId) {
const org = await this.client.getOrg(this.groupFilterId); const org = await this.client.getOrg(this.groupFilterId);
return [this.mapGroup(org)]; return [this.mapGroup(org)];
} }
const orgs = await this.client.getOrgs(opts);
return orgs.map((o) => this.mapGroup(o));
}
// If caller explicitly requests a specific page, respect it (no auto-pagination) async getGroupProjects(groupId: string, opts?: IListOptions): Promise<interfaces.data.IProject[]> {
if (opts?.page) { const org = await this.client.getOrg(groupId);
const orgs = await this.client.getOrgs(opts); const repos = await org.getRepos(opts);
return orgs.map((o) => this.mapGroup(o)); return repos.map((r) => this.mapProject(r));
} }
const allOrgs: plugins.giteaClient.IGiteaOrganization[] = []; // --- Branches / Tags ---
const perPage = opts?.perPage || 50;
let page = 1;
while (true) { async getBranches(projectFullPath: string, opts?: IListOptions): Promise<interfaces.data.IBranch[]> {
const orgs = await this.client.getOrgs({ ...opts, page, perPage }); const repo = await this.client.getRepo(projectFullPath);
allOrgs.push(...orgs); const branches = await repo.getBranches(opts);
if (orgs.length < perPage) break; return branches.map((b) => ({ name: b.name, commitSha: b.commitSha }));
page++; }
}
return allOrgs.map((o) => this.mapGroup(o)); async getTags(projectFullPath: string, opts?: IListOptions): Promise<interfaces.data.ITag[]> {
const repo = await this.client.getRepo(projectFullPath);
const tags = await repo.getTags(opts);
return tags.map((t) => ({ name: t.name, commitSha: t.commitSha }));
} }
// --- Project Secrets --- // --- Project Secrets ---
async getProjectSecrets(projectId: string): Promise<interfaces.data.ISecret[]> { async getProjectSecrets(projectId: string): Promise<interfaces.data.ISecret[]> {
const secrets = await this.client.getRepoSecrets(projectId); const repo = await this.client.getRepo(projectId);
const secrets = await repo.getSecrets();
return secrets.map((s) => this.mapSecret(s, 'project', projectId)); return secrets.map((s) => this.mapSecret(s, 'project', projectId));
} }
@@ -82,7 +66,8 @@ export class GiteaProvider extends BaseProvider {
key: string, key: string,
value: string, value: string,
): Promise<interfaces.data.ISecret> { ): Promise<interfaces.data.ISecret> {
await this.client.setRepoSecret(projectId, key, value); const repo = await this.client.getRepo(projectId);
await repo.setSecret(key, value);
return { key, value: '***', protected: false, masked: true, scope: 'project', scopeId: projectId, scopeName: projectId, connectionId: this.connectionId, environment: '*' }; return { key, value: '***', protected: false, masked: true, scope: 'project', scopeId: projectId, scopeName: projectId, connectionId: this.connectionId, environment: '*' };
} }
@@ -95,13 +80,15 @@ export class GiteaProvider extends BaseProvider {
} }
async deleteProjectSecret(projectId: string, key: string): Promise<void> { async deleteProjectSecret(projectId: string, key: string): Promise<void> {
await this.client.deleteRepoSecret(projectId, key); const repo = await this.client.getRepo(projectId);
await repo.deleteSecret(key);
} }
// --- Group Secrets --- // --- Group Secrets ---
async getGroupSecrets(groupId: string): Promise<interfaces.data.ISecret[]> { async getGroupSecrets(groupId: string): Promise<interfaces.data.ISecret[]> {
const secrets = await this.client.getOrgSecrets(groupId); const org = await this.client.getOrg(groupId);
const secrets = await org.getSecrets();
return secrets.map((s) => this.mapSecret(s, 'group', groupId)); return secrets.map((s) => this.mapSecret(s, 'group', groupId));
} }
@@ -110,7 +97,8 @@ export class GiteaProvider extends BaseProvider {
key: string, key: string,
value: string, value: string,
): Promise<interfaces.data.ISecret> { ): Promise<interfaces.data.ISecret> {
await this.client.setOrgSecret(groupId, key, value); const org = await this.client.getOrg(groupId);
await org.setSecret(key, value);
return { key, value: '***', protected: false, masked: true, scope: 'group', scopeId: groupId, scopeName: groupId, connectionId: this.connectionId, environment: '*' }; return { key, value: '***', protected: false, masked: true, scope: 'group', scopeId: groupId, scopeName: groupId, connectionId: this.connectionId, environment: '*' };
} }
@@ -123,16 +111,24 @@ export class GiteaProvider extends BaseProvider {
} }
async deleteGroupSecret(groupId: string, key: string): Promise<void> { async deleteGroupSecret(groupId: string, key: string): Promise<void> {
await this.client.deleteOrgSecret(groupId, key); const org = await this.client.getOrg(groupId);
await org.deleteSecret(key);
} }
// --- Pipelines (Action Runs) --- // --- Pipelines (Action Runs) ---
async getPipelines( async getPipelines(
projectId: string, projectId: string,
opts?: IListOptions, opts?: IPipelineListOptions,
): Promise<interfaces.data.IPipeline[]> { ): Promise<interfaces.data.IPipeline[]> {
const runs = await this.client.getActionRuns(projectId, opts); const repo = await this.client.getRepo(projectId);
const runs = await repo.getActionRuns({
page: opts?.page,
perPage: opts?.perPage,
status: opts?.status,
branch: opts?.ref,
event: opts?.source,
});
return runs.map((r) => this.mapPipeline(r, projectId)); return runs.map((r) => this.mapPipeline(r, projectId));
} }
@@ -140,90 +136,101 @@ export class GiteaProvider extends BaseProvider {
projectId: string, projectId: string,
pipelineId: string, pipelineId: string,
): Promise<interfaces.data.IPipelineJob[]> { ): Promise<interfaces.data.IPipelineJob[]> {
const jobs = await this.client.getActionRunJobs(projectId, Number(pipelineId)); // Use the client's internal method directly to avoid an extra getRepo call
return jobs.map((j) => this.mapJob(j, pipelineId)); const jobs = await this.client.requestGetActionRunJobs(projectId, Number(pipelineId));
return jobs.map((j) => {
const resolvedStatus = plugins.giteaClient.resolveGiteaStatus(j.status, j.conclusion);
return this.mapJob(resolvedStatus, j, pipelineId);
});
} }
async getJobLog(projectId: string, jobId: string): Promise<string> { async getJobLog(projectId: string, jobId: string): Promise<string> {
return this.client.getJobLog(projectId, Number(jobId)); return this.client.requestGetJobLog(projectId, Number(jobId));
} }
async retryPipeline(projectId: string, pipelineId: string): Promise<void> { async retryPipeline(projectId: string, pipelineId: string): Promise<void> {
await this.client.rerunAction(projectId, Number(pipelineId)); // Fetch the run to get its workflow path, then dispatch
const run = await this.client.requestGetActionRun(projectId, Number(pipelineId));
const wfId = plugins.giteaClient.extractWorkflowIdFromPath(run.path);
const ref = run.head_branch || 'main';
if (!wfId) {
throw new Error(`Cannot retry: no workflow ID found in path "${run.path}"`);
}
await this.client.requestDispatchWorkflow(projectId, wfId, ref);
} }
async cancelPipeline(projectId: string, pipelineId: string): Promise<void> { async cancelPipeline(_projectId: string, _pipelineId: string): Promise<void> {
await this.client.cancelAction(projectId, Number(pipelineId)); throw new Error('Cancel is not supported by Gitea 1.25');
} }
// --- Mappers --- // --- Mappers ---
private mapProject(r: plugins.giteaClient.IGiteaRepository): interfaces.data.IProject { private mapProject(r: plugins.giteaClient.GiteaRepository): interfaces.data.IProject {
return { return {
id: r.full_name || String(r.id), id: r.fullName || String(r.id),
name: r.name || '', name: r.name,
fullPath: r.full_name || '', fullPath: r.fullName,
description: r.description || '', description: r.description,
defaultBranch: r.default_branch || 'main', defaultBranch: r.defaultBranch,
webUrl: r.html_url || '', webUrl: r.htmlUrl,
connectionId: this.connectionId, connectionId: this.connectionId,
visibility: r.private ? 'private' : 'public', visibility: r.isPrivate ? 'private' : 'public',
topics: r.topics || [], topics: r.topics,
lastActivity: r.updated_at || '', lastActivity: r.updatedAt,
}; };
} }
private mapGroup(o: plugins.giteaClient.IGiteaOrganization): interfaces.data.IGroup { private mapGroup(o: plugins.giteaClient.GiteaOrganization): interfaces.data.IGroup {
return { return {
id: o.name || String(o.id), id: o.name || String(o.id),
name: o.name || '', name: o.name,
fullPath: o.name || '', fullPath: o.name,
description: o.description || '', description: o.description,
webUrl: `${this.baseUrl}/${o.name}`, webUrl: `${this.baseUrl}/${o.name}`,
connectionId: this.connectionId, connectionId: this.connectionId,
visibility: o.visibility || 'public', visibility: o.visibility || 'public',
projectCount: o.repo_count || 0, projectCount: o.repoCount,
}; };
} }
private mapSecret(s: plugins.giteaClient.IGiteaSecret, scope: 'project' | 'group', scopeId: string, scopeName?: string): interfaces.data.ISecret { private mapSecret(s: plugins.giteaClient.GiteaSecret, scope: 'project' | 'group', scopeId: string): interfaces.data.ISecret {
return { return {
key: s.name || '', key: s.name,
value: '***', value: '***',
protected: false, protected: false,
masked: true, masked: true,
scope, scope,
scopeId, scopeId,
scopeName: scopeName || scopeId, scopeName: scopeId,
connectionId: this.connectionId, connectionId: this.connectionId,
environment: '*', environment: '*',
}; };
} }
private mapPipeline(r: plugins.giteaClient.IGiteaActionRun, projectId: string): interfaces.data.IPipeline { private mapPipeline(r: plugins.giteaClient.GiteaActionRun, projectId: string): interfaces.data.IPipeline {
return { return {
id: String(r.id), id: String(r.id),
projectId, projectId,
projectName: projectId, projectName: projectId,
connectionId: this.connectionId, connectionId: this.connectionId,
status: this.mapStatus(r.status || r.conclusion), status: this.mapStatus(r.resolvedStatus),
ref: r.head_branch || '', ref: r.ref,
sha: r.head_sha || '', sha: r.headSha,
webUrl: r.html_url || '', webUrl: r.htmlUrl,
duration: r.run_duration || 0, duration: r.duration,
createdAt: r.created_at || '', createdAt: r.startedAt,
source: r.event || 'push', source: r.event || 'push',
}; };
} }
private mapJob(j: plugins.giteaClient.IGiteaActionRunJob, pipelineId: string): interfaces.data.IPipelineJob { private mapJob(resolvedStatus: string, j: plugins.giteaClient.IGiteaActionRunJob, pipelineId: string): interfaces.data.IPipelineJob {
return { return {
id: String(j.id), id: String(j.id),
pipelineId, pipelineId,
name: j.name || '', name: j.name || '',
stage: j.name || 'default', stage: j.name || 'default',
status: this.mapStatus(j.status || j.conclusion), status: this.mapStatus(resolvedStatus),
duration: j.run_duration || 0, duration: plugins.giteaClient.computeDuration(j.started_at, j.completed_at),
}; };
} }

View File

@@ -1,6 +1,6 @@
import * as plugins from '../plugins.ts'; import * as plugins from '../plugins.ts';
import type * as interfaces from '../../ts_interfaces/index.ts'; import type * as interfaces from '../../ts_interfaces/index.ts';
import { BaseProvider, type ITestConnectionResult, type IListOptions } from './classes.baseprovider.ts'; import { BaseProvider, type ITestConnectionResult, type IListOptions, type IPipelineListOptions } from './classes.baseprovider.ts';
/** /**
* GitLab API v4 provider implementation * GitLab API v4 provider implementation
@@ -18,77 +18,47 @@ export class GitLabProvider extends BaseProvider {
} }
async getProjects(opts?: IListOptions): Promise<interfaces.data.IProject[]> { async getProjects(opts?: IListOptions): Promise<interfaces.data.IProject[]> {
if (this.groupFilterId) { const projects = this.groupFilterId
// Auto-paginate group-scoped project listing ? await (await this.client.getGroup(this.groupFilterId)).getProjects(opts)
if (opts?.page) { : await this.client.getProjects(opts);
const projects = await this.client.getGroupProjects(this.groupFilterId, opts); return projects.map((p) => this.mapProject(p));
return projects.map((p) => this.mapProject(p));
}
const allProjects: plugins.gitlabClient.IGitLabProject[] = [];
const perPage = opts?.perPage || 50;
let page = 1;
while (true) {
const projects = await this.client.getGroupProjects(this.groupFilterId, { ...opts, page, perPage });
allProjects.push(...projects);
if (projects.length < perPage) break;
page++;
}
return allProjects.map((p) => this.mapProject(p));
}
if (opts?.page) {
const projects = await this.client.getProjects(opts);
return projects.map((p) => this.mapProject(p));
}
const allProjects: plugins.gitlabClient.IGitLabProject[] = [];
const perPage = opts?.perPage || 50;
let page = 1;
while (true) {
const projects = await this.client.getProjects({ ...opts, page, perPage });
allProjects.push(...projects);
if (projects.length < perPage) break;
page++;
}
return allProjects.map((p) => this.mapProject(p));
} }
async getGroups(opts?: IListOptions): Promise<interfaces.data.IGroup[]> { async getGroups(opts?: IListOptions): Promise<interfaces.data.IGroup[]> {
if (this.groupFilterId) { if (this.groupFilterId) {
// Auto-paginate descendant groups listing const group = await this.client.getGroup(this.groupFilterId);
if (opts?.page) { const descendants = await group.getDescendantGroups(opts);
const groups = await this.client.getDescendantGroups(this.groupFilterId, opts); return descendants.map((g) => this.mapGroup(g));
return groups.map((g) => this.mapGroup(g));
}
const allGroups: plugins.gitlabClient.IGitLabGroup[] = [];
const perPage = opts?.perPage || 50;
let page = 1;
while (true) {
const groups = await this.client.getDescendantGroups(this.groupFilterId, { ...opts, page, perPage });
allGroups.push(...groups);
if (groups.length < perPage) break;
page++;
}
return allGroups.map((g) => this.mapGroup(g));
} }
if (opts?.page) { const groups = await this.client.getGroups(opts);
const groups = await this.client.getGroups(opts); return groups.map((g) => this.mapGroup(g));
return groups.map((g) => this.mapGroup(g)); }
}
const allGroups: plugins.gitlabClient.IGitLabGroup[] = []; async getGroupProjects(groupId: string, opts?: IListOptions): Promise<interfaces.data.IProject[]> {
const perPage = opts?.perPage || 50; const group = await this.client.getGroup(groupId);
let page = 1; const projects = await group.getProjects(opts);
while (true) { return projects.map((p) => this.mapProject(p));
const groups = await this.client.getGroups({ ...opts, page, perPage }); }
allGroups.push(...groups);
if (groups.length < perPage) break; // --- Branches / Tags ---
page++;
} async getBranches(projectFullPath: string, opts?: IListOptions): Promise<interfaces.data.IBranch[]> {
return allGroups.map((g) => this.mapGroup(g)); const project = await this.client.getProject(projectFullPath);
const branches = await project.getBranches(opts);
return branches.map((b) => ({ name: b.name, commitSha: b.commitSha }));
}
async getTags(projectFullPath: string, opts?: IListOptions): Promise<interfaces.data.ITag[]> {
const project = await this.client.getProject(projectFullPath);
const tags = await project.getTags(opts);
return tags.map((t) => ({ name: t.name, commitSha: t.commitSha }));
} }
// --- Project Secrets (CI/CD Variables) --- // --- Project Secrets (CI/CD Variables) ---
async getProjectSecrets(projectId: string): Promise<interfaces.data.ISecret[]> { async getProjectSecrets(projectId: string): Promise<interfaces.data.ISecret[]> {
const vars = await this.client.getProjectVariables(projectId); const project = await this.client.getProject(projectId);
const vars = await project.getVariables();
return vars.map((v) => this.mapVariable(v, 'project', projectId)); return vars.map((v) => this.mapVariable(v, 'project', projectId));
} }
@@ -97,7 +67,8 @@ export class GitLabProvider extends BaseProvider {
key: string, key: string,
value: string, value: string,
): Promise<interfaces.data.ISecret> { ): Promise<interfaces.data.ISecret> {
const v = await this.client.createProjectVariable(projectId, key, value); const project = await this.client.getProject(projectId);
const v = await project.createVariable(key, value);
return this.mapVariable(v, 'project', projectId); return this.mapVariable(v, 'project', projectId);
} }
@@ -106,18 +77,21 @@ export class GitLabProvider extends BaseProvider {
key: string, key: string,
value: string, value: string,
): Promise<interfaces.data.ISecret> { ): Promise<interfaces.data.ISecret> {
const v = await this.client.updateProjectVariable(projectId, key, value); const project = await this.client.getProject(projectId);
const v = await project.updateVariable(key, value);
return this.mapVariable(v, 'project', projectId); return this.mapVariable(v, 'project', projectId);
} }
async deleteProjectSecret(projectId: string, key: string): Promise<void> { async deleteProjectSecret(projectId: string, key: string): Promise<void> {
await this.client.deleteProjectVariable(projectId, key); const project = await this.client.getProject(projectId);
await project.deleteVariable(key);
} }
// --- Group Secrets (CI/CD Variables) --- // --- Group Secrets (CI/CD Variables) ---
async getGroupSecrets(groupId: string): Promise<interfaces.data.ISecret[]> { async getGroupSecrets(groupId: string): Promise<interfaces.data.ISecret[]> {
const vars = await this.client.getGroupVariables(groupId); const group = await this.client.getGroup(groupId);
const vars = await group.getVariables();
return vars.map((v) => this.mapVariable(v, 'group', groupId)); return vars.map((v) => this.mapVariable(v, 'group', groupId));
} }
@@ -126,7 +100,8 @@ export class GitLabProvider extends BaseProvider {
key: string, key: string,
value: string, value: string,
): Promise<interfaces.data.ISecret> { ): Promise<interfaces.data.ISecret> {
const v = await this.client.createGroupVariable(groupId, key, value); const group = await this.client.getGroup(groupId);
const v = await group.createVariable(key, value);
return this.mapVariable(v, 'group', groupId); return this.mapVariable(v, 'group', groupId);
} }
@@ -135,21 +110,30 @@ export class GitLabProvider extends BaseProvider {
key: string, key: string,
value: string, value: string,
): Promise<interfaces.data.ISecret> { ): Promise<interfaces.data.ISecret> {
const v = await this.client.updateGroupVariable(groupId, key, value); const group = await this.client.getGroup(groupId);
const v = await group.updateVariable(key, value);
return this.mapVariable(v, 'group', groupId); return this.mapVariable(v, 'group', groupId);
} }
async deleteGroupSecret(groupId: string, key: string): Promise<void> { async deleteGroupSecret(groupId: string, key: string): Promise<void> {
await this.client.deleteGroupVariable(groupId, key); const group = await this.client.getGroup(groupId);
await group.deleteVariable(key);
} }
// --- Pipelines --- // --- Pipelines ---
async getPipelines( async getPipelines(
projectId: string, projectId: string,
opts?: IListOptions, opts?: IPipelineListOptions,
): Promise<interfaces.data.IPipeline[]> { ): Promise<interfaces.data.IPipeline[]> {
const pipelines = await this.client.getPipelines(projectId, opts); const project = await this.client.getProject(projectId);
const pipelines = await project.getPipelines({
page: opts?.page,
perPage: opts?.perPage,
status: opts?.status,
ref: opts?.ref,
source: opts?.source,
});
return pipelines.map((p) => this.mapPipeline(p, projectId)); return pipelines.map((p) => this.mapPipeline(p, projectId));
} }
@@ -157,83 +141,82 @@ export class GitLabProvider extends BaseProvider {
projectId: string, projectId: string,
pipelineId: string, pipelineId: string,
): Promise<interfaces.data.IPipelineJob[]> { ): Promise<interfaces.data.IPipelineJob[]> {
const jobs = await this.client.getPipelineJobs(projectId, Number(pipelineId)); const jobs = await this.client.requestGetPipelineJobs(projectId, Number(pipelineId));
return jobs.map((j) => this.mapJob(j, pipelineId)); return jobs.map((j) => this.mapJob(j, pipelineId));
} }
async getJobLog(projectId: string, jobId: string): Promise<string> { async getJobLog(projectId: string, jobId: string): Promise<string> {
return this.client.getJobLog(projectId, Number(jobId)); return this.client.requestGetJobLog(projectId, Number(jobId));
} }
async retryPipeline(projectId: string, pipelineId: string): Promise<void> { async retryPipeline(projectId: string, pipelineId: string): Promise<void> {
await this.client.retryPipeline(projectId, Number(pipelineId)); await this.client.requestRetryPipeline(projectId, Number(pipelineId));
} }
async cancelPipeline(projectId: string, pipelineId: string): Promise<void> { async cancelPipeline(projectId: string, pipelineId: string): Promise<void> {
await this.client.cancelPipeline(projectId, Number(pipelineId)); await this.client.requestCancelPipeline(projectId, Number(pipelineId));
} }
// --- Mappers --- // --- Mappers ---
private mapProject(p: plugins.gitlabClient.IGitLabProject): interfaces.data.IProject { private mapProject(p: plugins.gitlabClient.GitLabProject): interfaces.data.IProject {
return { return {
id: String(p.id), id: String(p.id),
name: p.name || '', name: p.name,
fullPath: p.path_with_namespace || '', fullPath: p.fullPath,
description: p.description || '', description: p.description,
defaultBranch: p.default_branch || 'main', defaultBranch: p.defaultBranch,
webUrl: p.web_url || '', webUrl: p.webUrl,
connectionId: this.connectionId, connectionId: this.connectionId,
visibility: p.visibility || 'private', visibility: p.visibility,
topics: p.topics || [], topics: p.topics,
lastActivity: p.last_activity_at || '', lastActivity: p.lastActivityAt,
}; };
} }
private mapGroup(g: plugins.gitlabClient.IGitLabGroup): interfaces.data.IGroup { private mapGroup(g: plugins.gitlabClient.GitLabGroup): interfaces.data.IGroup {
return { return {
id: String(g.id), id: String(g.id),
name: g.name || '', name: g.name,
fullPath: g.full_path || '', fullPath: g.fullPath,
description: g.description || '', description: g.description,
webUrl: g.web_url || '', webUrl: g.webUrl,
connectionId: this.connectionId, connectionId: this.connectionId,
visibility: g.visibility || 'private', visibility: g.visibility,
projectCount: 0, projectCount: 0,
}; };
} }
private mapVariable( private mapVariable(
v: plugins.gitlabClient.IGitLabVariable, v: plugins.gitlabClient.GitLabVariable,
scope: 'project' | 'group', scope: 'project' | 'group',
scopeId: string, scopeId: string,
scopeName?: string,
): interfaces.data.ISecret { ): interfaces.data.ISecret {
return { return {
key: v.key || '', key: v.key,
value: v.value || '***', value: v.value || '***',
protected: v.protected || false, protected: v.protected,
masked: v.masked || false, masked: v.masked,
scope, scope,
scopeId, scopeId,
scopeName: scopeName || scopeId, scopeName: scopeId,
connectionId: this.connectionId, connectionId: this.connectionId,
environment: v.environment_scope || '*', environment: v.environmentScope,
}; };
} }
private mapPipeline(p: plugins.gitlabClient.IGitLabPipeline, projectId: string): interfaces.data.IPipeline { private mapPipeline(p: plugins.gitlabClient.GitLabPipeline, projectId: string): interfaces.data.IPipeline {
return { return {
id: String(p.id), id: String(p.id),
projectId, projectId,
projectName: projectId, projectName: projectId,
connectionId: this.connectionId, connectionId: this.connectionId,
status: (p.status || 'pending') as interfaces.data.TPipelineStatus, status: (p.status || 'pending') as interfaces.data.TPipelineStatus,
ref: p.ref || '', ref: p.ref,
sha: p.sha || '', sha: p.sha,
webUrl: p.web_url || '', webUrl: p.webUrl,
duration: p.duration || 0, duration: p.duration,
createdAt: p.created_at || '', createdAt: p.createdAt,
source: p.source || 'push', source: p.source || 'push',
}; };
} }

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,9 @@
export interface IBranch {
name: string;
commitSha: string;
}
export interface ITag {
name: string;
commitSha: string;
}

View File

@@ -2,6 +2,7 @@ export * from './identity.ts';
export * from './connection.ts'; export * from './connection.ts';
export * from './project.ts'; export * from './project.ts';
export * from './group.ts'; export * from './group.ts';
export * from './branch.ts';
export * from './secret.ts'; export * from './secret.ts';
export * from './pipeline.ts'; export * from './pipeline.ts';
export * from './actionlog.ts'; export * from './actionlog.ts';

View File

@@ -9,7 +9,12 @@ export interface IReq_GetPipelines extends plugins.typedrequestInterfaces.implem
request: { request: {
identity: data.IIdentity; identity: data.IIdentity;
connectionId: string; connectionId: string;
projectId: string; projectId?: string;
viewMode?: 'current' | 'project' | 'group' | 'error';
groupId?: string;
status?: string;
sortBy?: 'created' | 'duration' | 'status';
timeRange?: '1h' | '6h' | '1d' | '3d' | '7d' | '30d';
page?: number; page?: number;
}; };
response: { response: {

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@serve.zone/gitops', name: '@serve.zone/gitops',
version: '2.8.0', version: '2.13.0',
description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs' description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs'
} }

View File

@@ -516,7 +516,12 @@ export const deleteSecretAction = dataStatePart.createAction<{
export const fetchPipelinesAction = dataStatePart.createAction<{ export const fetchPipelinesAction = dataStatePart.createAction<{
connectionId: string; connectionId: string;
projectId: string; projectId?: string;
viewMode?: 'current' | 'project' | 'group' | 'error';
groupId?: string;
status?: string;
sortBy?: 'created' | 'duration' | 'status';
timeRange?: '1h' | '6h' | '1d' | '3d' | '7d' | '30d';
}>(async (statePartArg, dataArg) => { }>(async (statePartArg, dataArg) => {
const context = getActionContext(); const context = getActionContext();
try { try {
@@ -527,6 +532,11 @@ export const fetchPipelinesAction = dataStatePart.createAction<{
identity: context.identity!, identity: context.identity!,
connectionId: dataArg.connectionId, connectionId: dataArg.connectionId,
projectId: dataArg.projectId, projectId: dataArg.projectId,
viewMode: dataArg.viewMode,
groupId: dataArg.groupId,
status: dataArg.status,
sortBy: dataArg.sortBy,
timeRange: dataArg.timeRange,
}); });
return { ...statePartArg.getState(), pipelines: response.pipelines }; return { ...statePartArg.getState(), pipelines: response.pipelines };
} catch (err) { } catch (err) {
@@ -1035,11 +1045,12 @@ export async function initSyncLogSocket(): Promise<void> {
), ),
); );
await plugins.typedsocket.TypedSocket.createClient( const typedsocketClient = await plugins.typedsocket.TypedSocket.createClient(
typedrouter, typedrouter,
plugins.typedsocket.TypedSocket.useWindowLocationOriginUrl(), plugins.typedsocket.TypedSocket.useWindowLocationOriginUrl(),
{ autoReconnect: true }, { autoReconnect: true },
); );
await typedsocketClient.setTag('syncLogClient', {});
} catch (err) { } catch (err) {
console.error('Failed to init sync log TypedSocket client:', err); console.error('Failed to init sync log TypedSocket client:', err);
syncLogSocketInitialized = false; syncLogSocketInitialized = false;

View File

@@ -11,6 +11,10 @@ import {
type TemplateResult, type TemplateResult,
} from '@design.estate/dees-element'; } from '@design.estate/dees-element';
type TViewMode = 'current' | 'project' | 'group' | 'error';
type TSortBy = 'created' | 'duration' | 'status';
type TTimeRange = '1h' | '6h' | '1d' | '3d' | '7d' | '30d';
@customElement('gitops-view-pipelines') @customElement('gitops-view-pipelines')
export class GitopsViewPipelines extends DeesElement { export class GitopsViewPipelines extends DeesElement {
@state() @state()
@@ -29,13 +33,16 @@ export class GitopsViewPipelines extends DeesElement {
currentJobLog: '', currentJobLog: '',
}; };
@state() @state() accessor selectedConnectionId: string = '';
accessor selectedConnectionId: string = ''; @state() accessor selectedProjectId: string = '';
@state() accessor selectedGroupId: string = '';
@state() @state() accessor viewMode: TViewMode = 'current';
accessor selectedProjectId: string = ''; @state() accessor sortBy: TSortBy = 'created';
@state() accessor timeRange: TTimeRange = '1d';
@state() accessor isLoading: boolean = false;
private _autoRefreshHandler: () => void; private _autoRefreshHandler: () => void;
private _logPollInterval: ReturnType<typeof setInterval> | null = null;
constructor() { constructor() {
super(); super();
@@ -56,6 +63,7 @@ export class GitopsViewPipelines extends DeesElement {
public override disconnectedCallback() { public override disconnectedCallback() {
super.disconnectedCallback(); super.disconnectedCallback();
document.removeEventListener('gitops-auto-refresh', this._autoRefreshHandler); document.removeEventListener('gitops-auto-refresh', this._autoRefreshHandler);
this.stopLogPolling();
} }
private handleAutoRefresh(): void { private handleAutoRefresh(): void {
@@ -88,11 +96,40 @@ export class GitopsViewPipelines extends DeesElement {
key: c.id, key: c.id,
})); }));
const viewModeOptions = [
{ option: 'Current', key: 'current' },
{ option: 'Project', key: 'project' },
{ option: 'Group', key: 'group' },
{ option: 'Error', key: 'error' },
];
const timeRangeOptions = [
{ option: '1 hour', key: '1h' },
{ option: '6 hours', key: '6h' },
{ option: '1 day', key: '1d' },
{ option: '3 days', key: '3d' },
{ option: '7 days', key: '7d' },
{ option: '30 days', key: '30d' },
];
const sortByOptions = [
{ option: 'Created', key: 'created' },
{ option: 'Duration', key: 'duration' },
{ option: 'Status', key: 'status' },
];
const projectOptions = this.dataState.projects.map((p) => ({ const projectOptions = this.dataState.projects.map((p) => ({
option: p.fullPath || p.name, option: p.fullPath || p.name,
key: p.id, key: p.id,
})); }));
const groupOptions = this.dataState.groups.map((g) => ({
option: g.fullPath || g.name,
key: g.id,
}));
const showMultiProjectColumns = this.viewMode !== 'project';
return html` return html`
<div class="view-title">Pipelines</div> <div class="view-title">Pipelines</div>
<div class="view-description">View and manage CI/CD pipelines</div> <div class="view-description">View and manage CI/CD pipelines</div>
@@ -103,15 +140,54 @@ export class GitopsViewPipelines extends DeesElement {
.selectedOption=${connectionOptions.find((o) => o.key === this.selectedConnectionId) || connectionOptions[0]} .selectedOption=${connectionOptions.find((o) => o.key === this.selectedConnectionId) || connectionOptions[0]}
@selectedOption=${(e: CustomEvent) => { @selectedOption=${(e: CustomEvent) => {
this.selectedConnectionId = e.detail.key; this.selectedConnectionId = e.detail.key;
this.loadProjects(); this.onConnectionChange();
}} }}
></dees-input-dropdown> ></dees-input-dropdown>
<dees-input-dropdown <dees-input-dropdown
.label=${'Project'} .label=${'View'}
.options=${projectOptions} .options=${viewModeOptions}
.selectedOption=${projectOptions.find((o) => o.key === this.selectedProjectId) || projectOptions[0]} .selectedOption=${viewModeOptions.find((o) => o.key === this.viewMode)}
@selectedOption=${(e: CustomEvent) => { @selectedOption=${(e: CustomEvent) => {
this.selectedProjectId = e.detail.key; this.onViewModeChange(e.detail.key);
}}
></dees-input-dropdown>
${this.viewMode === 'project' ? html`
<dees-input-dropdown
.label=${'Project'}
.options=${projectOptions}
.selectedOption=${projectOptions.find((o) => o.key === this.selectedProjectId) || projectOptions[0]}
@selectedOption=${(e: CustomEvent) => {
this.selectedProjectId = e.detail.key;
this.loadPipelines();
}}
></dees-input-dropdown>
` : ''}
${this.viewMode === 'group' ? html`
<dees-input-dropdown
.label=${'Group'}
.options=${groupOptions}
.selectedOption=${groupOptions.find((o) => o.key === this.selectedGroupId) || groupOptions[0]}
@selectedOption=${(e: CustomEvent) => {
this.selectedGroupId = e.detail.key;
this.loadPipelines();
}}
></dees-input-dropdown>
` : ''}
<dees-input-dropdown
.label=${'Time'}
.options=${timeRangeOptions}
.selectedOption=${timeRangeOptions.find((o) => o.key === this.timeRange)}
@selectedOption=${(e: CustomEvent) => {
this.timeRange = e.detail.key;
this.loadPipelines();
}}
></dees-input-dropdown>
<dees-input-dropdown
.label=${'Sort'}
.options=${sortByOptions}
.selectedOption=${sortByOptions.find((o) => o.key === this.sortBy)}
@selectedOption=${(e: CustomEvent) => {
this.sortBy = e.detail.key;
this.loadPipelines(); this.loadPipelines();
}} }}
></dees-input-dropdown> ></dees-input-dropdown>
@@ -119,21 +195,32 @@ export class GitopsViewPipelines extends DeesElement {
</div> </div>
<dees-table <dees-table
.heading1=${'CI/CD Pipelines'} .heading1=${'CI/CD Pipelines'}
.heading2=${'Pipeline runs for the selected project'} .heading2=${this.isLoading ? 'Loading...' : `${this.dataState.pipelines.length} pipeline runs`}
.data=${this.dataState.pipelines} .data=${this.dataState.pipelines}
.displayFunction=${(item: any) => ({ .displayFunction=${(item: any) => {
ID: item.id, const row: any = {};
Status: item.status, row['ID'] = item.id;
Ref: item.ref, row['Status'] = item.status;
Duration: item.duration ? `${Math.round(item.duration)}s` : '-', if (showMultiProjectColumns) {
Source: item.source, row['Project'] = item.projectName;
Created: item.createdAt ? new Date(item.createdAt).toLocaleString() : '-', }
})} row['Ref'] = item.ref;
row['Duration'] = item.duration ? `${Math.round(item.duration)}s` : '-';
row['Source'] = item.source;
row['Created'] = item.createdAt ? new Date(item.createdAt).toLocaleString() : '-';
return row;
}}
.dataActions=${[ .dataActions=${[
{
name: 'View Logs',
iconName: 'lucide:terminal',
type: ['inRow', 'contextmenu'],
actionFunc: async ({ item }: any) => { await this.openPipelineLogs(item); },
},
{ {
name: 'View Jobs', name: 'View Jobs',
iconName: 'lucide:list', iconName: 'lucide:list',
type: ['inRow', 'contextmenu'], type: ['contextmenu'],
actionFunc: async ({ item }: any) => { await this.viewJobs(item); }, actionFunc: async ({ item }: any) => { await this.viewJobs(item); },
}, },
{ {
@@ -142,22 +229,24 @@ export class GitopsViewPipelines extends DeesElement {
type: ['inRow', 'contextmenu'], type: ['inRow', 'contextmenu'],
actionFunc: async ({ item }: any) => { actionFunc: async ({ item }: any) => {
await appstate.dataStatePart.dispatchAction(appstate.retryPipelineAction, { await appstate.dataStatePart.dispatchAction(appstate.retryPipelineAction, {
connectionId: this.selectedConnectionId, connectionId: item.connectionId || this.selectedConnectionId,
projectId: this.selectedProjectId, projectId: item.projectId,
pipelineId: item.id, pipelineId: item.id,
}); });
await this.loadPipelines();
}, },
}, },
{ {
name: 'Cancel', name: 'Cancel',
iconName: 'lucide:xCircle', iconName: 'lucide:xCircle',
type: ['inRow', 'contextmenu'], type: ['contextmenu'],
actionFunc: async ({ item }: any) => { actionFunc: async ({ item }: any) => {
await appstate.dataStatePart.dispatchAction(appstate.cancelPipelineAction, { await appstate.dataStatePart.dispatchAction(appstate.cancelPipelineAction, {
connectionId: this.selectedConnectionId, connectionId: item.connectionId || this.selectedConnectionId,
projectId: this.selectedProjectId, projectId: item.projectId,
pipelineId: item.id, pipelineId: item.id,
}); });
await this.loadPipelines();
}, },
}, },
]} ]}
@@ -173,6 +262,7 @@ export class GitopsViewPipelines extends DeesElement {
if (navCtx?.connectionId && navCtx?.projectId) { if (navCtx?.connectionId && navCtx?.projectId) {
this.selectedConnectionId = navCtx.connectionId; this.selectedConnectionId = navCtx.connectionId;
this.selectedProjectId = navCtx.projectId; this.selectedProjectId = navCtx.projectId;
this.viewMode = 'project';
appstate.uiStatePart.dispatchAction(appstate.clearNavigationContextAction, null); appstate.uiStatePart.dispatchAction(appstate.clearNavigationContextAction, null);
await this.loadProjects(); await this.loadProjects();
await this.loadPipelines(); await this.loadPipelines();
@@ -182,7 +272,39 @@ export class GitopsViewPipelines extends DeesElement {
const conns = appstate.connectionsStatePart.getState().connections; const conns = appstate.connectionsStatePart.getState().connections;
if (conns.length > 0 && !this.selectedConnectionId) { if (conns.length > 0 && !this.selectedConnectionId) {
this.selectedConnectionId = conns[0].id; this.selectedConnectionId = conns[0].id;
// In 'current' mode, load pipelines immediately
await this.loadPipelines();
}
}
// ---------------------------------------------------------------------------
// Data loading
// ---------------------------------------------------------------------------
private async onConnectionChange() {
this.selectedProjectId = '';
this.selectedGroupId = '';
if (this.viewMode === 'project') {
await this.loadProjects(); await this.loadProjects();
} else if (this.viewMode === 'group') {
await this.loadGroups();
} else {
await this.loadPipelines();
}
}
private onViewModeChange(newMode: TViewMode) {
this.stopLogPolling();
this.viewMode = newMode;
this.selectedProjectId = '';
this.selectedGroupId = '';
if (newMode === 'current' || newMode === 'error') {
this.loadPipelines();
} else if (newMode === 'project') {
this.loadProjects();
} else if (newMode === 'group') {
this.loadGroups();
} }
} }
@@ -193,18 +315,221 @@ export class GitopsViewPipelines extends DeesElement {
}); });
} }
private async loadPipelines() { private async loadGroups() {
if (!this.selectedConnectionId || !this.selectedProjectId) return; if (!this.selectedConnectionId) return;
await appstate.dataStatePart.dispatchAction(appstate.fetchPipelinesAction, { await appstate.dataStatePart.dispatchAction(appstate.fetchGroupsAction, {
connectionId: this.selectedConnectionId, connectionId: this.selectedConnectionId,
projectId: this.selectedProjectId,
}); });
} }
private async loadPipelines() {
if (!this.selectedConnectionId) return;
// For project mode, require a project selection
if (this.viewMode === 'project' && !this.selectedProjectId) return;
// For group mode, require a group selection
if (this.viewMode === 'group' && !this.selectedGroupId) return;
this.isLoading = true;
try {
await appstate.dataStatePart.dispatchAction(appstate.fetchPipelinesAction, {
connectionId: this.selectedConnectionId,
projectId: this.viewMode === 'project' ? this.selectedProjectId : undefined,
viewMode: this.viewMode,
groupId: this.viewMode === 'group' ? this.selectedGroupId : undefined,
sortBy: this.sortBy,
timeRange: this.timeRange,
});
} finally {
this.isLoading = false;
}
}
// ---------------------------------------------------------------------------
// Pipeline log viewing
// ---------------------------------------------------------------------------
private async openPipelineLogs(pipeline: any) {
await appstate.dataStatePart.dispatchAction(appstate.fetchPipelineJobsAction, {
connectionId: pipeline.connectionId || this.selectedConnectionId,
projectId: pipeline.projectId,
pipelineId: pipeline.id,
});
const jobs = appstate.dataStatePart.getState().pipelineJobs;
let activeJobId: string | null = null;
const modal = await plugins.deesCatalog.DeesModal.createAndShow({
heading: `Pipeline #${pipeline.id} - Logs`,
content: html`
<style>
.log-container {
display: flex;
gap: 0;
min-height: 400px;
max-height: 70vh;
color: #ccc;
font-family: 'Intel One Mono', 'Fira Code', monospace;
}
.job-list {
width: 220px;
flex-shrink: 0;
border-right: 1px solid #333;
overflow-y: auto;
}
.job-entry {
padding: 10px 14px;
cursor: pointer;
border-bottom: 1px solid #282828;
transition: background 0.15s;
}
.job-entry:hover {
background: #1a1a2e;
}
.job-entry.active {
background: #1a2a3a;
border-left: 3px solid #00acff;
padding-left: 11px;
}
.job-name {
font-weight: 600;
font-size: 13px;
color: #eee;
}
.job-meta {
font-size: 11px;
color: #888;
margin-top: 3px;
}
.job-status-badge {
display: inline-block;
padding: 1px 6px;
border-radius: 3px;
font-size: 10px;
font-weight: 600;
text-transform: uppercase;
}
.job-status-badge.running { background: #1a2a3a; color: #00acff; }
.job-status-badge.success { background: #1a3a1a; color: #00ff88; }
.job-status-badge.failed { background: #3a1a1a; color: #ff4444; }
.job-status-badge.pending { background: #3a3a1a; color: #ffaa00; }
.job-status-badge.canceled { background: #2a2a2a; color: #999; }
.log-output {
flex: 1;
overflow: auto;
background: #0d0d0d;
padding: 12px;
}
.log-output pre {
margin: 0;
white-space: pre-wrap;
word-break: break-all;
font-family: 'Intel One Mono', 'Fira Code', monospace;
font-size: 12px;
line-height: 1.6;
color: #ccc;
}
.no-log {
color: #666;
padding: 40px 20px;
text-align: center;
font-size: 13px;
}
</style>
<div class="log-container">
<div class="job-list">
${jobs.map((job: any) => html`
<div
class="job-entry"
data-job-id="${job.id}"
@click=${async (e: Event) => {
// Update active state visually
const container = (e.target as HTMLElement).closest('.log-container');
container?.querySelectorAll('.job-entry').forEach((el: Element) => el.classList.remove('active'));
(e.target as HTMLElement).closest('.job-entry')?.classList.add('active');
activeJobId = job.id;
await this.selectJobForLog(job, pipeline, container);
}}
>
<div class="job-name">${job.name}</div>
<div class="job-meta">
<span class="job-status-badge ${job.status}">${job.status}</span>
${job.stage ? ` ${job.stage}` : ''}
${job.duration ? ` - ${Math.round(job.duration)}s` : ''}
</div>
</div>
`)}
${jobs.length === 0 ? html`<div class="no-log">No jobs found.</div>` : ''}
</div>
<div class="log-output">
<pre class="job-log-pre"><span class="no-log">Select a job to view its log.</span></pre>
</div>
</div>
`,
menuOptions: [
{
name: 'Close',
action: async (modalRef: any) => {
this.stopLogPolling();
modalRef.destroy();
},
},
],
});
}
private async selectJobForLog(job: any, pipeline: any, container: Element | null) {
this.stopLogPolling();
// Fetch initial log
await this.fetchAndDisplayLog(job, pipeline, container);
// If job is running/pending, poll every 3 seconds
if (job.status === 'running' || job.status === 'pending' || job.status === 'waiting') {
this._logPollInterval = setInterval(async () => {
await this.fetchAndDisplayLog(job, pipeline, container);
}, 3000);
}
}
private async fetchAndDisplayLog(job: any, pipeline: any, container: Element | null) {
try {
await appstate.dataStatePart.dispatchAction(appstate.fetchJobLogAction, {
connectionId: pipeline.connectionId || this.selectedConnectionId,
projectId: pipeline.projectId,
jobId: job.id,
});
const log = appstate.dataStatePart.getState().currentJobLog;
const pre = container?.querySelector('.job-log-pre');
if (pre) {
pre.textContent = log || '(No output yet)';
// Auto-scroll to bottom
const logOutput = pre.closest('.log-output');
if (logOutput) {
logOutput.scrollTop = logOutput.scrollHeight;
}
}
} catch (err) {
console.error('Failed to fetch job log:', err);
}
}
private stopLogPolling() {
if (this._logPollInterval !== null) {
clearInterval(this._logPollInterval);
this._logPollInterval = null;
}
}
// ---------------------------------------------------------------------------
// Legacy job view (accessible via context menu)
// ---------------------------------------------------------------------------
private async viewJobs(pipeline: any) { private async viewJobs(pipeline: any) {
await appstate.dataStatePart.dispatchAction(appstate.fetchPipelineJobsAction, { await appstate.dataStatePart.dispatchAction(appstate.fetchPipelineJobsAction, {
connectionId: this.selectedConnectionId, connectionId: pipeline.connectionId || this.selectedConnectionId,
projectId: this.selectedProjectId, projectId: pipeline.projectId,
pipelineId: pipeline.id, pipelineId: pipeline.id,
}); });