feat(sync): add sync subsystem: SyncManager, OpsServer sync handlers, Sync UI and state, provider groupFilter support, and realtime sync log streaming via TypedSocket

This commit is contained in:
2026-02-28 16:33:53 +00:00
parent 2f050744bc
commit f7e16aa350
30 changed files with 2983 additions and 21 deletions

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/gitops',
version: '2.7.1',
version: '2.8.0',
description: 'GitOps management app for Gitea and GitLab - manage secrets, browse projects, view CI pipelines, and stream build logs'
}

View File

@@ -77,6 +77,15 @@ export class ConnectionManager {
for (const key of keys) {
const conn = await this.storageManager.getJSON<interfaces.data.IProviderConnection>(key);
if (conn) {
// Migrate legacy baseGroup/baseGroupId property names
if ((conn as any).baseGroup !== undefined && conn.groupFilter === undefined) {
conn.groupFilter = (conn as any).baseGroup;
delete (conn as any).baseGroup;
}
if ((conn as any).baseGroupId !== undefined && conn.groupFilterId === undefined) {
conn.groupFilterId = (conn as any).baseGroupId;
delete (conn as any).baseGroupId;
}
if (conn.token.startsWith(KEYCHAIN_PREFIX)) {
// Token is in keychain — retrieve it
const realToken = await this.smartSecret.getSecret(conn.id);
@@ -142,6 +151,7 @@ export class ConnectionManager {
providerType: interfaces.data.TProviderType,
baseUrl: string,
token: string,
groupFilter?: string,
): Promise<interfaces.data.IProviderConnection> {
const connection: interfaces.data.IProviderConnection = {
id: crypto.randomUUID(),
@@ -151,6 +161,7 @@ export class ConnectionManager {
token,
createdAt: Date.now(),
status: 'disconnected',
groupFilter: groupFilter || undefined,
};
this.connections.push(connection);
await this.persistConnection(connection);
@@ -160,13 +171,17 @@ export class ConnectionManager {
async updateConnection(
id: string,
updates: { name?: string; baseUrl?: string; token?: string },
updates: { name?: string; baseUrl?: string; token?: string; groupFilter?: string },
): Promise<interfaces.data.IProviderConnection> {
const conn = this.connections.find((c) => c.id === id);
if (!conn) throw new Error(`Connection not found: ${id}`);
if (updates.name) conn.name = updates.name;
if (updates.baseUrl) conn.baseUrl = updates.baseUrl.replace(/\/+$/, '');
if (updates.token) conn.token = updates.token;
if (updates.groupFilter !== undefined) {
conn.groupFilter = updates.groupFilter || undefined;
conn.groupFilterId = undefined; // Will be re-resolved on next test
}
await this.persistConnection(conn);
return { ...conn, token: '***' };
}
@@ -196,10 +211,39 @@ export class ConnectionManager {
const provider = this.getProvider(id);
const result = await provider.testConnection();
conn.status = result.ok ? 'connected' : 'error';
// Resolve group filter ID if connection has a groupFilter
if (result.ok && conn.groupFilter) {
await this.resolveGroupFilterId(conn);
}
await this.persistConnection(conn);
return result;
}
/**
* Resolves a human-readable groupFilter to the provider-specific group ID.
*/
private async resolveGroupFilterId(conn: interfaces.data.IProviderConnection): Promise<void> {
if (!conn.groupFilter) {
conn.groupFilterId = undefined;
return;
}
try {
if (conn.providerType === 'gitlab') {
const gitlabClient = new plugins.gitlabClient.GitLabClient(conn.baseUrl, conn.token);
const group = await gitlabClient.getGroupByPath(conn.groupFilter);
conn.groupFilterId = String(group.id);
logger.info(`Resolved group filter "${conn.groupFilter}" to ID ${conn.groupFilterId}`);
} else {
// For Gitea, the org name IS the ID
conn.groupFilterId = conn.groupFilter;
logger.info(`Group filter for Gitea connection set to org "${conn.groupFilterId}"`);
}
} catch (err) {
logger.warn(`Failed to resolve group filter "${conn.groupFilter}": ${err}`);
conn.groupFilterId = undefined;
}
}
/**
* Factory: returns the correct provider instance for a connection ID
*/
@@ -209,9 +253,9 @@ export class ConnectionManager {
switch (conn.providerType) {
case 'gitea':
return new GiteaProvider(conn.id, conn.baseUrl, conn.token);
return new GiteaProvider(conn.id, conn.baseUrl, conn.token, conn.groupFilterId);
case 'gitlab':
return new GitLabProvider(conn.id, conn.baseUrl, conn.token);
return new GitLabProvider(conn.id, conn.baseUrl, conn.token, conn.groupFilterId);
default:
throw new Error(`Unknown provider type: ${conn.providerType}`);
}

View File

@@ -2,6 +2,7 @@ import * as plugins from '../plugins.ts';
import { logger } from '../logging.ts';
import { ConnectionManager } from './connectionmanager.ts';
import { ActionLog } from './actionlog.ts';
import { SyncManager } from './syncmanager.ts';
import { OpsServer } from '../opsserver/index.ts';
import { StorageManager } from '../storage/index.ts';
import { CacheDb, CacheCleaner, CachedProject, CachedSecret, SecretsScanService } from '../cache/index.ts';
@@ -18,11 +19,14 @@ export class GitopsApp {
public opsServer: OpsServer;
public cacheDb: CacheDb;
public cacheCleaner: CacheCleaner;
public syncManager!: SyncManager;
public secretsScanService!: SecretsScanService;
private scanIntervalId: number | null = null;
private paths: ReturnType<typeof resolvePaths>;
constructor() {
const paths = resolvePaths();
this.paths = paths;
this.storageManager = new StorageManager({
backend: 'filesystem',
fsPath: paths.defaultStoragePath,
@@ -51,6 +55,15 @@ export class GitopsApp {
// Initialize connection manager (loads saved connections)
await this.connectionManager.init();
// Initialize sync manager
this.syncManager = new SyncManager(
this.storageManager,
this.connectionManager,
this.actionLog,
this.paths.syncMirrorsPath,
);
await this.syncManager.init();
// Initialize secrets scan service with 24h auto-scan
this.secretsScanService = new SecretsScanService(this.connectionManager);
const SCAN_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
@@ -80,6 +93,7 @@ export class GitopsApp {
clearInterval(this.scanIntervalId);
this.scanIntervalId = null;
}
await this.syncManager.stop();
await this.opsServer.stop();
this.cacheCleaner.stop();
await this.cacheDb.stop();

1600
ts/classes/syncmanager.ts Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -2,15 +2,60 @@
* Logging utilities for GitOps
*/
import type { ISyncLogEntry } from '../ts_interfaces/data/sync.ts';
type LogLevel = 'info' | 'success' | 'warn' | 'error' | 'debug';
const SYNC_LOG_MAX = 500;
class Logger {
private debugMode = false;
private syncLogBuffer: ISyncLogEntry[] = [];
private broadcastFn?: (entry: ISyncLogEntry) => void;
constructor() {
this.debugMode = Deno.args.includes('--debug') || Deno.env.get('DEBUG') === 'true';
}
/**
* Set the broadcast function used to push sync log entries to connected clients.
*/
setBroadcastFn(fn: (entry: ISyncLogEntry) => void): void {
this.broadcastFn = fn;
}
/**
* Log a sync-related message to both the console and the ring buffer.
* Also broadcasts to connected frontends via TypedSocket if available.
*/
syncLog(level: ISyncLogEntry['level'], message: string, source?: string): void {
// Also log to console
this.log(level, message);
const entry: ISyncLogEntry = {
timestamp: Date.now(),
level,
message,
source,
};
this.syncLogBuffer.push(entry);
if (this.syncLogBuffer.length > SYNC_LOG_MAX) {
this.syncLogBuffer.splice(0, this.syncLogBuffer.length - SYNC_LOG_MAX);
}
if (this.broadcastFn) {
this.broadcastFn(entry);
}
}
/**
* Get recent sync log entries.
*/
getSyncLogs(limit = 100): ISyncLogEntry[] {
return this.syncLogBuffer.slice(-limit);
}
log(level: LogLevel, message: string, ...args: unknown[]): void {
const prefix = this.getPrefix(level);
const formattedMessage = `${prefix} ${message}`;

View File

@@ -20,6 +20,7 @@ export class OpsServer {
public webhookHandler!: handlers.WebhookHandler;
public actionsHandler!: handlers.ActionsHandler;
public actionLogHandler!: handlers.ActionLogHandler;
public syncHandler!: handlers.SyncHandler;
constructor(gitopsAppRef: GitopsApp) {
this.gitopsAppRef = gitopsAppRef;
@@ -63,6 +64,7 @@ export class OpsServer {
this.logsHandler = new handlers.LogsHandler(this);
this.actionsHandler = new handlers.ActionsHandler(this);
this.actionLogHandler = new handlers.ActionLogHandler(this);
this.syncHandler = new handlers.SyncHandler(this);
logger.success('OpsServer TypedRequest handlers initialized');
}

View File

@@ -39,6 +39,7 @@ export class ConnectionsHandler {
dataArg.providerType,
dataArg.baseUrl,
dataArg.token,
dataArg.groupFilter,
);
this.actionLog.append({
actionType: 'create',
@@ -65,12 +66,14 @@ export class ConnectionsHandler {
name: dataArg.name,
baseUrl: dataArg.baseUrl,
token: dataArg.token,
groupFilter: dataArg.groupFilter,
},
);
const fields = [
dataArg.name && 'name',
dataArg.baseUrl && 'baseUrl',
dataArg.token && 'token',
dataArg.groupFilter !== undefined && 'groupFilter',
].filter(Boolean).join(', ');
this.actionLog.append({
actionType: 'update',

View File

@@ -8,3 +8,4 @@ export { LogsHandler } from './logs.handler.ts';
export { WebhookHandler } from './webhook.handler.ts';
export { ActionsHandler } from './actions.handler.ts';
export { ActionLogHandler } from './actionlog.handler.ts';
export { SyncHandler } from './sync.handler.ts';

View File

@@ -0,0 +1,222 @@
import * as plugins from '../../plugins.ts';
import type { OpsServer } from '../classes.opsserver.ts';
import * as interfaces from '../../../ts_interfaces/index.ts';
import { requireValidIdentity } from '../helpers/guards.ts';
import { logger } from '../../logging.ts';
export class SyncHandler {
public typedrouter = new plugins.typedrequest.TypedRouter();
constructor(private opsServerRef: OpsServer) {
this.opsServerRef.typedrouter.addTypedRouter(this.typedrouter);
this.registerHandlers();
this.setupBroadcast();
}
/**
* Wire up the logger's broadcast function to push sync log entries
* to all connected frontends via TypedSocket.
*/
private setupBroadcast(): void {
logger.setBroadcastFn((entry) => {
try {
const typedsocket = this.opsServerRef.server?.typedserver?.typedsocket;
if (!typedsocket) return;
typedsocket.findAllTargetConnectionsByTag('allClients').then((connections) => {
for (const conn of connections) {
typedsocket
.createTypedRequest<interfaces.requests.IReq_PushSyncLog>('pushSyncLog', conn)
.fire({ entry })
.catch(() => {});
}
}).catch(() => {});
} catch {
// Server may not be ready yet — ignore
}
});
}
private get syncManager() {
return this.opsServerRef.gitopsAppRef.syncManager;
}
private get actionLog() {
return this.opsServerRef.gitopsAppRef.actionLog;
}
private registerHandlers(): void {
// Get all sync configs
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetSyncConfigs>(
'getSyncConfigs',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
return { configs: this.syncManager.getConfigs() };
},
),
);
// Create sync config
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_CreateSyncConfig>(
'createSyncConfig',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
const config = await this.syncManager.createConfig({
name: dataArg.name,
sourceConnectionId: dataArg.sourceConnectionId,
targetConnectionId: dataArg.targetConnectionId,
targetGroupOffset: dataArg.targetGroupOffset,
intervalMinutes: dataArg.intervalMinutes,
enforceDelete: dataArg.enforceDelete,
enforceGroupDelete: dataArg.enforceGroupDelete,
addMirrorHint: dataArg.addMirrorHint,
});
this.actionLog.append({
actionType: 'create',
entityType: 'sync',
entityId: config.id,
entityName: config.name,
details: `Created sync config "${config.name}" (${config.intervalMinutes}m interval)`,
username: dataArg.identity.username,
});
return { config };
},
),
);
// Update sync config
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_UpdateSyncConfig>(
'updateSyncConfig',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
const config = await this.syncManager.updateConfig(dataArg.syncConfigId, {
name: dataArg.name,
targetGroupOffset: dataArg.targetGroupOffset,
intervalMinutes: dataArg.intervalMinutes,
enforceDelete: dataArg.enforceDelete,
enforceGroupDelete: dataArg.enforceGroupDelete,
addMirrorHint: dataArg.addMirrorHint,
});
this.actionLog.append({
actionType: 'update',
entityType: 'sync',
entityId: config.id,
entityName: config.name,
details: `Updated sync config "${config.name}"`,
username: dataArg.identity.username,
});
return { config };
},
),
);
// Delete sync config
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteSyncConfig>(
'deleteSyncConfig',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
const config = this.syncManager.getConfig(dataArg.syncConfigId);
await this.syncManager.deleteConfig(dataArg.syncConfigId);
this.actionLog.append({
actionType: 'delete',
entityType: 'sync',
entityId: dataArg.syncConfigId,
entityName: config?.name || dataArg.syncConfigId,
details: `Deleted sync config "${config?.name || dataArg.syncConfigId}"`,
username: dataArg.identity.username,
});
return { ok: true };
},
),
);
// Pause/resume sync config
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_PauseSyncConfig>(
'pauseSyncConfig',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
const config = await this.syncManager.pauseConfig(
dataArg.syncConfigId,
dataArg.paused,
);
this.actionLog.append({
actionType: dataArg.paused ? 'pause' : 'resume',
entityType: 'sync',
entityId: config.id,
entityName: config.name,
details: `${dataArg.paused ? 'Paused' : 'Resumed'} sync config "${config.name}"`,
username: dataArg.identity.username,
});
return { config };
},
),
);
// Trigger sync manually
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_TriggerSync>(
'triggerSync',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
const config = this.syncManager.getConfig(dataArg.syncConfigId);
if (!config) {
return { ok: false, message: 'Sync config not found' };
}
// Fire and forget — force=true bypasses paused check for manual triggers
this.syncManager.executeSync(dataArg.syncConfigId, true).catch((err) => {
console.error(`Manual sync trigger failed: ${err}`);
});
this.actionLog.append({
actionType: 'sync',
entityType: 'sync',
entityId: config.id,
entityName: config.name,
details: `Manually triggered sync "${config.name}"`,
username: dataArg.identity.username,
});
return { ok: true, message: 'Sync triggered' };
},
),
);
// Preview sync (dry run — shows source → target mappings)
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_PreviewSync>(
'previewSync',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
const result = await this.syncManager.previewSync(dataArg.syncConfigId);
return { mappings: result.mappings, deletions: result.deletions, groupDeletions: result.groupDeletions };
},
),
);
// Get repo statuses for a sync config
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetSyncRepoStatuses>(
'getSyncRepoStatuses',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
const statuses = await this.syncManager.getRepoStatuses(dataArg.syncConfigId);
return { statuses };
},
),
);
// Get recent sync log entries
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetSyncLogs>(
'getSyncLogs',
async (dataArg) => {
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
const logs = logger.getSyncLogs(dataArg.limit || 200);
return { logs };
},
),
);
}
}

View File

@@ -4,6 +4,7 @@ export interface IGitopsPaths {
gitopsHomeDir: string;
defaultStoragePath: string;
defaultTsmDbPath: string;
syncMirrorsPath: string;
}
/**
@@ -15,5 +16,6 @@ export function resolvePaths(baseDir?: string): IGitopsPaths {
gitopsHomeDir: home,
defaultStoragePath: path.join(home, 'storage'),
defaultTsmDbPath: path.join(home, 'tsmdb'),
syncMirrorsPath: path.join(home, 'mirrors'),
};
}

View File

@@ -16,11 +16,16 @@ export interface IListOptions {
* Subclasses implement Gitea API v1 or GitLab API v4.
*/
export abstract class BaseProvider {
public readonly groupFilterId?: string;
constructor(
public readonly connectionId: string,
public readonly baseUrl: string,
protected readonly token: string,
) {}
groupFilterId?: string,
) {
this.groupFilterId = groupFilterId;
}
// Connection
abstract testConnection(): Promise<ITestConnectionResult>;

View File

@@ -8,8 +8,8 @@ import { BaseProvider, type ITestConnectionResult, type IListOptions } from './c
export class GiteaProvider extends BaseProvider {
private client: plugins.giteaClient.GiteaClient;
constructor(connectionId: string, baseUrl: string, token: string) {
super(connectionId, baseUrl, token);
constructor(connectionId: string, baseUrl: string, token: string, groupFilterId?: string) {
super(connectionId, baseUrl, token, groupFilterId);
this.client = new plugins.giteaClient.GiteaClient(baseUrl, token);
}
@@ -18,9 +18,14 @@ export class GiteaProvider extends BaseProvider {
}
async getProjects(opts?: IListOptions): Promise<interfaces.data.IProject[]> {
// Use org-scoped listing when groupFilterId is set
const fetchFn = this.groupFilterId
? (o: IListOptions) => this.client.getOrgRepos(this.groupFilterId!, o)
: (o: IListOptions) => this.client.getRepos(o);
// If caller explicitly requests a specific page, respect it (no auto-pagination)
if (opts?.page) {
const repos = await this.client.getRepos(opts);
const repos = await fetchFn(opts);
return repos.map((r) => this.mapProject(r));
}
@@ -29,7 +34,7 @@ export class GiteaProvider extends BaseProvider {
let page = 1;
while (true) {
const repos = await this.client.getRepos({ ...opts, page, perPage });
const repos = await fetchFn({ ...opts, page, perPage });
allRepos.push(...repos);
if (repos.length < perPage) break;
page++;
@@ -39,6 +44,12 @@ export class GiteaProvider extends BaseProvider {
}
async getGroups(opts?: IListOptions): Promise<interfaces.data.IGroup[]> {
// When groupFilterId is set, return only that single org
if (this.groupFilterId) {
const org = await this.client.getOrg(this.groupFilterId);
return [this.mapGroup(org)];
}
// If caller explicitly requests a specific page, respect it (no auto-pagination)
if (opts?.page) {
const orgs = await this.client.getOrgs(opts);

View File

@@ -8,8 +8,8 @@ import { BaseProvider, type ITestConnectionResult, type IListOptions } from './c
export class GitLabProvider extends BaseProvider {
private client: plugins.gitlabClient.GitLabClient;
constructor(connectionId: string, baseUrl: string, token: string) {
super(connectionId, baseUrl, token);
constructor(connectionId: string, baseUrl: string, token: string, groupFilterId?: string) {
super(connectionId, baseUrl, token, groupFilterId);
this.client = new plugins.gitlabClient.GitLabClient(baseUrl, token);
}
@@ -18,13 +18,71 @@ export class GitLabProvider extends BaseProvider {
}
async getProjects(opts?: IListOptions): Promise<interfaces.data.IProject[]> {
const projects = await this.client.getProjects(opts);
return projects.map((p) => this.mapProject(p));
if (this.groupFilterId) {
// Auto-paginate group-scoped project listing
if (opts?.page) {
const projects = await this.client.getGroupProjects(this.groupFilterId, opts);
return projects.map((p) => this.mapProject(p));
}
const allProjects: plugins.gitlabClient.IGitLabProject[] = [];
const perPage = opts?.perPage || 50;
let page = 1;
while (true) {
const projects = await this.client.getGroupProjects(this.groupFilterId, { ...opts, page, perPage });
allProjects.push(...projects);
if (projects.length < perPage) break;
page++;
}
return allProjects.map((p) => this.mapProject(p));
}
if (opts?.page) {
const projects = await this.client.getProjects(opts);
return projects.map((p) => this.mapProject(p));
}
const allProjects: plugins.gitlabClient.IGitLabProject[] = [];
const perPage = opts?.perPage || 50;
let page = 1;
while (true) {
const projects = await this.client.getProjects({ ...opts, page, perPage });
allProjects.push(...projects);
if (projects.length < perPage) break;
page++;
}
return allProjects.map((p) => this.mapProject(p));
}
async getGroups(opts?: IListOptions): Promise<interfaces.data.IGroup[]> {
const groups = await this.client.getGroups(opts);
return groups.map((g) => this.mapGroup(g));
if (this.groupFilterId) {
// Auto-paginate descendant groups listing
if (opts?.page) {
const groups = await this.client.getDescendantGroups(this.groupFilterId, opts);
return groups.map((g) => this.mapGroup(g));
}
const allGroups: plugins.gitlabClient.IGitLabGroup[] = [];
const perPage = opts?.perPage || 50;
let page = 1;
while (true) {
const groups = await this.client.getDescendantGroups(this.groupFilterId, { ...opts, page, perPage });
allGroups.push(...groups);
if (groups.length < perPage) break;
page++;
}
return allGroups.map((g) => this.mapGroup(g));
}
if (opts?.page) {
const groups = await this.client.getGroups(opts);
return groups.map((g) => this.mapGroup(g));
}
const allGroups: plugins.gitlabClient.IGitLabGroup[] = [];
const perPage = opts?.perPage || 50;
let page = 1;
while (true) {
const groups = await this.client.getGroups({ ...opts, page, perPage });
allGroups.push(...groups);
if (groups.length < perPage) break;
page++;
}
return allGroups.map((g) => this.mapGroup(g));
}
// --- Project Secrets (CI/CD Variables) ---