serve.zone/gitops
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sync): add branch & tag listing support and improve sync mirroring and sync log routing

Browse Source
This commit is contained in:
Jürgen Kunz
2026-03-02 10:37:07 +00:00
parent 75d35405dc
commit 56403224c0
16 changed files with 411 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ts/classes/gitopsapp.ts
View File

@@ -70,7 +70,6 @@ export class GitopsApp {
this.storageManager,
this.connectionManager,
this.actionLog,
this.paths.syncMirrorsPath,
);
await this.syncManager.init();

351
ts/classes/syncmanager.ts
View File

@@ -4,6 +4,7 @@ import type * as interfaces from '../../ts_interfaces/index.ts';
import type { ConnectionManager } from './connectionmanager.ts';
import type { ActionLog } from './actionlog.ts';
import type { StorageManager } from '../storage/index.ts';
import type { BaseProvider } from '../providers/classes.baseprovider.ts';
const SYNC_PREFIX = '/sync/';
const SYNC_STATUS_PREFIX = '/sync-status/';
@@ -19,15 +20,19 @@ export class SyncManager {
private runningSync: Set<string> = new Set();
private syncedGroupMeta: Set<string> = new Set();
private currentSyncConfig: interfaces.data.ISyncConfig | null = null;
private avatarUploadCache: Map<string, string> = new Map();
private mirrorsPath = '';
constructor(
private storageManager: StorageManager,
private connectionManager: ConnectionManager,
private actionLog: ActionLog,
private mirrorsPath: string,
) {}
async init(): Promise<void> {
// Create temp directory for mirrors (RAM-backed on most Linux systems via tmpfs)
this.mirrorsPath = await Deno.makeTempDir({ prefix: 'gitops-mirrors-' });
await this.loadConfigs();
for (const config of this.configs) {
if (config.status === 'active') {
@@ -44,6 +49,12 @@ export class SyncManager {
clearInterval(timer);
}
this.timers.clear();
// Clean up temp mirrors directory
if (this.mirrorsPath) {
try {
await Deno.remove(this.mirrorsPath, { recursive: true });
} catch { /* may already be gone */ }
}
}
// ============================================================================
@@ -290,7 +301,7 @@ export class SyncManager {
const batch = projects.slice(i, i + CONCURRENCY);
await Promise.all(batch.map(async (project) => {
try {
logger.syncLog('info', `Syncing ${project.fullPath}...`, 'git');
logger.syncLog('info', `Syncing ${project.fullPath}...`, 'sync');
await this.syncRepo(config, project, sourceConn, targetConn);
synced++;
await this.updateRepoStatus(config.id, project.fullPath, {
@@ -298,7 +309,7 @@ export class SyncManager {
lastSyncAt: Date.now(),
lastSyncError: undefined,
});
logger.syncLog('success', `Synced ${project.fullPath}`, 'git');
logger.syncLog('success', `Synced ${project.fullPath}`, 'sync');
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
await this.updateRepoStatus(config.id, project.fullPath, {
@@ -306,7 +317,7 @@ export class SyncManager {
lastSyncError: errMsg,
lastSyncAt: Date.now(),
});
logger.syncLog('error', `Sync failed for ${project.fullPath}: ${errMsg}`, 'git');
logger.syncLog('error', `Sync failed for ${project.fullPath}: ${errMsg}`, 'sync');
}
}));
}
@@ -372,21 +383,38 @@ export class SyncManager {
// Ensure target group/project hierarchy exists
await this.ensureTargetExists(targetConn, targetFullPath, project, sourceConn, sourceConn.groupFilter, config.targetGroupOffset);
// API-based ref comparison (fast path — avoids git clone when refs already match)
const sourceProvider = this.connectionManager.getProvider(sourceConn.id);
const targetProvider = this.connectionManager.getProvider(targetConn.id);
const apiRefsMatch = await this.refsMatchViaApi(
sourceProvider, targetProvider, project.fullPath, targetFullPath,
);
if (apiRefsMatch === true) {
logger.syncLog('info', `Refs match via API for ${project.fullPath}, skipping git`, 'api');
await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetFullPath);
return;
}
// Clone or fetch from source
try {
const exists = await this.dirExists(mirrorDir);
if (!exists) {
await Deno.mkdir(mirrorDir, { recursive: true });
await this.runGit(['clone', '--bare', sourceUrl, '.'], mirrorDir);
} else {
// Update source remote URL in case it changed
try {
await this.runGit(['remote', 'set-url', 'origin', sourceUrl], mirrorDir);
} catch {
// Ignore errors
}
await this.runGit(['fetch', '--prune', 'origin'], mirrorDir);
}
// Ensure fetch refspec is configured (bare clones don't set one by default,
// which prevents tracking branch renames like master -> main)
await this.runGit(
['config', 'remote.origin.fetch', '+refs/heads/*:refs/heads/*'], mirrorDir,
);
// Update source remote URL in case connection changed
try {
await this.runGit(['remote', 'set-url', 'origin', sourceUrl], mirrorDir);
} catch {
// Ignore errors
}
// Fetch latest refs from source (--prune removes branches deleted on remote)
await this.runGit(['fetch', '--prune', 'origin'], mirrorDir);
} catch (err: any) {
const msg = err instanceof Error ? err.message : String(err);
if (msg.includes("couldn't find remote ref HEAD")) {
@@ -421,26 +449,33 @@ export class SyncManager {
});
}
// Phase 1: push all refs without pruning (ensures target has all source branches)
await this.runGit([
'push', 'target',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*',
], mirrorDir);
// Compare refs to determine if push is needed
const refsAlreadyMatch = !isUnrelated && await this.refsMatch(mirrorDir);
// Phase 2: sync default_branch now that all branches exist on target
await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath, targetFullPath);
if (refsAlreadyMatch) {
logger.syncLog('info', `Refs already match for ${project.fullPath}, skipping push`, 'api');
} else {
// Phase 1: push all refs without pruning (ensures target has all source branches)
await this.runGit([
'push', 'target',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*',
], mirrorDir);
// Phase 2b: unprotect stale branches on target so --prune can delete them
await this.unprotectStaleBranches(targetConn, targetFullPath, mirrorDir);
// Phase 2: sync default_branch now that all branches exist on target
await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath, targetFullPath);
// Phase 3: push with --prune to remove stale branches (safe now that default_branch is correct)
await this.runGit([
'push', 'target',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*',
'--prune',
], mirrorDir);
// Phase 2b: unprotect stale branches on target so --prune can delete them
await this.unprotectStaleBranches(targetConn, targetFullPath, mirrorDir);
// Phase 3: push with --prune to remove stale branches (safe now that default_branch is correct)
await this.runGit([
'push', 'target',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*',
'--prune',
], mirrorDir);
}
// Sync project metadata (description, visibility, topics, default_branch, avatar)
await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetFullPath);
@@ -1007,10 +1042,14 @@ export class SyncManager {
if (data[0] === 0x89 && data[1] === 0x50) return 'image/png';
if (data[0] === 0xFF && data[1] === 0xD8) return 'image/jpeg';
if (data[0] === 0x47 && data[1] === 0x49) return 'image/gif';
// SVG: text-based XML, no magic bytes — check content
const textStart = new TextDecoder().decode(data.slice(0, 200));
if (textStart.includes('<svg') || textStart.includes('<?xml')) return 'image/svg+xml';
// Fallback: check URL extension
if (url.includes('.png')) return 'image/png';
if (url.includes('.jpg') || url.includes('.jpeg')) return 'image/jpeg';
if (url.includes('.gif')) return 'image/gif';
if (url.includes('.svg')) return 'image/svg+xml';
return 'image/png'; // default
}
@@ -1146,7 +1185,6 @@ export class SyncManager {
if (sourceGroup) {
const groupMeta = this.extractGroupMeta(sourceConn, sourceGroup);
if (groupMeta.avatarUrl) {
logger.syncLog('info', `Applying group avatar to ${targetFullPath}`, 'api');
await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, groupMeta.avatarUrl, targetProject);
groupAvatarApplied = true;
}
@@ -1293,24 +1331,41 @@ export class SyncManager {
meta: { description: string; visibility: string; topics: string[]; defaultBranch: string },
): Promise<void> {
if (conn.providerType === 'gitlab') {
// Update description, visibility, topics (always safe)
await this.rawApiCall(conn, 'PUT', `/api/v4/projects/${rawProject.id}`, {
description: meta.description,
visibility: this.normalizeVisibility(meta.visibility),
topics: meta.topics,
default_branch: meta.defaultBranch,
});
// Update default_branch separately — may fail if the branch doesn't exist in git
try {
await this.rawApiCall(conn, 'PUT', `/api/v4/projects/${rawProject.id}`, {
default_branch: meta.defaultBranch,
});
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
logger.syncLog('warn', `Could not set default_branch to "${meta.defaultBranch}" for ${fullPath}: ${errMsg}`, 'api');
}
} else {
const segments = fullPath.split('/');
const repo = segments.pop()!;
const owner = segments[0] || '';
const encodedOwner = encodeURIComponent(owner);
const encodedRepo = encodeURIComponent(repo);
// Update description, visibility, default_branch
// Update description, visibility
await this.rawApiCall(conn, 'PATCH', `/api/v1/repos/${encodedOwner}/${encodedRepo}`, {
description: meta.description,
private: this.normalizeVisibility(meta.visibility) === 'private',
default_branch: meta.defaultBranch,
});
// Update default_branch separately — may fail if the branch doesn't exist in git
try {
await this.rawApiCall(conn, 'PATCH', `/api/v1/repos/${encodedOwner}/${encodedRepo}`, {
default_branch: meta.defaultBranch,
});
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
logger.syncLog('warn', `Could not set default_branch to "${meta.defaultBranch}" for ${fullPath}: ${errMsg}`, 'api');
}
// Topics are a separate endpoint in Gitea
await this.rawApiCall(conn, 'PUT', `/api/v1/repos/${encodedOwner}/${encodedRepo}/topics`, {
topics: meta.topics,
@@ -1355,25 +1410,55 @@ export class SyncManager {
private async syncProjectAvatar(
sourceConn: interfaces.data.IProviderConnection,
targetConn: interfaces.data.IProviderConnection,
sourceFullPath: string,
_sourceFullPath: string,
targetFullPath: string,
sourceAvatarUrl: string,
targetRawProject: any,
): Promise<void> {
// Resolve relative avatar URLs
const resolvedUrl = sourceAvatarUrl.startsWith('http')
const resolvedSourceUrl = sourceAvatarUrl.startsWith('http')
? sourceAvatarUrl
: `${sourceConn.baseUrl.replace(/\/+$/, '')}${sourceAvatarUrl}`;
const avatarData = await this.rawBinaryFetch(sourceConn, resolvedUrl);
if (!avatarData || avatarData.length === 0) return;
const sourceAvatarData = await this.rawBinaryFetch(sourceConn, resolvedSourceUrl);
if (!sourceAvatarData || sourceAvatarData.length === 0) return;
// Skip SVG avatars — not supported by GitLab project endpoints
const mimeType = this.guessAvatarMimeType(sourceAvatarData, resolvedSourceUrl);
if (mimeType === 'image/svg+xml') {
logger.syncLog('warn', `Skipping SVG avatar for ${targetFullPath} (not supported by target)`, 'api');
return;
}
// Check in-memory cache: skip if source hasn't changed since last upload
const sourceHash = await this.hashBytes(sourceAvatarData);
const cacheKey = `project:${targetFullPath}`;
if (this.avatarUploadCache.get(cacheKey) === sourceHash) {
return; // Source avatar unchanged since last upload
}
// Compare with target's current avatar to avoid unnecessary uploads
const targetMeta = this.extractProjectMeta(targetConn, targetRawProject);
if (targetMeta.avatarUrl) {
try {
const resolvedTargetUrl = targetMeta.avatarUrl.startsWith('http')
? targetMeta.avatarUrl
: `${targetConn.baseUrl.replace(/\/+$/, '')}${targetMeta.avatarUrl}`;
const targetAvatarData = await this.rawBinaryFetch(targetConn, resolvedTargetUrl);
if (targetAvatarData && this.binaryEqual(sourceAvatarData, targetAvatarData)) {
this.avatarUploadCache.set(cacheKey, sourceHash);
return; // Avatars are identical — skip upload
}
} catch {
// Failed to fetch target avatar — proceed with upload as safe fallback
}
}
logger.syncLog('info', `Syncing avatar for ${targetFullPath}...`, 'api');
if (targetConn.providerType === 'gitlab') {
// GitLab: multipart upload
const mimeType = this.guessAvatarMimeType(avatarData, resolvedUrl);
const blob = new Blob([avatarData.buffer as ArrayBuffer], { type: mimeType });
const blob = new Blob([sourceAvatarData.buffer as ArrayBuffer], { type: mimeType });
const ext = mimeType.split('/')[1] || 'png';
const formData = new FormData();
formData.append('avatar', blob, `avatar.${ext}`);
@@ -1387,13 +1472,15 @@ export class SyncManager {
const segments = targetFullPath.split('/');
const repo = segments.pop()!;
const owner = segments[0] || '';
const base64Image = this.uint8ArrayToBase64(avatarData);
const base64Image = this.uint8ArrayToBase64(sourceAvatarData);
await this.rawApiCall(
targetConn, 'POST',
`/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/avatar`,
{ image: base64Image },
);
}
this.avatarUploadCache.set(cacheKey, sourceHash);
}
private async removeProjectAvatar(
@@ -1423,18 +1510,48 @@ export class SyncManager {
sourceAvatarUrl: string,
targetRawGroup: any,
): Promise<void> {
const resolvedUrl = sourceAvatarUrl.startsWith('http')
const resolvedSourceUrl = sourceAvatarUrl.startsWith('http')
? sourceAvatarUrl
: `${sourceConn.baseUrl.replace(/\/+$/, '')}${sourceAvatarUrl}`;
const avatarData = await this.rawBinaryFetch(sourceConn, resolvedUrl);
if (!avatarData || avatarData.length === 0) return;
const sourceAvatarData = await this.rawBinaryFetch(sourceConn, resolvedSourceUrl);
if (!sourceAvatarData || sourceAvatarData.length === 0) return;
// Skip SVG avatars — not supported by GitLab project endpoints
const mimeType = this.guessAvatarMimeType(sourceAvatarData, resolvedSourceUrl);
if (mimeType === 'image/svg+xml') {
logger.syncLog('warn', `Skipping SVG avatar for group ${targetGroupPath} (not supported by target)`, 'api');
return;
}
// Check in-memory cache: skip if source hasn't changed since last upload
const sourceHash = await this.hashBytes(sourceAvatarData);
const cacheKey = `group:${targetGroupPath}`;
if (this.avatarUploadCache.get(cacheKey) === sourceHash) {
return; // Source avatar unchanged since last upload
}
// Compare with target's current avatar to avoid unnecessary uploads
const targetMeta = this.extractGroupMeta(targetConn, targetRawGroup);
if (targetMeta.avatarUrl) {
try {
const resolvedTargetUrl = targetMeta.avatarUrl.startsWith('http')
? targetMeta.avatarUrl
: `${targetConn.baseUrl.replace(/\/+$/, '')}${targetMeta.avatarUrl}`;
const targetAvatarData = await this.rawBinaryFetch(targetConn, resolvedTargetUrl);
if (targetAvatarData && this.binaryEqual(sourceAvatarData, targetAvatarData)) {
this.avatarUploadCache.set(cacheKey, sourceHash);
return; // Avatars are identical — skip upload
}
} catch {
// Failed to fetch target avatar — proceed with upload as safe fallback
}
}
logger.syncLog('info', `Syncing avatar for group ${targetGroupPath}...`, 'api');
if (targetConn.providerType === 'gitlab') {
const mimeType = this.guessAvatarMimeType(avatarData, resolvedUrl);
const blob = new Blob([avatarData.buffer as ArrayBuffer], { type: mimeType });
const blob = new Blob([sourceAvatarData.buffer as ArrayBuffer], { type: mimeType });
const ext = mimeType.split('/')[1] || 'png';
const formData = new FormData();
formData.append('avatar', blob, `avatar.${ext}`);
@@ -1445,13 +1562,15 @@ export class SyncManager {
);
} else {
const orgName = targetGroupPath.split('/')[0] || targetGroupPath;
const base64Image = this.uint8ArrayToBase64(avatarData);
const base64Image = this.uint8ArrayToBase64(sourceAvatarData);
await this.rawApiCall(
targetConn, 'POST',
`/api/v1/orgs/${encodeURIComponent(orgName)}/avatar`,
{ image: base64Image },
);
}
this.avatarUploadCache.set(cacheKey, sourceHash);
}
private async removeGroupAvatar(
@@ -1471,6 +1590,19 @@ export class SyncManager {
}
}
private binaryEqual(a: Uint8Array, b: Uint8Array): boolean {
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) return false;
}
return true;
}
private async hashBytes(data: Uint8Array): Promise<string> {
const hashBuffer = await crypto.subtle.digest('SHA-256', data.buffer as ArrayBuffer);
return Array.from(new Uint8Array(hashBuffer)).map(b => b.toString(16).padStart(2, '0')).join('');
}
private uint8ArrayToBase64(bytes: Uint8Array): string {
let binary = '';
for (let i = 0; i < bytes.length; i++) {
@@ -1690,6 +1822,133 @@ export class SyncManager {
}
}
/**
* Fetch all branch and tag SHAs from a repo via provider API.
* Returns null on any error (safe fallback to git-based comparison).
*/
private async listRefsViaProvider(
provider: BaseProvider,
fullPath: string,
): Promise<{ branches: Map<string, string>; tags: Map<string, string> } | null> {
try {
const [branches, tags] = await Promise.all([
provider.getBranches(fullPath),
provider.getTags(fullPath),
]);
return {
branches: new Map(branches.map((b) => [b.name, b.commitSha])),
tags: new Map(tags.map((t) => [t.name, t.commitSha])),
};
} catch {
return null;
}
}
/**
* Compare refs between source and target via provider API (no git clone needed).
* Returns true (match), false (differ), or null (can't determine — fall through to git).
*/
private async refsMatchViaApi(
sourceProvider: BaseProvider,
targetProvider: BaseProvider,
sourceFullPath: string,
targetFullPath: string,
): Promise<boolean | null> {
const [sourceRefs, targetRefs] = await Promise.all([
this.listRefsViaProvider(sourceProvider, sourceFullPath),
this.listRefsViaProvider(targetProvider, targetFullPath),
]);
if (!sourceRefs || !targetRefs) return null;
// Compare branches
if (sourceRefs.branches.size !== targetRefs.branches.size) return false;
for (const [name, sha] of sourceRefs.branches) {
if (targetRefs.branches.get(name) !== sha) return false;
}
// Compare tags
if (sourceRefs.tags.size !== targetRefs.tags.size) return false;
for (const [name, sha] of sourceRefs.tags) {
if (targetRefs.tags.get(name) !== sha) return false;
}
return true;
}
/**
* Compare local refs (source) with target remote refs.
* Returns true when all branches and tags are identical — no push needed.
*/
private async refsMatch(mirrorDir: string): Promise<boolean> {
try {
// Local branches (source)
const localHeadsRaw = await this.runGit(
['for-each-ref', '--format=%(refname:strip=2) %(objectname)', 'refs/heads/'], mirrorDir,
);
// Target branches (fetched by checkUnrelatedHistory)
const targetHeadsRaw = await this.runGit(
['for-each-ref', '--format=%(refname:strip=3) %(objectname)', 'refs/remotes/target/'], mirrorDir,
);
// Local tags
const localTagsRaw = await this.runGit(
['for-each-ref', '--format=%(refname:strip=2) %(objectname)', 'refs/tags/'], mirrorDir,
);
// Target tags via ls-remote (avoids shared refs/tags/ namespace ambiguity in bare repos)
const targetTagsRaw = await this.runGit(['ls-remote', '--tags', 'target'], mirrorDir);
const parseRefLines = (raw: string): Map<string, string> => {
const map = new Map<string, string>();
for (const line of raw.trim().split('\n')) {
if (!line.trim()) continue;
const parts = line.trim().split(/\s+/);
if (parts.length >= 2) {
map.set(parts[0], parts[1]);
}
}
return map;
};
const parseLsRemoteTags = (raw: string): Map<string, string> => {
const map = new Map<string, string>();
for (const line of raw.trim().split('\n')) {
if (!line.trim()) continue;
// Skip ^{} dereference lines
if (line.includes('^{}')) continue;
const parts = line.trim().split(/\s+/);
if (parts.length >= 2) {
// parts[0] = sha, parts[1] = refs/tags/name
const tagName = parts[1].replace('refs/tags/', '');
map.set(tagName, parts[0]);
}
}
return map;
};
const localHeads = parseRefLines(localHeadsRaw);
const targetHeads = parseRefLines(targetHeadsRaw);
const localTags = parseRefLines(localTagsRaw);
const targetTags = parseLsRemoteTags(targetTagsRaw);
// Compare branches
if (localHeads.size !== targetHeads.size) return false;
for (const [name, sha] of localHeads) {
if (targetHeads.get(name) !== sha) return false;
}
// Compare tags
if (localTags.size !== targetTags.size) return false;
for (const [name, sha] of localTags) {
if (targetTags.get(name) !== sha) return false;
}
return true;
} catch {
// On any error, fall back to pushing (safe default)
return false;
}
}
private async runGit(args: string[], cwd?: string): Promise<string> {
const cmd = new Deno.Command('git', {
args,