import * as plugins from '../plugins.ts'; import { logger } from '../logging.ts'; import type * as interfaces from '../../ts_interfaces/index.ts'; import type { ConnectionManager } from './connectionmanager.ts'; import type { ActionLog } from './actionlog.ts'; import type { StorageManager } from '../storage/index.ts'; import type { BaseProvider } from '../providers/classes.baseprovider.ts'; const SYNC_PREFIX = '/sync/'; const SYNC_STATUS_PREFIX = '/sync-status/'; /** * Manages sync configurations and executes periodic git mirror operations. * Each sync config defines a source → target connection mapping. * Repos are mirrored using bare git repos stored on disk. */ export class SyncManager { private configs: interfaces.data.ISyncConfig[] = []; private timers: Map = new Map(); private runningSync: Set = new Set(); private syncedGroupMeta: Set = new Set(); private currentSyncConfig: interfaces.data.ISyncConfig | null = null; private avatarUploadCache: Map = new Map(); private mirrorsPath = ''; constructor( private storageManager: StorageManager, private connectionManager: ConnectionManager, private actionLog: ActionLog, ) {} async init(): Promise { // Create temp directory for mirrors (RAM-backed on most Linux systems via tmpfs) this.mirrorsPath = await Deno.makeTempDir({ prefix: 'gitops-mirrors-' }); await this.loadConfigs(); for (const config of this.configs) { if (config.status === 'active') { this.startTimer(config); } } if (this.configs.length > 0) { logger.info(`SyncManager loaded ${this.configs.length} sync config(s)`); } } async stop(): Promise { for (const [_id, timer] of this.timers) { clearInterval(timer); } this.timers.clear(); // Clean up temp mirrors directory if (this.mirrorsPath) { try { await Deno.remove(this.mirrorsPath, { recursive: true }); } catch { /* may already be gone */ } } } // ============================================================================ // CRUD // ============================================================================ getConfigs(): interfaces.data.ISyncConfig[] { return [...this.configs]; } getConfig(id: string): interfaces.data.ISyncConfig | undefined { return this.configs.find((c) => c.id === id); } async createConfig(data: { name: string; sourceConnectionId: string; targetConnectionId: string; targetGroupOffset?: string; intervalMinutes?: number; enforceDelete?: boolean; enforceGroupDelete?: boolean; addMirrorHint?: boolean; useGroupAvatarsForProjects?: boolean; }): Promise { const config: interfaces.data.ISyncConfig = { id: crypto.randomUUID(), name: data.name, sourceConnectionId: data.sourceConnectionId, targetConnectionId: data.targetConnectionId, targetGroupOffset: data.targetGroupOffset, intervalMinutes: data.intervalMinutes || 5, status: 'paused', lastSyncAt: 0, reposSynced: 0, enforceDelete: data.enforceDelete ?? false, enforceGroupDelete: data.enforceGroupDelete ?? false, addMirrorHint: data.addMirrorHint ?? false, useGroupAvatarsForProjects: data.useGroupAvatarsForProjects ?? false, createdAt: Date.now(), }; this.validateSyncConfig(config); this.configs.push(config); await this.persistConfig(config); logger.success(`Sync config created (paused): ${config.name}`); return config; } async updateConfig( id: string, updates: { name?: string; targetGroupOffset?: string; intervalMinutes?: number; enforceDelete?: boolean; enforceGroupDelete?: boolean; addMirrorHint?: boolean; useGroupAvatarsForProjects?: boolean }, ): Promise { const config = this.configs.find((c) => c.id === id); if (!config) throw new Error(`Sync config not found: ${id}`); if (updates.name) config.name = updates.name; if (updates.intervalMinutes) config.intervalMinutes = updates.intervalMinutes; if (updates.enforceDelete !== undefined) config.enforceDelete = updates.enforceDelete; if (updates.enforceGroupDelete !== undefined) config.enforceGroupDelete = updates.enforceGroupDelete; if (updates.addMirrorHint !== undefined) config.addMirrorHint = updates.addMirrorHint; if (updates.useGroupAvatarsForProjects !== undefined) config.useGroupAvatarsForProjects = updates.useGroupAvatarsForProjects; if (updates.targetGroupOffset !== undefined) config.targetGroupOffset = updates.targetGroupOffset; this.validateSyncConfig(config); await this.persistConfig(config); // Restart timer with new interval if (config.status === 'active') { this.startTimer(config); } return config; } async deleteConfig(id: string): Promise { const idx = this.configs.findIndex((c) => c.id === id); if (idx === -1) throw new Error(`Sync config not found: ${id}`); this.stopTimer(id); this.configs.splice(idx, 1); await this.storageManager.delete(`${SYNC_PREFIX}${id}.json`); // Clean up repo statuses const statusKeys = await this.storageManager.list(`${SYNC_STATUS_PREFIX}${id}/`); for (const key of statusKeys) { await this.storageManager.delete(key); } // Clean up mirror directory const mirrorDir = plugins.path.join(this.mirrorsPath, id); try { await Deno.remove(mirrorDir, { recursive: true }); } catch { // Directory may not exist } logger.info(`Sync config deleted: ${id}`); } async pauseConfig(id: string, paused: boolean): Promise { const config = this.configs.find((c) => c.id === id); if (!config) throw new Error(`Sync config not found: ${id}`); config.status = paused ? 'paused' : 'active'; await this.persistConfig(config); if (paused) { this.stopTimer(id); } else { this.startTimer(config); } logger.info(`Sync config ${paused ? 'paused' : 'resumed'}: ${config.name}`); return config; } async getRepoStatuses(syncConfigId: string): Promise { const keys = await this.storageManager.list(`${SYNC_STATUS_PREFIX}${syncConfigId}/`); const statuses: interfaces.data.ISyncRepoStatus[] = []; for (const key of keys) { const status = await this.storageManager.getJSON(key); if (status) statuses.push(status); } return statuses; } // ============================================================================ // Preview // ============================================================================ async previewSync(configId: string): Promise<{ mappings: Array<{ sourceFullPath: string; targetFullPath: string }>; deletions: string[]; groupDeletions: string[]; }> { const config = this.configs.find((c) => c.id === configId); if (!config) throw new Error(`Sync config not found: ${configId}`); logger.syncLog('info', `Preview started for "${config.name}"`, 'preview'); const sourceConn = this.connectionManager.getConnection(config.sourceConnectionId); const targetConn = this.connectionManager.getConnection(config.targetConnectionId); if (!sourceConn) throw new Error(`Source connection not found: ${config.sourceConnectionId}`); if (!targetConn) throw new Error(`Target connection not found: ${config.targetConnectionId}`); const sourceProvider = this.connectionManager.getProvider(config.sourceConnectionId); const targetProvider = this.connectionManager.getProvider(config.targetConnectionId); logger.syncLog('info', `Fetching source projects from "${sourceConn.name}"...`, 'preview'); const allProjects = await sourceProvider.getProjects(); const projects = allProjects.filter(p => !this.isObsoletePath(p.fullPath)); logger.syncLog('info', `Found ${projects.length} source projects (${allProjects.length - projects.length} obsolete excluded)`, 'preview'); const mappings = projects.map((project) => { const targetFullPath = this.computeTargetFullPath( project.fullPath, sourceConn.groupFilter, config.targetGroupOffset, ); return { sourceFullPath: project.fullPath, targetFullPath }; }); // Compute repo deletions when enforce-delete is enabled let deletions: string[] = []; if (config.enforceDelete) { logger.syncLog('info', 'Computing repo deletions (enforce-delete enabled)...', 'preview'); const expectedTargetPaths = new Set( mappings.map((m) => m.targetFullPath.toLowerCase()), ); const scopePrefix = config.targetGroupOffset; const targetProjects = await targetProvider.getProjects(); for (const tp of targetProjects) { if (this.isObsoletePath(tp.fullPath)) continue; if (scopePrefix && !tp.fullPath.toLowerCase().startsWith(scopePrefix.toLowerCase() + '/')) { continue; } if (!expectedTargetPaths.has(tp.fullPath.toLowerCase())) { deletions.push(tp.fullPath); } } } // Compute group deletions when enforce-group-delete is enabled let groupDeletions: string[] = []; if (config.enforceGroupDelete) { logger.syncLog('info', 'Computing group deletions (enforce-group-delete enabled)...', 'preview'); const sourceGroups = await sourceProvider.getGroups(); const expectedTargetGroups = new Set(); for (const sg of sourceGroups) { const targetPath = this.computeTargetFullPath( sg.fullPath, sourceConn.groupFilter, config.targetGroupOffset, ); expectedTargetGroups.add(targetPath.toLowerCase()); } // Also include the offset itself and the obsolete group if (config.targetGroupOffset) { expectedTargetGroups.add(config.targetGroupOffset.toLowerCase()); expectedTargetGroups.add(`${config.targetGroupOffset}/obsolete`.toLowerCase()); } const targetGroups = await targetProvider.getGroups(); const scopePrefix = config.targetGroupOffset; for (const tg of targetGroups) { if (this.isObsoletePath(tg.fullPath)) continue; if (scopePrefix && !tg.fullPath.toLowerCase().startsWith(scopePrefix.toLowerCase() + '/')) { continue; } if (!expectedTargetGroups.has(tg.fullPath.toLowerCase())) { groupDeletions.push(tg.fullPath); } } } logger.syncLog('success', `Preview complete: ${mappings.length} mappings, ${deletions.length} deletions, ${groupDeletions.length} group deletions`, 'preview'); return { mappings, deletions, groupDeletions }; } // ============================================================================ // Sync Engine // ============================================================================ async executeSync(configId: string, force = false): Promise { if (this.runningSync.has(configId)) { logger.warn(`Sync ${configId} already running, skipping`); return; } const config = this.configs.find((c) => c.id === configId); if (!config) return; if (config.status === 'paused' && !force) return; this.runningSync.add(configId); this.syncedGroupMeta.clear(); this.currentSyncConfig = config; const startTime = Date.now(); logger.syncLog('info', `Sync started for "${config.name}"`, 'sync'); try { const sourceConn = this.connectionManager.getConnection(config.sourceConnectionId); const targetConn = this.connectionManager.getConnection(config.targetConnectionId); if (!sourceConn) throw new Error(`Source connection not found: ${config.sourceConnectionId}`); if (!targetConn) throw new Error(`Target connection not found: ${config.targetConnectionId}`); this.validateSyncConfig(config); // Get all projects from source const sourceProvider = this.connectionManager.getProvider(config.sourceConnectionId); logger.syncLog('info', `Fetching source projects from "${sourceConn.name}"...`, 'api'); const allProjects = await sourceProvider.getProjects(); const projects = allProjects.filter(p => !this.isObsoletePath(p.fullPath)); logger.syncLog('info', `Found ${projects.length} source projects (${allProjects.length - projects.length} obsolete excluded)`, 'api'); let synced = 0; const CONCURRENCY = 10; for (let i = 0; i < projects.length; i += CONCURRENCY) { const batch = projects.slice(i, i + CONCURRENCY); await Promise.all(batch.map(async (project) => { try { logger.syncLog('info', `Syncing ${project.fullPath}...`, 'sync'); await this.syncRepo(config, project, sourceConn, targetConn); synced++; await this.updateRepoStatus(config.id, project.fullPath, { status: 'synced', lastSyncAt: Date.now(), lastSyncError: undefined, }); logger.syncLog('success', `Synced ${project.fullPath}`, 'sync'); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); await this.updateRepoStatus(config.id, project.fullPath, { status: 'error', lastSyncError: errMsg, lastSyncAt: Date.now(), }); logger.syncLog('error', `Sync failed for ${project.fullPath}: ${errMsg}`, 'sync'); } })); } // Enforce deletion: move stale target repos to obsolete if (config.enforceDelete) { logger.syncLog('info', 'Checking for stale target repos...', 'sync'); await this.enforceDeleteStaleRepos(config, projects, sourceConn, targetConn); } // Enforce group deletion: move stale target groups to obsolete if (config.enforceGroupDelete) { logger.syncLog('info', 'Checking for stale target groups...', 'sync'); await this.enforceDeleteStaleGroups(config, sourceConn, targetConn); } config.lastSyncAt = Date.now(); config.reposSynced = synced; config.lastSyncDurationMs = Date.now() - startTime; config.lastSyncError = undefined; if (config.status === 'error') config.status = 'active'; await this.persistConfig(config); logger.syncLog('success', `Sync complete for "${config.name}": ${synced}/${projects.length} repos in ${config.lastSyncDurationMs}ms`, 'sync'); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); config.lastSyncError = errMsg; config.lastSyncAt = Date.now(); config.lastSyncDurationMs = Date.now() - startTime; config.status = 'error'; await this.persistConfig(config); logger.syncLog('error', `Sync config "${config.name}" failed: ${errMsg}`, 'sync'); } finally { this.runningSync.delete(configId); } } // ============================================================================ // Single Repo Sync // ============================================================================ private async syncRepo( config: interfaces.data.ISyncConfig, project: interfaces.data.IProject, sourceConn: interfaces.data.IProviderConnection, targetConn: interfaces.data.IProviderConnection, ): Promise { const targetFullPath = this.computeTargetFullPath( project.fullPath, sourceConn.groupFilter, config.targetGroupOffset, ); // Build authenticated git URLs const sourceUrl = this.buildAuthUrl(sourceConn, project.fullPath); const targetUrl = this.buildAuthUrl(targetConn, targetFullPath); // Mirror directory for this repo const mirrorDir = plugins.path.join( this.mirrorsPath, config.id, this.sanitizePath(project.fullPath), ); // Ensure target group/project hierarchy exists await this.ensureTargetExists(targetConn, targetFullPath, project, sourceConn, sourceConn.groupFilter, config.targetGroupOffset); // API-based ref comparison (fast path — avoids git clone when refs already match) const sourceProvider = this.connectionManager.getProvider(sourceConn.id); const targetProvider = this.connectionManager.getProvider(targetConn.id); const apiRefsMatch = await this.refsMatchViaApi( sourceProvider, targetProvider, project.fullPath, targetFullPath, ); if (apiRefsMatch === true) { logger.syncLog('info', `Refs match via API for ${project.fullPath}, skipping git`, 'api'); await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetFullPath); return; } // Clone or fetch from source try { const exists = await this.dirExists(mirrorDir); if (!exists) { await Deno.mkdir(mirrorDir, { recursive: true }); await this.runGit(['clone', '--bare', sourceUrl, '.'], mirrorDir); } // Ensure fetch refspec is configured (bare clones don't set one by default, // which prevents tracking branch renames like master -> main) await this.runGit( ['config', 'remote.origin.fetch', '+refs/heads/*:refs/heads/*'], mirrorDir, ); // Update source remote URL in case connection changed try { await this.runGit(['remote', 'set-url', 'origin', sourceUrl], mirrorDir); } catch { // Ignore errors } // Fetch latest refs from source (--prune removes branches deleted on remote) await this.runGit(['fetch', '--prune', 'origin'], mirrorDir); } catch (err: any) { const msg = err instanceof Error ? err.message : String(err); if (msg.includes("couldn't find remote ref HEAD")) { logger.syncLog('warn', `Skipping empty repo ${project.fullPath} (no HEAD ref)`, 'git'); return; } throw err; } // Set up target remote and push const remotes = await this.runGit(['remote'], mirrorDir); if (!remotes.includes('target')) { await this.runGit(['remote', 'add', 'target', targetUrl], mirrorDir); } else { await this.runGit(['remote', 'set-url', 'target', targetUrl], mirrorDir); } // Check for unrelated history before mirror-pushing const isUnrelated = await this.checkUnrelatedHistory(mirrorDir); if (isUnrelated) { logger.syncLog('warn', `Target "${targetFullPath}" has unrelated history — moving to obsolete`, 'git'); await this.moveToObsolete(targetConn, targetFullPath, config.targetGroupOffset); // Re-create fresh target await this.ensureTargetExists(targetConn, targetFullPath, project, sourceConn, sourceConn.groupFilter, config.targetGroupOffset); this.actionLog.append({ actionType: 'obsolete', entityType: 'sync', entityId: config.id, entityName: config.name, details: `Moved unrelated repo "${targetFullPath}" to obsolete`, username: 'system', }); } // Compare refs to determine if push is needed const refsAlreadyMatch = !isUnrelated && await this.refsMatch(mirrorDir); if (refsAlreadyMatch) { logger.syncLog('info', `Refs already match for ${project.fullPath}, skipping push`, 'api'); } else { // Phase 1: push all refs without pruning (ensures target has all source branches) await this.runGit([ 'push', 'target', '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*', ], mirrorDir); // Phase 2: sync default_branch now that all branches exist on target await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath, targetFullPath); // Phase 2b: unprotect stale branches on target so --prune can delete them await this.unprotectStaleBranches(targetConn, targetFullPath, mirrorDir); // Phase 3: push with --prune to remove stale branches (safe now that default_branch is correct) await this.runGit([ 'push', 'target', '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*', '--prune', ], mirrorDir); } // Sync project metadata (description, visibility, topics, default_branch, avatar) await this.syncProjectMetadata(config, sourceConn, targetConn, project.fullPath, targetFullPath); } // ============================================================================ // Target Hierarchy Creation // ============================================================================ private async ensureTargetExists( targetConn: interfaces.data.IProviderConnection, targetFullPath: string, sourceProject: interfaces.data.IProject, sourceConn?: interfaces.data.IProviderConnection, sourceGroupFilter?: string, targetGroupOffset?: string, ): Promise { const segments = targetFullPath.split('/'); const projectName = segments.pop()!; const groupSegments = segments; if (targetConn.providerType === 'gitlab') { await this.ensureGitLabTarget(targetConn, groupSegments, projectName, sourceProject, sourceConn, sourceGroupFilter, targetGroupOffset); } else { await this.ensureGiteaTarget(targetConn, groupSegments, projectName, sourceProject, sourceConn, sourceGroupFilter, targetGroupOffset); } } private async ensureGitLabTarget( conn: interfaces.data.IProviderConnection, groupSegments: string[], projectName: string, sourceProject: interfaces.data.IProject, sourceConn?: interfaces.data.IProviderConnection, sourceGroupFilter?: string, targetGroupOffset?: string, ): Promise { const client = new plugins.gitlabClient.GitLabClient(conn.baseUrl, conn.token); // Walk group hierarchy top-down, creating each if needed let parentId: number | undefined = undefined; let currentPath = ''; for (const segment of groupSegments) { currentPath = currentPath ? `${currentPath}/${segment}` : segment; try { const group = await client.getGroupByPath(currentPath); parentId = group.id; } catch { // Group doesn't exist — create it try { const newGroup = await client.createGroup(segment, segment, parentId); parentId = newGroup.id; logger.info(`Created GitLab group: ${currentPath}`); } catch (createErr: any) { // 409 = already exists (race condition), try fetching again if (String(createErr).includes('409') || String(createErr).includes('already')) { const group = await client.getGroupByPath(currentPath); parentId = group.id; } else { throw createErr; } } } // Sync group metadata from source (once per group per sync cycle) if (sourceConn && !this.syncedGroupMeta.has(currentPath)) { const sourceGroupPath = this.reverseTargetGroupPath(currentPath, sourceGroupFilter, targetGroupOffset); if (sourceGroupPath) { this.syncedGroupMeta.add(currentPath); await this.syncGroupMetadata(sourceConn, conn, sourceGroupPath, currentPath); } } } // Create the project if it doesn't exist const projectPath = groupSegments.length > 0 ? `${groupSegments.join('/')}/${projectName}` : projectName; try { // Check if project exists by path await client.getGroupByPath(projectPath); // If this succeeds, it's actually a group, not a project... unlikely but handle } catch { // Project doesn't exist as a group path; try creating it try { await client.createProject(projectName, { path: projectName, namespaceId: parentId, description: sourceProject.description, visibility: sourceProject.visibility || 'private', }); logger.info(`Created GitLab project: ${projectPath}`); } catch (createErr: any) { // Already exists is fine if (!String(createErr).includes('409') && !String(createErr).includes('already been taken')) { throw createErr; } } } } private async ensureGiteaTarget( conn: interfaces.data.IProviderConnection, groupSegments: string[], projectName: string, sourceProject: interfaces.data.IProject, sourceConn?: interfaces.data.IProviderConnection, sourceGroupFilter?: string, targetGroupOffset?: string, ): Promise { const client = new plugins.giteaClient.GiteaClient(conn.baseUrl, conn.token); // Gitea has flat orgs (no nesting). Use the first segment as org name. // If there are nested segments, join them into the repo name. const orgName = groupSegments[0] || conn.groupFilter || 'default'; const repoName = groupSegments.length > 1 ? [...groupSegments.slice(1), projectName].join('-') : projectName; // Ensure org exists try { await client.getOrg(orgName); } catch { try { await client.createOrg(orgName, { visibility: 'public' }); logger.info(`Created Gitea org: ${orgName}`); } catch (createErr: any) { if (!String(createErr).includes('409') && !String(createErr).includes('already')) { throw createErr; } } } // Sync org metadata from source if we have the source connection if (sourceConn && groupSegments[0] && !this.syncedGroupMeta.has(groupSegments[0])) { const sourceGroupPath = this.reverseTargetGroupPath(groupSegments[0], sourceGroupFilter, targetGroupOffset); if (sourceGroupPath) { this.syncedGroupMeta.add(groupSegments[0]); await this.syncGroupMetadata(sourceConn, conn, sourceGroupPath, groupSegments[0]); } } // Ensure repo exists in org try { await client.createOrgRepo(orgName, repoName, { description: sourceProject.description, private: sourceProject.visibility !== 'public', }); logger.info(`Created Gitea repo: ${orgName}/${repoName}`); } catch (createErr: any) { // Already exists is fine if (!String(createErr).includes('409') && !String(createErr).includes('already')) { throw createErr; } } } // ============================================================================ // Enforce Delete — remove stale target repos // ============================================================================ private async enforceDeleteStaleRepos( config: interfaces.data.ISyncConfig, sourceProjects: interfaces.data.IProject[], sourceConn: interfaces.data.IProviderConnection, targetConn: interfaces.data.IProviderConnection, ): Promise { // Build set of expected target fullPaths from source const expectedTargetPaths = new Set(); for (const project of sourceProjects) { const targetFullPath = this.computeTargetFullPath( project.fullPath, sourceConn.groupFilter, config.targetGroupOffset, ); expectedTargetPaths.add(targetFullPath.toLowerCase()); } // Scope prefix — only delete repos under this prefix const scopePrefix = config.targetGroupOffset; // List all target projects (filtered by connection's groupFilter) const targetProvider = this.connectionManager.getProvider(config.targetConnectionId); const targetProjects = await targetProvider.getProjects(); // Delete target projects not in the expected set, scoped to the prefix for (const targetProject of targetProjects) { if (this.isObsoletePath(targetProject.fullPath)) continue; // Skip repos outside our managed prefix if (scopePrefix && !targetProject.fullPath.toLowerCase().startsWith(scopePrefix.toLowerCase() + '/')) { continue; } if (!expectedTargetPaths.has(targetProject.fullPath.toLowerCase())) { try { await this.moveToObsolete(targetConn, targetProject.fullPath, config.targetGroupOffset); logger.syncLog('warn', `Moved stale target repo "${targetProject.fullPath}" to obsolete`, 'sync'); this.actionLog.append({ actionType: 'obsolete', entityType: 'sync', entityId: config.id, entityName: config.name, details: `Moved stale target repo "${targetProject.fullPath}" to obsolete`, username: 'system', }); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('error', `Enforce-delete failed for "${targetProject.fullPath}": ${errMsg}`, 'sync'); } } } } // ============================================================================ // Enforce Delete — remove stale target groups/orgs // ============================================================================ private async enforceDeleteStaleGroups( config: interfaces.data.ISyncConfig, sourceConn: interfaces.data.IProviderConnection, targetConn: interfaces.data.IProviderConnection, ): Promise { const sourceProvider = this.connectionManager.getProvider(config.sourceConnectionId); const targetProvider = this.connectionManager.getProvider(config.targetConnectionId); // Build expected target group paths from source groups const sourceGroups = await sourceProvider.getGroups(); const expectedTargetGroups = new Set(); for (const sg of sourceGroups) { const targetPath = this.computeTargetFullPath( sg.fullPath, sourceConn.groupFilter, config.targetGroupOffset, ); expectedTargetGroups.add(targetPath.toLowerCase()); } // Always keep the offset itself and the obsolete group if (config.targetGroupOffset) { expectedTargetGroups.add(config.targetGroupOffset.toLowerCase()); expectedTargetGroups.add(`${config.targetGroupOffset}/obsolete`.toLowerCase()); } // Find stale groups const targetGroups = await targetProvider.getGroups(); const scopePrefix = config.targetGroupOffset; const staleGroups: interfaces.data.IGroup[] = []; for (const tg of targetGroups) { if (this.isObsoletePath(tg.fullPath)) continue; if (scopePrefix && !tg.fullPath.toLowerCase().startsWith(scopePrefix.toLowerCase() + '/')) { continue; } if (!expectedTargetGroups.has(tg.fullPath.toLowerCase())) { staleGroups.push(tg); } } if (staleGroups.length === 0) return; // Sort by path depth (shallowest first) to move top-level groups first staleGroups.sort((a, b) => { const depthA = a.fullPath.split('/').length; const depthB = b.fullPath.split('/').length; return depthA - depthB; }); // Track moved prefixes to skip children already moved with their parent const movedPrefixes: string[] = []; for (const group of staleGroups) { // Skip if a parent was already moved (GitLab moves children along) const isChildOfMoved = movedPrefixes.some( (prefix) => group.fullPath.toLowerCase().startsWith(prefix.toLowerCase() + '/'), ); if (isChildOfMoved && targetConn.providerType === 'gitlab') continue; try { if (targetConn.providerType === 'gitlab') { await this.moveGroupToObsolete(targetConn, group.fullPath, config.targetGroupOffset); } else { await this.moveGiteaOrgToObsolete(targetConn, group.fullPath, config.targetGroupOffset); } movedPrefixes.push(group.fullPath); logger.syncLog('warn', `Moved stale group "${group.fullPath}" to obsolete`, 'sync'); this.actionLog.append({ actionType: 'obsolete', entityType: 'sync', entityId: config.id, entityName: config.name, details: `Moved stale target group "${group.fullPath}" to obsolete`, username: 'system', }); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('error', `Enforce-group-delete failed for "${group.fullPath}": ${errMsg}`, 'sync'); } } } /** * Move a GitLab group to the obsolete group with a unique suffix. * Transfers the entire group (including subgroups and projects). */ private async moveGroupToObsolete( targetConn: interfaces.data.IProviderConnection, groupFullPath: string, basePath?: string, ): Promise { const suffix = this.generateSuffix(); const obsoleteTarget = await this.ensureObsoleteGroup(targetConn, basePath); if (obsoleteTarget.type !== 'gitlab') return; // Get group by path const group = await this.rawApiCall( targetConn, 'GET', `/api/v4/groups/${encodeURIComponent(groupFullPath)}`, ); // Transfer group to be a child of the obsolete group await this.rawApiCall( targetConn, 'POST', `/api/v4/groups/${group.id}/transfer`, { group_id: obsoleteTarget.groupId }, ); // Rename with suffix + set private const originalPath = groupFullPath.split('/').pop()!; await this.rawApiCall( targetConn, 'PUT', `/api/v4/groups/${group.id}`, { name: `${originalPath}-${suffix}`, path: `${originalPath}-${suffix}`, visibility: 'private' }, ); logger.info(`Moved GitLab group "${groupFullPath}" to obsolete as "${originalPath}-${suffix}"`); } /** * Move all repos in a Gitea org to obsolete, then delete the empty org. * Gitea orgs can't be transferred, so we move repos individually. */ private async moveGiteaOrgToObsolete( targetConn: interfaces.data.IProviderConnection, orgName: string, basePath?: string, ): Promise { // List all repos in the stale org (auto-paginate) const allRepos: any[] = []; let page = 1; const perPage = 50; while (true) { const repos = await this.rawApiCall( targetConn, 'GET', `/api/v1/orgs/${encodeURIComponent(orgName)}/repos?page=${page}&limit=${perPage}`, ); const repoList = Array.isArray(repos) ? repos : []; allRepos.push(...repoList); if (repoList.length < perPage) break; page++; } // Move each repo to obsolete for (const repo of allRepos) { try { await this.moveToObsolete(targetConn, `${orgName}/${repo.name}`, basePath); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.error(`Failed to move repo "${orgName}/${repo.name}" to obsolete: ${errMsg}`); } } // Delete the now-empty org try { await this.rawApiCall( targetConn, 'DELETE', `/api/v1/orgs/${encodeURIComponent(orgName)}`, ); logger.info(`Deleted empty Gitea org: ${orgName}`); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.error(`Failed to delete empty Gitea org "${orgName}": ${errMsg}`); } } // ============================================================================ // Helpers // ============================================================================ /** * Returns true if the given full path belongs to an obsolete namespace. * Matches path segments named "obsolete" or ending with "-obsolete". */ private isObsoletePath(fullPath: string): boolean { const segments = fullPath.toLowerCase().split('/'); return segments.some(s => s === 'obsolete' || s.endsWith('-obsolete')); } private buildAuthUrl(conn: interfaces.data.IProviderConnection, repoPath: string): string { const url = new URL(conn.baseUrl); if (conn.providerType === 'gitlab') { return `${url.protocol}//oauth2:${conn.token}@${url.host}/${repoPath}.git`; } else { return `${url.protocol}//gitea-token:${conn.token}@${url.host}/${repoPath}.git`; } } private computeRelativePath(fullPath: string, groupFilter?: string): string { if (!groupFilter) return fullPath; if (fullPath.startsWith(groupFilter + '/')) { return fullPath.substring(groupFilter.length + 1); } return fullPath; } /** * Reverse-map a target group path to the corresponding source group path. * Returns null if the target path is part of the offset itself (not a source-derived group). */ private reverseTargetGroupPath( targetGroupPath: string, sourceGroupFilter?: string, targetGroupOffset?: string, ): string | null { let relativePath = targetGroupPath; // Strip the target offset prefix if (targetGroupOffset) { if (targetGroupPath === targetGroupOffset) { // This IS the offset group itself, not a source-derived group return null; } if (targetGroupPath.startsWith(targetGroupOffset + '/')) { relativePath = targetGroupPath.substring(targetGroupOffset.length + 1); } else { // Target path is not under the offset — can't reverse-map return null; } } // Re-add the source group filter prefix if (sourceGroupFilter) { return `${sourceGroupFilter}/${relativePath}`; } return relativePath; } private computeTargetFullPath( sourceFullPath: string, sourceGroupFilter?: string, targetGroupOffset?: string, ): string { const relativePath = this.computeRelativePath(sourceFullPath, sourceGroupFilter); return targetGroupOffset ? `${targetGroupOffset}/${relativePath}` : relativePath; } /** * Validates that the sync config's targetGroupOffset is reachable * by the target connection's groupFilter. Throws when enforceDelete * is on and the offset is outside the filter scope. */ private validateSyncConfig(config: { targetConnectionId: string; targetGroupOffset?: string; enforceDelete: boolean; }): void { if (!config.targetGroupOffset) return; const targetConn = this.connectionManager.getConnection(config.targetConnectionId); if (!targetConn?.groupFilter) return; const offset = config.targetGroupOffset.toLowerCase(); const filter = targetConn.groupFilter.toLowerCase(); const inScope = offset === filter || offset.startsWith(filter + '/'); if (!inScope && config.enforceDelete) { throw new Error( `Target group offset "${config.targetGroupOffset}" is outside target connection's ` + `group filter "${targetConn.groupFilter}". With enforce-delete enabled, the sync ` + `engine cannot list repos under the offset. Either change the offset to be within ` + `"${targetConn.groupFilter}/...", remove the target's group filter, or disable enforce-delete.` ); } if (!inScope) { logger.warn( `Target group offset "${config.targetGroupOffset}" is outside target connection's ` + `group filter "${targetConn.groupFilter}". Git pushes will work but repos won't ` + `appear in the target connection's listings.` ); } } // ============================================================================ // Metadata Sync // ============================================================================ /** * Download binary data (e.g. avatar image) with provider auth headers. * Returns null on 404 or error. */ private async rawBinaryFetch( conn: interfaces.data.IProviderConnection, url: string, ): Promise { try { const headers: Record = {}; if (conn.providerType === 'gitlab') { headers['PRIVATE-TOKEN'] = conn.token; } else { headers['Authorization'] = `token ${conn.token}`; } const resp = await fetch(url, { headers }); if (!resp.ok) { await resp.body?.cancel(); return null; } return new Uint8Array(await resp.arrayBuffer()); } catch { return null; } } /** * Raw multipart call for avatar uploads (GitLab requires multipart FormData). */ private async rawMultipartCall( conn: interfaces.data.IProviderConnection, method: string, apiPath: string, formData: FormData, ): Promise { const baseUrl = conn.baseUrl.replace(/\/+$/, ''); const url = `${baseUrl}${apiPath}`; const headers: Record = {}; if (conn.providerType === 'gitlab') { headers['PRIVATE-TOKEN'] = conn.token; } else { headers['Authorization'] = `token ${conn.token}`; } // Do NOT set Content-Type — let fetch set the multipart boundary const resp = await fetch(url, { method, headers, body: formData }); if (!resp.ok) { const text = await resp.text(); throw new Error(`${method} ${apiPath}: ${resp.status} - ${text}`); } try { return await resp.json(); } catch { return undefined; } } /** * Normalize visibility values between GitLab and Gitea. * GitLab: "public" | "internal" | "private" * Gitea: "public" | "limited" | "private" */ private normalizeVisibility(visibility: string): string { const v = visibility?.toLowerCase() || 'private'; if (v === 'limited') return 'internal'; return v; } /** * Guess MIME type from binary content or URL for avatar uploads. */ private guessAvatarMimeType(data: Uint8Array, url: string): string { // Check magic bytes if (data[0] === 0x89 && data[1] === 0x50) return 'image/png'; if (data[0] === 0xFF && data[1] === 0xD8) return 'image/jpeg'; if (data[0] === 0x47 && data[1] === 0x49) return 'image/gif'; // SVG: text-based XML, no magic bytes — check content const textStart = new TextDecoder().decode(data.slice(0, 200)); if (textStart.includes(' { try { const sourceProject = await this.fetchProjectRaw(sourceConn, sourceFullPath); if (!sourceProject) return; const targetProject = await this.fetchProjectRaw(targetConn, targetFullPath); if (!targetProject) return; const sourceBranch = sourceProject.default_branch || 'main'; const targetBranch = targetProject.default_branch || 'main'; if (sourceBranch !== targetBranch) { logger.syncLog('info', `Updating default branch for ${targetFullPath}: ${targetBranch} -> ${sourceBranch}`, 'api'); if (targetConn.providerType === 'gitlab') { await this.rawApiCall(targetConn, 'PUT', `/api/v4/projects/${targetProject.id}`, { default_branch: sourceBranch, }); } else { const segments = targetFullPath.split('/'); const repo = segments.pop()!; const owner = segments[0] || ''; await this.rawApiCall(targetConn, 'PATCH', `/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, { default_branch: sourceBranch, }); } } } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('warn', `Pre-push default_branch sync failed for ${targetFullPath}: ${errMsg}`, 'api'); } } /** * Unprotect branches on the target that no longer exist in the source, * so that git push --prune can delete them. */ private async unprotectStaleBranches( targetConn: interfaces.data.IProviderConnection, targetFullPath: string, mirrorDir: string, ): Promise { if (targetConn.providerType !== 'gitlab') return; try { const targetProject = await this.fetchProjectRaw(targetConn, targetFullPath); if (!targetProject) return; const client = new plugins.gitlabClient.GitLabClient(targetConn.baseUrl, targetConn.token); const protectedBranches = await client.getProtectedBranches(targetProject.id); if (protectedBranches.length === 0) return; // Get list of branches in the local mirror (= source branches) const localBranchOutput = await this.runGit(['branch', '--list'], mirrorDir); const localBranches = new Set( localBranchOutput.split('\n').map(b => b.trim().replace(/^\* /, '')).filter(Boolean), ); for (const pb of protectedBranches) { if (!localBranches.has(pb.name)) { logger.syncLog('info', `Unprotecting stale branch "${pb.name}" on ${targetFullPath}`, 'api'); await client.unprotectBranch(targetProject.id, pb.name); } } } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('warn', `Failed to unprotect stale branches for ${targetFullPath}: ${errMsg}`, 'api'); } } /** * Sync project metadata (description, visibility, topics, default_branch, avatar) * from source to target after the git push. */ private async syncProjectMetadata( config: interfaces.data.ISyncConfig, sourceConn: interfaces.data.IProviderConnection, targetConn: interfaces.data.IProviderConnection, sourceFullPath: string, targetFullPath: string, ): Promise { try { // Fetch source project raw JSON const sourceProject = await this.fetchProjectRaw(sourceConn, sourceFullPath); if (!sourceProject) return; // Fetch target project raw JSON const targetProject = await this.fetchProjectRaw(targetConn, targetFullPath); if (!targetProject) return; // Extract normalized metadata from both const sourceMeta = this.extractProjectMeta(sourceConn, sourceProject); const targetMeta = this.extractProjectMeta(targetConn, targetProject); // Append mirror hint to description if enabled if (config.addMirrorHint) { const mirrorUrl = `${sourceConn.baseUrl.replace(/\/+$/, '')}/${sourceFullPath}`; sourceMeta.description = `${sourceMeta.description}\n\n(This is a mirror of ${mirrorUrl})`.trim(); } // Diff and update text metadata const changes: string[] = []; if (sourceMeta.description !== targetMeta.description) changes.push('description'); if (this.normalizeVisibility(sourceMeta.visibility) !== this.normalizeVisibility(targetMeta.visibility)) changes.push('visibility'); if (JSON.stringify([...sourceMeta.topics].sort()) !== JSON.stringify([...targetMeta.topics].sort())) changes.push('topics'); if (sourceMeta.defaultBranch !== targetMeta.defaultBranch) changes.push('default_branch'); if (changes.length > 0) { logger.syncLog('info', `Syncing metadata for ${targetFullPath}: ${changes.join(', ')}`, 'api'); await this.updateProjectMeta(targetConn, targetFullPath, targetProject, sourceMeta); logger.syncLog('success', `Updated metadata for ${targetFullPath}: ${changes.join(', ')}`, 'api'); } // Sync avatar if (sourceMeta.avatarUrl) { await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, sourceMeta.avatarUrl, targetProject); } else if (config.useGroupAvatarsForProjects) { // Project has no avatar — inherit from parent group const groupPath = sourceFullPath.substring(0, sourceFullPath.lastIndexOf('/')); let groupAvatarApplied = false; if (groupPath) { try { const sourceGroup = await this.fetchGroupRaw(sourceConn, groupPath); if (sourceGroup) { const groupMeta = this.extractGroupMeta(sourceConn, sourceGroup); if (groupMeta.avatarUrl) { await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, groupMeta.avatarUrl, targetProject); groupAvatarApplied = true; } } } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('warn', `Group avatar sync failed for ${targetFullPath}: ${errMsg}`, 'api'); } } // If group also has no avatar, remove target avatar if (!groupAvatarApplied && targetMeta.avatarUrl) { await this.removeProjectAvatar(targetConn, targetFullPath, targetProject); } } else { // No source avatar, no group fallback — remove target avatar if present if (targetMeta.avatarUrl) { await this.removeProjectAvatar(targetConn, targetFullPath, targetProject); } } } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('error', `Metadata sync failed for ${targetFullPath}: ${errMsg}`, 'api'); } } /** * Sync group/org metadata (description, visibility, avatar) from source to target. */ private async syncGroupMetadata( sourceConn: interfaces.data.IProviderConnection, targetConn: interfaces.data.IProviderConnection, sourceGroupPath: string, targetGroupPath: string, ): Promise { try { const sourceGroup = await this.fetchGroupRaw(sourceConn, sourceGroupPath); if (!sourceGroup) return; const targetGroup = await this.fetchGroupRaw(targetConn, targetGroupPath); if (!targetGroup) return; const sourceMeta = this.extractGroupMeta(sourceConn, sourceGroup); const targetMeta = this.extractGroupMeta(targetConn, targetGroup); // Append mirror hint to description if enabled if (this.currentSyncConfig?.addMirrorHint) { const mirrorUrl = `${sourceConn.baseUrl.replace(/\/+$/, '')}/${sourceGroupPath}`; sourceMeta.description = `${sourceMeta.description}\n\n(This is a mirror of ${mirrorUrl})`.trim(); } const changes: string[] = []; if (sourceMeta.description !== targetMeta.description) changes.push('description'); if (this.normalizeVisibility(sourceMeta.visibility) !== this.normalizeVisibility(targetMeta.visibility)) changes.push('visibility'); if (changes.length > 0) { logger.syncLog('info', `Syncing group metadata for ${targetGroupPath}: ${changes.join(', ')}`, 'api'); await this.updateGroupMeta(targetConn, targetGroupPath, targetGroup, sourceMeta); logger.syncLog('success', `Updated group metadata for ${targetGroupPath}: ${changes.join(', ')}`, 'api'); } // Sync avatar if (sourceMeta.avatarUrl) { await this.syncGroupAvatar(sourceConn, targetConn, sourceGroupPath, targetGroupPath, sourceMeta.avatarUrl, targetGroup); } else if (targetMeta.avatarUrl) { await this.removeGroupAvatar(targetConn, targetGroupPath, targetGroup); } } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('error', `Group metadata sync failed for ${targetGroupPath}: ${errMsg}`, 'api'); } } // ---- Raw metadata fetchers ---- private async fetchProjectRaw(conn: interfaces.data.IProviderConnection, fullPath: string): Promise { if (conn.providerType === 'gitlab') { return await this.rawApiCall(conn, 'GET', `/api/v4/projects/${encodeURIComponent(fullPath)}`); } else { const segments = fullPath.split('/'); const repo = segments.pop()!; const owner = segments[0] || ''; return await this.rawApiCall(conn, 'GET', `/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`); } } private async fetchGroupRaw(conn: interfaces.data.IProviderConnection, groupPath: string): Promise { if (conn.providerType === 'gitlab') { return await this.rawApiCall(conn, 'GET', `/api/v4/groups/${encodeURIComponent(groupPath)}`); } else { // Gitea orgs are flat — the group path IS the org name const orgName = groupPath.split('/')[0] || groupPath; return await this.rawApiCall(conn, 'GET', `/api/v1/orgs/${encodeURIComponent(orgName)}`); } } // ---- Metadata extractors ---- private extractProjectMeta(conn: interfaces.data.IProviderConnection, raw: any): { description: string; visibility: string; topics: string[]; defaultBranch: string; avatarUrl: string; } { if (conn.providerType === 'gitlab') { return { description: raw.description || '', visibility: raw.visibility || 'private', topics: raw.topics || [], defaultBranch: raw.default_branch || 'main', avatarUrl: raw.avatar_url || '', }; } else { return { description: raw.description || '', visibility: raw.private ? 'private' : 'public', topics: raw.topics || [], defaultBranch: raw.default_branch || 'main', avatarUrl: raw.avatar_url || '', }; } } private extractGroupMeta(conn: interfaces.data.IProviderConnection, raw: any): { description: string; visibility: string; avatarUrl: string; } { if (conn.providerType === 'gitlab') { return { description: raw.description || '', visibility: raw.visibility || 'private', avatarUrl: raw.avatar_url || '', }; } else { return { description: raw.description || '', visibility: raw.visibility || 'public', avatarUrl: raw.avatar_url || '', }; } } // ---- Metadata updaters ---- private async updateProjectMeta( conn: interfaces.data.IProviderConnection, fullPath: string, rawProject: any, meta: { description: string; visibility: string; topics: string[]; defaultBranch: string }, ): Promise { if (conn.providerType === 'gitlab') { // Update description, visibility, topics (always safe) await this.rawApiCall(conn, 'PUT', `/api/v4/projects/${rawProject.id}`, { description: meta.description, visibility: this.normalizeVisibility(meta.visibility), topics: meta.topics, }); // Update default_branch separately — may fail if the branch doesn't exist in git try { await this.rawApiCall(conn, 'PUT', `/api/v4/projects/${rawProject.id}`, { default_branch: meta.defaultBranch, }); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('warn', `Could not set default_branch to "${meta.defaultBranch}" for ${fullPath}: ${errMsg}`, 'api'); } } else { const segments = fullPath.split('/'); const repo = segments.pop()!; const owner = segments[0] || ''; const encodedOwner = encodeURIComponent(owner); const encodedRepo = encodeURIComponent(repo); // Update description, visibility await this.rawApiCall(conn, 'PATCH', `/api/v1/repos/${encodedOwner}/${encodedRepo}`, { description: meta.description, private: this.normalizeVisibility(meta.visibility) === 'private', }); // Update default_branch separately — may fail if the branch doesn't exist in git try { await this.rawApiCall(conn, 'PATCH', `/api/v1/repos/${encodedOwner}/${encodedRepo}`, { default_branch: meta.defaultBranch, }); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); logger.syncLog('warn', `Could not set default_branch to "${meta.defaultBranch}" for ${fullPath}: ${errMsg}`, 'api'); } // Topics are a separate endpoint in Gitea await this.rawApiCall(conn, 'PUT', `/api/v1/repos/${encodedOwner}/${encodedRepo}/topics`, { topics: meta.topics, }); } } private async updateGroupMeta( conn: interfaces.data.IProviderConnection, groupPath: string, rawGroup: any, meta: { description: string; visibility: string }, ): Promise { if (conn.providerType === 'gitlab') { try { await this.rawApiCall(conn, 'PUT', `/api/v4/groups/${rawGroup.id}`, { description: meta.description, visibility: this.normalizeVisibility(meta.visibility), }); } catch (err) { const errMsg = err instanceof Error ? err.message : String(err); if (errMsg.includes('visibility_level') || errMsg.includes('visibility')) { logger.syncLog('warn', `Cannot sync visibility for group ${groupPath} (contains projects with higher visibility), syncing description only`, 'api'); await this.rawApiCall(conn, 'PUT', `/api/v4/groups/${rawGroup.id}`, { description: meta.description, }); } else { throw err; } } } else { const orgName = groupPath.split('/')[0] || groupPath; await this.rawApiCall(conn, 'PATCH', `/api/v1/orgs/${encodeURIComponent(orgName)}`, { description: meta.description, visibility: this.normalizeVisibility(meta.visibility) === 'private' ? 'private' : 'public', }); } } // ---- Avatar sync ---- private async syncProjectAvatar( sourceConn: interfaces.data.IProviderConnection, targetConn: interfaces.data.IProviderConnection, _sourceFullPath: string, targetFullPath: string, sourceAvatarUrl: string, targetRawProject: any, ): Promise { // Resolve relative avatar URLs const resolvedSourceUrl = sourceAvatarUrl.startsWith('http') ? sourceAvatarUrl : `${sourceConn.baseUrl.replace(/\/+$/, '')}${sourceAvatarUrl}`; const sourceAvatarData = await this.rawBinaryFetch(sourceConn, resolvedSourceUrl); if (!sourceAvatarData || sourceAvatarData.length === 0) return; // Skip SVG avatars — not supported by GitLab project endpoints const mimeType = this.guessAvatarMimeType(sourceAvatarData, resolvedSourceUrl); if (mimeType === 'image/svg+xml') { logger.syncLog('warn', `Skipping SVG avatar for ${targetFullPath} (not supported by target)`, 'api'); return; } // Check in-memory cache: skip if source hasn't changed since last upload const sourceHash = await this.hashBytes(sourceAvatarData); const cacheKey = `project:${targetFullPath}`; if (this.avatarUploadCache.get(cacheKey) === sourceHash) { return; // Source avatar unchanged since last upload } // Compare with target's current avatar to avoid unnecessary uploads const targetMeta = this.extractProjectMeta(targetConn, targetRawProject); if (targetMeta.avatarUrl) { try { const resolvedTargetUrl = targetMeta.avatarUrl.startsWith('http') ? targetMeta.avatarUrl : `${targetConn.baseUrl.replace(/\/+$/, '')}${targetMeta.avatarUrl}`; const targetAvatarData = await this.rawBinaryFetch(targetConn, resolvedTargetUrl); if (targetAvatarData && this.binaryEqual(sourceAvatarData, targetAvatarData)) { this.avatarUploadCache.set(cacheKey, sourceHash); return; // Avatars are identical — skip upload } } catch { // Failed to fetch target avatar — proceed with upload as safe fallback } } logger.syncLog('info', `Syncing avatar for ${targetFullPath}...`, 'api'); if (targetConn.providerType === 'gitlab') { // GitLab: multipart upload const blob = new Blob([sourceAvatarData.buffer as ArrayBuffer], { type: mimeType }); const ext = mimeType.split('/')[1] || 'png'; const formData = new FormData(); formData.append('avatar', blob, `avatar.${ext}`); await this.rawMultipartCall( targetConn, 'PUT', `/api/v4/projects/${targetRawProject.id}`, formData, ); } else { // Gitea: base64 JSON upload const segments = targetFullPath.split('/'); const repo = segments.pop()!; const owner = segments[0] || ''; const base64Image = this.uint8ArrayToBase64(sourceAvatarData); await this.rawApiCall( targetConn, 'POST', `/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/avatar`, { image: base64Image }, ); } this.avatarUploadCache.set(cacheKey, sourceHash); } private async removeProjectAvatar( targetConn: interfaces.data.IProviderConnection, targetFullPath: string, targetRawProject: any, ): Promise { logger.syncLog('info', `Removing avatar from ${targetFullPath}`, 'api'); if (targetConn.providerType === 'gitlab') { await this.rawApiCall(targetConn, 'PUT', `/api/v4/projects/${targetRawProject.id}`, { avatar: '', }); } else { const segments = targetFullPath.split('/'); const repo = segments.pop()!; const owner = segments[0] || ''; await this.rawApiCall(targetConn, 'DELETE', `/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/avatar`); } } private async syncGroupAvatar( sourceConn: interfaces.data.IProviderConnection, targetConn: interfaces.data.IProviderConnection, _sourceGroupPath: string, targetGroupPath: string, sourceAvatarUrl: string, targetRawGroup: any, ): Promise { const resolvedSourceUrl = sourceAvatarUrl.startsWith('http') ? sourceAvatarUrl : `${sourceConn.baseUrl.replace(/\/+$/, '')}${sourceAvatarUrl}`; const sourceAvatarData = await this.rawBinaryFetch(sourceConn, resolvedSourceUrl); if (!sourceAvatarData || sourceAvatarData.length === 0) return; // Skip SVG avatars — not supported by GitLab project endpoints const mimeType = this.guessAvatarMimeType(sourceAvatarData, resolvedSourceUrl); if (mimeType === 'image/svg+xml') { logger.syncLog('warn', `Skipping SVG avatar for group ${targetGroupPath} (not supported by target)`, 'api'); return; } // Check in-memory cache: skip if source hasn't changed since last upload const sourceHash = await this.hashBytes(sourceAvatarData); const cacheKey = `group:${targetGroupPath}`; if (this.avatarUploadCache.get(cacheKey) === sourceHash) { return; // Source avatar unchanged since last upload } // Compare with target's current avatar to avoid unnecessary uploads const targetMeta = this.extractGroupMeta(targetConn, targetRawGroup); if (targetMeta.avatarUrl) { try { const resolvedTargetUrl = targetMeta.avatarUrl.startsWith('http') ? targetMeta.avatarUrl : `${targetConn.baseUrl.replace(/\/+$/, '')}${targetMeta.avatarUrl}`; const targetAvatarData = await this.rawBinaryFetch(targetConn, resolvedTargetUrl); if (targetAvatarData && this.binaryEqual(sourceAvatarData, targetAvatarData)) { this.avatarUploadCache.set(cacheKey, sourceHash); return; // Avatars are identical — skip upload } } catch { // Failed to fetch target avatar — proceed with upload as safe fallback } } logger.syncLog('info', `Syncing avatar for group ${targetGroupPath}...`, 'api'); if (targetConn.providerType === 'gitlab') { const blob = new Blob([sourceAvatarData.buffer as ArrayBuffer], { type: mimeType }); const ext = mimeType.split('/')[1] || 'png'; const formData = new FormData(); formData.append('avatar', blob, `avatar.${ext}`); await this.rawMultipartCall( targetConn, 'PUT', `/api/v4/groups/${targetRawGroup.id}`, formData, ); } else { const orgName = targetGroupPath.split('/')[0] || targetGroupPath; const base64Image = this.uint8ArrayToBase64(sourceAvatarData); await this.rawApiCall( targetConn, 'POST', `/api/v1/orgs/${encodeURIComponent(orgName)}/avatar`, { image: base64Image }, ); } this.avatarUploadCache.set(cacheKey, sourceHash); } private async removeGroupAvatar( targetConn: interfaces.data.IProviderConnection, targetGroupPath: string, targetRawGroup: any, ): Promise { logger.syncLog('info', `Removing avatar from group ${targetGroupPath}`, 'api'); if (targetConn.providerType === 'gitlab') { await this.rawApiCall(targetConn, 'PUT', `/api/v4/groups/${targetRawGroup.id}`, { avatar: '', }); } else { const orgName = targetGroupPath.split('/')[0] || targetGroupPath; await this.rawApiCall(targetConn, 'DELETE', `/api/v1/orgs/${encodeURIComponent(orgName)}/avatar`); } } private binaryEqual(a: Uint8Array, b: Uint8Array): boolean { if (a.length !== b.length) return false; for (let i = 0; i < a.length; i++) { if (a[i] !== b[i]) return false; } return true; } private async hashBytes(data: Uint8Array): Promise { const hashBuffer = await crypto.subtle.digest('SHA-256', data.buffer as ArrayBuffer); return Array.from(new Uint8Array(hashBuffer)).map(b => b.toString(16).padStart(2, '0')).join(''); } private uint8ArrayToBase64(bytes: Uint8Array): string { let binary = ''; for (let i = 0; i < bytes.length; i++) { binary += String.fromCharCode(bytes[i]); } return btoa(binary); } // ============================================================================ // Obsolete — move repos instead of deleting/overwriting // ============================================================================ /** * Raw HTTP call for API endpoints not supported by the client libraries. */ private async rawApiCall( conn: interfaces.data.IProviderConnection, method: string, apiPath: string, body?: any, ): Promise { const baseUrl = conn.baseUrl.replace(/\/+$/, ''); const url = `${baseUrl}${apiPath}`; const headers: Record = { 'Content-Type': 'application/json' }; if (conn.providerType === 'gitlab') { headers['PRIVATE-TOKEN'] = conn.token; } else { headers['Authorization'] = `token ${conn.token}`; } const resp = await fetch(url, { method, headers, body: body ? JSON.stringify(body) : undefined, }); if (!resp.ok) { const text = await resp.text(); throw new Error(`${method} ${apiPath}: ${resp.status} - ${text}`); } try { return await resp.json(); } catch { return undefined; } } private generateSuffix(): string { return crypto.randomUUID().substring(0, 6); } /** * Ensure an "obsolete" group/org exists under the target base path. * Returns the obsolete group/org identifier needed for transfers. */ private async ensureObsoleteGroup( targetConn: interfaces.data.IProviderConnection, basePath?: string, ): Promise<{ type: 'gitlab'; groupId: number } | { type: 'gitea'; orgName: string }> { if (targetConn.providerType === 'gitlab') { const client = new plugins.gitlabClient.GitLabClient(targetConn.baseUrl, targetConn.token); // Walk the basePath to find the parent group, then create "obsolete" subgroup let parentId: number | undefined; if (basePath) { const parentGroup = await client.getGroupByPath(basePath); parentId = parentGroup.id; } // Try to get existing obsolete group const obsoletePath = basePath ? `${basePath}/obsolete` : 'obsolete'; try { const group = await client.getGroupByPath(obsoletePath); return { type: 'gitlab', groupId: group.id }; } catch { // Doesn't exist — create it try { const newGroup = await client.createGroup('obsolete', 'obsolete', parentId); // Set to private via raw API (createGroup defaults to private already) logger.info(`Created GitLab obsolete group: ${obsoletePath}`); return { type: 'gitlab', groupId: newGroup.id }; } catch (createErr: any) { if (String(createErr).includes('409') || String(createErr).includes('already')) { const group = await client.getGroupByPath(obsoletePath); return { type: 'gitlab', groupId: group.id }; } throw createErr; } } } else { // Gitea: flat orgs — create "{org}-obsolete" const client = new plugins.giteaClient.GiteaClient(targetConn.baseUrl, targetConn.token); const segments = basePath ? basePath.split('/') : []; const orgName = segments[0] || targetConn.groupFilter || 'default'; const obsoleteOrg = `${orgName}-obsolete`; try { await client.getOrg(obsoleteOrg); } catch { try { await client.createOrg(obsoleteOrg, { visibility: 'private' }); logger.info(`Created Gitea obsolete org: ${obsoleteOrg}`); } catch (createErr: any) { if (!String(createErr).includes('409') && !String(createErr).includes('already')) { throw createErr; } } } return { type: 'gitea', orgName: obsoleteOrg }; } } /** * Move a target repo to the "obsolete" group with a unique suffix. * The project is also set to private. */ private async moveToObsolete( targetConn: interfaces.data.IProviderConnection, targetFullPath: string, basePath?: string, ): Promise { const suffix = this.generateSuffix(); const obsoleteTarget = await this.ensureObsoleteGroup(targetConn, basePath); if (obsoleteTarget.type === 'gitlab') { // 1. Get project by path const project = await this.rawApiCall( targetConn, 'GET', `/api/v4/projects/${encodeURIComponent(targetFullPath)}`, ); const projectId = project.id; // 2. Transfer to obsolete group await this.rawApiCall( targetConn, 'PUT', `/api/v4/projects/${projectId}/transfer`, { namespace: obsoleteTarget.groupId }, ); // 3. Rename with suffix + set private const originalPath = targetFullPath.split('/').pop()!; await this.rawApiCall( targetConn, 'PUT', `/api/v4/projects/${projectId}`, { name: `${originalPath}-${suffix}`, path: `${originalPath}-${suffix}`, visibility: 'private' }, ); logger.info(`Moved GitLab project "${targetFullPath}" to obsolete as "${originalPath}-${suffix}"`); } else { // Gitea: parse owner/repo const segments = targetFullPath.split('/'); const repo = segments.pop()!; const owner = segments[0] || ''; // 1. Transfer to obsolete org await this.rawApiCall( targetConn, 'POST', `/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/transfer`, { new_owner: obsoleteTarget.orgName }, ); // 2. Rename with suffix + set private await this.rawApiCall( targetConn, 'PATCH', `/api/v1/repos/${encodeURIComponent(obsoleteTarget.orgName)}/${encodeURIComponent(repo)}`, { name: `${repo}-${suffix}`, private: true }, ); logger.info(`Moved Gitea repo "${targetFullPath}" to obsolete as "${repo}-${suffix}"`); } } /** * Check whether the target remote in a bare mirror dir has unrelated history * compared to the source (origin). Returns true if histories are completely disjoint. */ private async checkUnrelatedHistory(mirrorDir: string): Promise { // Fetch target refs into the bare mirror try { await this.runGit(['fetch', 'target'], mirrorDir); } catch { // Target is empty or unreachable — not unrelated return false; } // Get any source ref const sourceRefs = await this.runGit( ['for-each-ref', '--format=%(objectname)', 'refs/heads/'], mirrorDir, ); const sourceRef = sourceRefs.trim().split('\n')[0]; if (!sourceRef) return false; // Source is empty // Get any target ref const targetRefs = await this.runGit( ['for-each-ref', '--format=%(objectname)', 'refs/remotes/target/'], mirrorDir, ); const targetRef = targetRefs.trim().split('\n')[0]; if (!targetRef) return false; // Target is empty // Check for common ancestor try { await this.runGit(['merge-base', sourceRef, targetRef], mirrorDir); return false; // Common ancestor found — related } catch { return true; // No common ancestor — unrelated } } private sanitizePath(fullPath: string): string { return fullPath.replace(/[^a-zA-Z0-9._/-]/g, '_'); } private async dirExists(dirPath: string): Promise { try { const stat = await Deno.stat(dirPath); return stat.isDirectory; } catch { return false; } } /** * Fetch all branch and tag SHAs from a repo via provider API. * Returns null on any error (safe fallback to git-based comparison). */ private async listRefsViaProvider( provider: BaseProvider, fullPath: string, ): Promise<{ branches: Map; tags: Map } | null> { try { const [branches, tags] = await Promise.all([ provider.getBranches(fullPath), provider.getTags(fullPath), ]); return { branches: new Map(branches.map((b) => [b.name, b.commitSha])), tags: new Map(tags.map((t) => [t.name, t.commitSha])), }; } catch { return null; } } /** * Compare refs between source and target via provider API (no git clone needed). * Returns true (match), false (differ), or null (can't determine — fall through to git). */ private async refsMatchViaApi( sourceProvider: BaseProvider, targetProvider: BaseProvider, sourceFullPath: string, targetFullPath: string, ): Promise { const [sourceRefs, targetRefs] = await Promise.all([ this.listRefsViaProvider(sourceProvider, sourceFullPath), this.listRefsViaProvider(targetProvider, targetFullPath), ]); if (!sourceRefs || !targetRefs) return null; // Compare branches if (sourceRefs.branches.size !== targetRefs.branches.size) return false; for (const [name, sha] of sourceRefs.branches) { if (targetRefs.branches.get(name) !== sha) return false; } // Compare tags if (sourceRefs.tags.size !== targetRefs.tags.size) return false; for (const [name, sha] of sourceRefs.tags) { if (targetRefs.tags.get(name) !== sha) return false; } return true; } /** * Compare local refs (source) with target remote refs. * Returns true when all branches and tags are identical — no push needed. */ private async refsMatch(mirrorDir: string): Promise { try { // Local branches (source) const localHeadsRaw = await this.runGit( ['for-each-ref', '--format=%(refname:strip=2) %(objectname)', 'refs/heads/'], mirrorDir, ); // Target branches (fetched by checkUnrelatedHistory) const targetHeadsRaw = await this.runGit( ['for-each-ref', '--format=%(refname:strip=3) %(objectname)', 'refs/remotes/target/'], mirrorDir, ); // Local tags const localTagsRaw = await this.runGit( ['for-each-ref', '--format=%(refname:strip=2) %(objectname)', 'refs/tags/'], mirrorDir, ); // Target tags via ls-remote (avoids shared refs/tags/ namespace ambiguity in bare repos) const targetTagsRaw = await this.runGit(['ls-remote', '--tags', 'target'], mirrorDir); const parseRefLines = (raw: string): Map => { const map = new Map(); for (const line of raw.trim().split('\n')) { if (!line.trim()) continue; const parts = line.trim().split(/\s+/); if (parts.length >= 2) { map.set(parts[0], parts[1]); } } return map; }; const parseLsRemoteTags = (raw: string): Map => { const map = new Map(); for (const line of raw.trim().split('\n')) { if (!line.trim()) continue; // Skip ^{} dereference lines if (line.includes('^{}')) continue; const parts = line.trim().split(/\s+/); if (parts.length >= 2) { // parts[0] = sha, parts[1] = refs/tags/name const tagName = parts[1].replace('refs/tags/', ''); map.set(tagName, parts[0]); } } return map; }; const localHeads = parseRefLines(localHeadsRaw); const targetHeads = parseRefLines(targetHeadsRaw); const localTags = parseRefLines(localTagsRaw); const targetTags = parseLsRemoteTags(targetTagsRaw); // Compare branches if (localHeads.size !== targetHeads.size) return false; for (const [name, sha] of localHeads) { if (targetHeads.get(name) !== sha) return false; } // Compare tags if (localTags.size !== targetTags.size) return false; for (const [name, sha] of localTags) { if (targetTags.get(name) !== sha) return false; } return true; } catch { // On any error, fall back to pushing (safe default) return false; } } private async runGit(args: string[], cwd?: string): Promise { const cmd = new Deno.Command('git', { args, cwd, stdout: 'piped', stderr: 'piped', env: { ...Deno.env.toObject(), GIT_TERMINAL_PROMPT: '0' }, }); const output = await cmd.output(); if (!output.success) { const stderr = new TextDecoder().decode(output.stderr); throw new Error(`git ${args[0]} failed: ${stderr.trim()}`); } return new TextDecoder().decode(output.stdout); } // ============================================================================ // Persistence // ============================================================================ private async loadConfigs(): Promise { const keys = await this.storageManager.list(SYNC_PREFIX); this.configs = []; for (const key of keys) { const config = await this.storageManager.getJSON(key); if (config) this.configs.push(config); } } private async persistConfig(config: interfaces.data.ISyncConfig): Promise { await this.storageManager.setJSON(`${SYNC_PREFIX}${config.id}.json`, config); } private async updateRepoStatus( syncConfigId: string, sourceFullPath: string, updates: Partial, ): Promise { const hash = this.sanitizePath(sourceFullPath).replace(/\//g, '__'); const key = `${SYNC_STATUS_PREFIX}${syncConfigId}/${hash}.json`; let status = await this.storageManager.getJSON(key); if (!status) { status = { id: hash, syncConfigId, sourceFullPath, targetFullPath: '', lastSyncAt: 0, status: 'pending', }; } Object.assign(status, updates); await this.storageManager.setJSON(key, status); } // ============================================================================ // Timer Management // ============================================================================ private startTimer(config: interfaces.data.ISyncConfig): void { this.stopTimer(config.id); const intervalMs = config.intervalMinutes * 60 * 1000; const timerId = setInterval(() => { this.executeSync(config.id).catch((err) => logger.error(`Scheduled sync for ${config.name} failed: ${err}`) ); }, intervalMs); Deno.unrefTimer(timerId); this.timers.set(config.id, timerId); } private stopTimer(configId: string): void { const timerId = this.timers.get(configId); if (timerId !== undefined) { clearInterval(timerId); this.timers.delete(configId); } } }