feat(sync): add parallel sync, fix default branch and protected branch issues, add group avatars option
- Set sync concurrency to 10 for faster parallel repo syncing - Three-phase push: push refs, sync default_branch via API, then push with --prune - Unprotect stale protected branches on target before pruning - Handle group visibility 400 errors gracefully (skip visibility, sync description only) - Add useGroupAvatarsForProjects option: projects without avatars inherit group avatar - Upgrade @apiclient.xyz/gitlab to v2.3.0 (getProtectedBranches, unprotectBranch)
This commit is contained in:
@@ -67,6 +67,7 @@ export class SyncManager {
|
||||
enforceDelete?: boolean;
|
||||
enforceGroupDelete?: boolean;
|
||||
addMirrorHint?: boolean;
|
||||
useGroupAvatarsForProjects?: boolean;
|
||||
}): Promise<interfaces.data.ISyncConfig> {
|
||||
const config: interfaces.data.ISyncConfig = {
|
||||
id: crypto.randomUUID(),
|
||||
@@ -81,6 +82,7 @@ export class SyncManager {
|
||||
enforceDelete: data.enforceDelete ?? false,
|
||||
enforceGroupDelete: data.enforceGroupDelete ?? false,
|
||||
addMirrorHint: data.addMirrorHint ?? false,
|
||||
useGroupAvatarsForProjects: data.useGroupAvatarsForProjects ?? false,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
this.validateSyncConfig(config);
|
||||
@@ -92,7 +94,7 @@ export class SyncManager {
|
||||
|
||||
async updateConfig(
|
||||
id: string,
|
||||
updates: { name?: string; targetGroupOffset?: string; intervalMinutes?: number; enforceDelete?: boolean; enforceGroupDelete?: boolean; addMirrorHint?: boolean },
|
||||
updates: { name?: string; targetGroupOffset?: string; intervalMinutes?: number; enforceDelete?: boolean; enforceGroupDelete?: boolean; addMirrorHint?: boolean; useGroupAvatarsForProjects?: boolean },
|
||||
): Promise<interfaces.data.ISyncConfig> {
|
||||
const config = this.configs.find((c) => c.id === id);
|
||||
if (!config) throw new Error(`Sync config not found: ${id}`);
|
||||
@@ -101,6 +103,7 @@ export class SyncManager {
|
||||
if (updates.enforceDelete !== undefined) config.enforceDelete = updates.enforceDelete;
|
||||
if (updates.enforceGroupDelete !== undefined) config.enforceGroupDelete = updates.enforceGroupDelete;
|
||||
if (updates.addMirrorHint !== undefined) config.addMirrorHint = updates.addMirrorHint;
|
||||
if (updates.useGroupAvatarsForProjects !== undefined) config.useGroupAvatarsForProjects = updates.useGroupAvatarsForProjects;
|
||||
if (updates.targetGroupOffset !== undefined) config.targetGroupOffset = updates.targetGroupOffset;
|
||||
this.validateSyncConfig(config);
|
||||
await this.persistConfig(config);
|
||||
@@ -282,7 +285,7 @@ export class SyncManager {
|
||||
logger.syncLog('info', `Found ${projects.length} source projects (${allProjects.length - projects.length} obsolete excluded)`, 'api');
|
||||
|
||||
let synced = 0;
|
||||
const CONCURRENCY = 4;
|
||||
const CONCURRENCY = 10;
|
||||
for (let i = 0; i < projects.length; i += CONCURRENCY) {
|
||||
const batch = projects.slice(i, i + CONCURRENCY);
|
||||
await Promise.all(batch.map(async (project) => {
|
||||
@@ -418,6 +421,20 @@ export class SyncManager {
|
||||
});
|
||||
}
|
||||
|
||||
// Phase 1: push all refs without pruning (ensures target has all source branches)
|
||||
await this.runGit([
|
||||
'push', 'target',
|
||||
'+refs/heads/*:refs/heads/*',
|
||||
'+refs/tags/*:refs/tags/*',
|
||||
], mirrorDir);
|
||||
|
||||
// Phase 2: sync default_branch now that all branches exist on target
|
||||
await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath, targetFullPath);
|
||||
|
||||
// Phase 2b: unprotect stale branches on target so --prune can delete them
|
||||
await this.unprotectStaleBranches(targetConn, targetFullPath, mirrorDir);
|
||||
|
||||
// Phase 3: push with --prune to remove stale branches (safe now that default_branch is correct)
|
||||
await this.runGit([
|
||||
'push', 'target',
|
||||
'+refs/heads/*:refs/heads/*',
|
||||
@@ -997,6 +1014,81 @@ export class SyncManager {
|
||||
return 'image/png'; // default
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-push: ensure target's default_branch matches source so --prune won't delete it.
|
||||
*/
|
||||
private async syncDefaultBranchBeforePush(
|
||||
sourceConn: interfaces.data.IProviderConnection,
|
||||
targetConn: interfaces.data.IProviderConnection,
|
||||
sourceFullPath: string,
|
||||
targetFullPath: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const sourceProject = await this.fetchProjectRaw(sourceConn, sourceFullPath);
|
||||
if (!sourceProject) return;
|
||||
const targetProject = await this.fetchProjectRaw(targetConn, targetFullPath);
|
||||
if (!targetProject) return;
|
||||
|
||||
const sourceBranch = sourceProject.default_branch || 'main';
|
||||
const targetBranch = targetProject.default_branch || 'main';
|
||||
|
||||
if (sourceBranch !== targetBranch) {
|
||||
logger.syncLog('info', `Updating default branch for ${targetFullPath}: ${targetBranch} -> ${sourceBranch}`, 'api');
|
||||
if (targetConn.providerType === 'gitlab') {
|
||||
await this.rawApiCall(targetConn, 'PUT', `/api/v4/projects/${targetProject.id}`, {
|
||||
default_branch: sourceBranch,
|
||||
});
|
||||
} else {
|
||||
const segments = targetFullPath.split('/');
|
||||
const repo = segments.pop()!;
|
||||
const owner = segments[0] || '';
|
||||
await this.rawApiCall(targetConn, 'PATCH', `/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, {
|
||||
default_branch: sourceBranch,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
logger.syncLog('warn', `Pre-push default_branch sync failed for ${targetFullPath}: ${errMsg}`, 'api');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unprotect branches on the target that no longer exist in the source,
|
||||
* so that git push --prune can delete them.
|
||||
*/
|
||||
private async unprotectStaleBranches(
|
||||
targetConn: interfaces.data.IProviderConnection,
|
||||
targetFullPath: string,
|
||||
mirrorDir: string,
|
||||
): Promise<void> {
|
||||
if (targetConn.providerType !== 'gitlab') return;
|
||||
try {
|
||||
const targetProject = await this.fetchProjectRaw(targetConn, targetFullPath);
|
||||
if (!targetProject) return;
|
||||
|
||||
const client = new plugins.gitlabClient.GitLabClient(targetConn.baseUrl, targetConn.token);
|
||||
const protectedBranches = await client.getProtectedBranches(targetProject.id);
|
||||
if (protectedBranches.length === 0) return;
|
||||
|
||||
// Get list of branches in the local mirror (= source branches)
|
||||
const localBranchOutput = await this.runGit(['branch', '--list'], mirrorDir);
|
||||
const localBranches = new Set(
|
||||
localBranchOutput.split('\n').map(b => b.trim().replace(/^\* /, '')).filter(Boolean),
|
||||
);
|
||||
|
||||
for (const pb of protectedBranches) {
|
||||
if (!localBranches.has(pb.name)) {
|
||||
logger.syncLog('info', `Unprotecting stale branch "${pb.name}" on ${targetFullPath}`, 'api');
|
||||
await client.unprotectBranch(targetProject.id, pb.name);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
logger.syncLog('warn', `Failed to unprotect stale branches for ${targetFullPath}: ${errMsg}`, 'api');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync project metadata (description, visibility, topics, default_branch, avatar)
|
||||
* from source to target after the git push.
|
||||
@@ -1044,6 +1136,24 @@ export class SyncManager {
|
||||
// Sync avatar
|
||||
if (sourceMeta.avatarUrl) {
|
||||
await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, sourceMeta.avatarUrl, targetProject);
|
||||
} else if (config.useGroupAvatarsForProjects) {
|
||||
// Project has no avatar — inherit from parent group
|
||||
const groupPath = sourceFullPath.substring(0, sourceFullPath.lastIndexOf('/'));
|
||||
if (groupPath) {
|
||||
try {
|
||||
const sourceGroup = await this.fetchGroupRaw(sourceConn, groupPath);
|
||||
if (sourceGroup) {
|
||||
const groupMeta = this.extractGroupMeta(sourceConn, sourceGroup);
|
||||
if (groupMeta.avatarUrl) {
|
||||
logger.syncLog('info', `Applying group avatar to ${targetFullPath}`, 'api');
|
||||
await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, groupMeta.avatarUrl, targetProject);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
logger.syncLog('warn', `Group avatar sync failed for ${targetFullPath}: ${errMsg}`, 'api');
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
@@ -1202,10 +1312,22 @@ export class SyncManager {
|
||||
meta: { description: string; visibility: string },
|
||||
): Promise<void> {
|
||||
if (conn.providerType === 'gitlab') {
|
||||
await this.rawApiCall(conn, 'PUT', `/api/v4/groups/${rawGroup.id}`, {
|
||||
description: meta.description,
|
||||
visibility: this.normalizeVisibility(meta.visibility),
|
||||
});
|
||||
try {
|
||||
await this.rawApiCall(conn, 'PUT', `/api/v4/groups/${rawGroup.id}`, {
|
||||
description: meta.description,
|
||||
visibility: this.normalizeVisibility(meta.visibility),
|
||||
});
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
if (errMsg.includes('visibility_level') || errMsg.includes('visibility')) {
|
||||
logger.syncLog('warn', `Cannot sync visibility for group ${groupPath} (contains projects with higher visibility), syncing description only`, 'api');
|
||||
await this.rawApiCall(conn, 'PUT', `/api/v4/groups/${rawGroup.id}`, {
|
||||
description: meta.description,
|
||||
});
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const orgName = groupPath.split('/')[0] || groupPath;
|
||||
await this.rawApiCall(conn, 'PATCH', `/api/v1/orgs/${encodeURIComponent(orgName)}`, {
|
||||
|
||||
Reference in New Issue
Block a user