serve.zone/gitops
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sync): add parallel sync, fix default branch and protected branch issues, add group avatars option

Browse Source 
- Set sync concurrency to 10 for faster parallel repo syncing
- Three-phase push: push refs, sync default_branch via API, then push with --prune
- Unprotect stale protected branches on target before pruning
- Handle group visibility 400 errors gracefully (skip visibility, sync description only)
- Add useGroupAvatarsForProjects option: projects without avatars inherit group avatar
- Upgrade @apiclient.xyz/gitlab to v2.3.0 (getProtectedBranches, unprotectBranch)
This commit is contained in:
Jürgen Kunz
2026-02-28 17:39:28 +00:00
parent c9a758b417
commit 44ac2e430f
8 changed files with 148 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -17,7 +17,7 @@
"@api.global/typedserver": "8.4.0",
"@api.global/typedsocket": "^4.1.0",
"@apiclient.xyz/gitea": "1.2.0",
"@apiclient.xyz/gitlab": "2.2.0",
"@apiclient.xyz/gitlab": "2.3.0",
"@design.estate/dees-catalog": "^3.43.3",
"@design.estate/dees-element": "^2.1.6"
},

134
ts/classes/syncmanager.ts
View File

@@ -67,6 +67,7 @@ export class SyncManager {
enforceDelete?: boolean;
enforceGroupDelete?: boolean;
addMirrorHint?: boolean;
useGroupAvatarsForProjects?: boolean;
}): Promise<interfaces.data.ISyncConfig> {
const config: interfaces.data.ISyncConfig = {
id: crypto.randomUUID(),
@@ -81,6 +82,7 @@ export class SyncManager {
enforceDelete: data.enforceDelete ?? false,
enforceGroupDelete: data.enforceGroupDelete ?? false,
addMirrorHint: data.addMirrorHint ?? false,
useGroupAvatarsForProjects: data.useGroupAvatarsForProjects ?? false,
createdAt: Date.now(),
};
this.validateSyncConfig(config);
@@ -92,7 +94,7 @@ export class SyncManager {
async updateConfig(
id: string,
updates: { name?: string; targetGroupOffset?: string; intervalMinutes?: number; enforceDelete?: boolean; enforceGroupDelete?: boolean; addMirrorHint?: boolean },
updates: { name?: string; targetGroupOffset?: string; intervalMinutes?: number; enforceDelete?: boolean; enforceGroupDelete?: boolean; addMirrorHint?: boolean; useGroupAvatarsForProjects?: boolean },
): Promise<interfaces.data.ISyncConfig> {
const config = this.configs.find((c) => c.id === id);
if (!config) throw new Error(`Sync config not found: ${id}`);
@@ -101,6 +103,7 @@ export class SyncManager {
if (updates.enforceDelete !== undefined) config.enforceDelete = updates.enforceDelete;
if (updates.enforceGroupDelete !== undefined) config.enforceGroupDelete = updates.enforceGroupDelete;
if (updates.addMirrorHint !== undefined) config.addMirrorHint = updates.addMirrorHint;
if (updates.useGroupAvatarsForProjects !== undefined) config.useGroupAvatarsForProjects = updates.useGroupAvatarsForProjects;
if (updates.targetGroupOffset !== undefined) config.targetGroupOffset = updates.targetGroupOffset;
this.validateSyncConfig(config);
await this.persistConfig(config);
@@ -282,7 +285,7 @@ export class SyncManager {
logger.syncLog('info', `Found ${projects.length} source projects (${allProjects.length - projects.length} obsolete excluded)`, 'api');
let synced = 0;
const CONCURRENCY = 4;
const CONCURRENCY = 10;
for (let i = 0; i < projects.length; i += CONCURRENCY) {
const batch = projects.slice(i, i + CONCURRENCY);
await Promise.all(batch.map(async (project) => {
@@ -418,6 +421,20 @@ export class SyncManager {
});
}
// Phase 1: push all refs without pruning (ensures target has all source branches)
await this.runGit([
'push', 'target',
'+refs/heads/*:refs/heads/*',
'+refs/tags/*:refs/tags/*',
], mirrorDir);
// Phase 2: sync default_branch now that all branches exist on target
await this.syncDefaultBranchBeforePush(sourceConn, targetConn, project.fullPath, targetFullPath);
// Phase 2b: unprotect stale branches on target so --prune can delete them
await this.unprotectStaleBranches(targetConn, targetFullPath, mirrorDir);
// Phase 3: push with --prune to remove stale branches (safe now that default_branch is correct)
await this.runGit([
'push', 'target',
'+refs/heads/*:refs/heads/*',
@@ -997,6 +1014,81 @@ export class SyncManager {
return 'image/png'; // default
}
/**
* Pre-push: ensure target's default_branch matches source so --prune won't delete it.
*/
private async syncDefaultBranchBeforePush(
sourceConn: interfaces.data.IProviderConnection,
targetConn: interfaces.data.IProviderConnection,
sourceFullPath: string,
targetFullPath: string,
): Promise<void> {
try {
const sourceProject = await this.fetchProjectRaw(sourceConn, sourceFullPath);
if (!sourceProject) return;
const targetProject = await this.fetchProjectRaw(targetConn, targetFullPath);
if (!targetProject) return;
const sourceBranch = sourceProject.default_branch || 'main';
const targetBranch = targetProject.default_branch || 'main';
if (sourceBranch !== targetBranch) {
logger.syncLog('info', `Updating default branch for ${targetFullPath}: ${targetBranch} -> ${sourceBranch}`, 'api');
if (targetConn.providerType === 'gitlab') {
await this.rawApiCall(targetConn, 'PUT', `/api/v4/projects/${targetProject.id}`, {
default_branch: sourceBranch,
});
} else {
const segments = targetFullPath.split('/');
const repo = segments.pop()!;
const owner = segments[0] || '';
await this.rawApiCall(targetConn, 'PATCH', `/api/v1/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, {
default_branch: sourceBranch,
});
}
}
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
logger.syncLog('warn', `Pre-push default_branch sync failed for ${targetFullPath}: ${errMsg}`, 'api');
}
}
/**
* Unprotect branches on the target that no longer exist in the source,
* so that git push --prune can delete them.
*/
private async unprotectStaleBranches(
targetConn: interfaces.data.IProviderConnection,
targetFullPath: string,
mirrorDir: string,
): Promise<void> {
if (targetConn.providerType !== 'gitlab') return;
try {
const targetProject = await this.fetchProjectRaw(targetConn, targetFullPath);
if (!targetProject) return;
const client = new plugins.gitlabClient.GitLabClient(targetConn.baseUrl, targetConn.token);
const protectedBranches = await client.getProtectedBranches(targetProject.id);
if (protectedBranches.length === 0) return;
// Get list of branches in the local mirror (= source branches)
const localBranchOutput = await this.runGit(['branch', '--list'], mirrorDir);
const localBranches = new Set(
localBranchOutput.split('\n').map(b => b.trim().replace(/^\* /, '')).filter(Boolean),
);
for (const pb of protectedBranches) {
if (!localBranches.has(pb.name)) {
logger.syncLog('info', `Unprotecting stale branch "${pb.name}" on ${targetFullPath}`, 'api');
await client.unprotectBranch(targetProject.id, pb.name);
}
}
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
logger.syncLog('warn', `Failed to unprotect stale branches for ${targetFullPath}: ${errMsg}`, 'api');
}
}
/**
* Sync project metadata (description, visibility, topics, default_branch, avatar)
* from source to target after the git push.
@@ -1044,6 +1136,24 @@ export class SyncManager {
// Sync avatar
if (sourceMeta.avatarUrl) {
await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, sourceMeta.avatarUrl, targetProject);
} else if (config.useGroupAvatarsForProjects) {
// Project has no avatar — inherit from parent group
const groupPath = sourceFullPath.substring(0, sourceFullPath.lastIndexOf('/'));
if (groupPath) {
try {
const sourceGroup = await this.fetchGroupRaw(sourceConn, groupPath);
if (sourceGroup) {
const groupMeta = this.extractGroupMeta(sourceConn, sourceGroup);
if (groupMeta.avatarUrl) {
logger.syncLog('info', `Applying group avatar to ${targetFullPath}`, 'api');
await this.syncProjectAvatar(sourceConn, targetConn, sourceFullPath, targetFullPath, groupMeta.avatarUrl, targetProject);
}
}
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
logger.syncLog('warn', `Group avatar sync failed for ${targetFullPath}: ${errMsg}`, 'api');
}
}
}
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
@@ -1202,10 +1312,22 @@ export class SyncManager {
meta: { description: string; visibility: string },
): Promise<void> {
if (conn.providerType === 'gitlab') {
await this.rawApiCall(conn, 'PUT', `/api/v4/groups/${rawGroup.id}`, {
description: meta.description,
visibility: this.normalizeVisibility(meta.visibility),
});
try {
await this.rawApiCall(conn, 'PUT', `/api/v4/groups/${rawGroup.id}`, {
description: meta.description,
visibility: this.normalizeVisibility(meta.visibility),
});
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
if (errMsg.includes('visibility_level') || errMsg.includes('visibility')) {
logger.syncLog('warn', `Cannot sync visibility for group ${groupPath} (contains projects with higher visibility), syncing description only`, 'api');
await this.rawApiCall(conn, 'PUT', `/api/v4/groups/${rawGroup.id}`, {
description: meta.description,
});
} else {
throw err;
}
}
} else {
const orgName = groupPath.split('/')[0] || groupPath;
await this.rawApiCall(conn, 'PATCH', `/api/v1/orgs/${encodeURIComponent(orgName)}`, {

2
ts/opsserver/handlers/sync.handler.ts
View File

@@ -71,6 +71,7 @@ export class SyncHandler {
enforceDelete: dataArg.enforceDelete,
enforceGroupDelete: dataArg.enforceGroupDelete,
addMirrorHint: dataArg.addMirrorHint,
useGroupAvatarsForProjects: dataArg.useGroupAvatarsForProjects,
});
this.actionLog.append({
actionType: 'create',
@@ -98,6 +99,7 @@ export class SyncHandler {
enforceDelete: dataArg.enforceDelete,
enforceGroupDelete: dataArg.enforceGroupDelete,
addMirrorHint: dataArg.addMirrorHint,
useGroupAvatarsForProjects: dataArg.useGroupAvatarsForProjects,
});
this.actionLog.append({
actionType: 'update',

2
ts_bundled/bundle.ts
View File

File diff suppressed because one or more lines are too long

1
ts_interfaces/data/sync.ts
View File

@@ -15,6 +15,7 @@ export interface ISyncConfig {
enforceDelete: boolean; // When true, stale target repos are moved to obsolete
enforceGroupDelete: boolean; // When true, stale target groups/orgs are moved to obsolete
addMirrorHint?: boolean; // When true, target descriptions get "(This is a mirror of ...)" appended
useGroupAvatarsForProjects?: boolean; // When true, projects without avatars inherit the group avatar
createdAt: number;
}

2
ts_interfaces/requests/sync.ts
View File

@@ -29,6 +29,7 @@ export interface IReq_CreateSyncConfig extends plugins.typedrequestInterfaces.im
enforceDelete?: boolean;
enforceGroupDelete?: boolean;
addMirrorHint?: boolean;
useGroupAvatarsForProjects?: boolean;
};
response: {
config: data.ISyncConfig;
@@ -49,6 +50,7 @@ export interface IReq_UpdateSyncConfig extends plugins.typedrequestInterfaces.im
enforceDelete?: boolean;
enforceGroupDelete?: boolean;
addMirrorHint?: boolean;
useGroupAvatarsForProjects?: boolean;
};
response: {
config: data.ISyncConfig;

2
ts_web/appstate.ts
View File

@@ -742,6 +742,7 @@ export const createSyncConfigAction = syncStatePart.createAction<{
enforceDelete?: boolean;
enforceGroupDelete?: boolean;
addMirrorHint?: boolean;
useGroupAvatarsForProjects?: boolean;
}>(async (statePartArg, dataArg) => {
const context = getActionContext();
try {
@@ -769,6 +770,7 @@ export const updateSyncConfigAction = syncStatePart.createAction<{
enforceDelete?: boolean;
enforceGroupDelete?: boolean;
addMirrorHint?: boolean;
useGroupAvatarsForProjects?: boolean;
}>(async (statePartArg, dataArg) => {
const context = getActionContext();
try {

13
ts_web/elements/views/sync/index.ts
View File

@@ -104,6 +104,7 @@ export class GitopsViewSync extends DeesElement {
'Enforce Delete': item.enforceDelete ? 'Yes' : 'No',
'Enforce Group Delete': item.enforceGroupDelete ? 'Yes' : 'No',
'Mirror Hint': item.addMirrorHint ? 'Yes' : 'No',
'Group Avatars': item.useGroupAvatarsForProjects ? 'Yes' : 'No',
'Last Sync': item.lastSyncAt ? new Date(item.lastSyncAt).toLocaleString() : 'Never',
Repos: String(item.reposSynced),
};
@@ -288,6 +289,9 @@ export class GitopsViewSync extends DeesElement {
<div class="form-row">
<dees-input-checkbox .label=${'Add Mirror Hint'} .key=${'addMirrorHint'} .value=${false} .description=${'When enabled, target descriptions get "(This is a mirror of ...)" appended.'}></dees-input-checkbox>
</div>
<div class="form-row">
<dees-input-checkbox .label=${'Group Avatars for Projects'} .key=${'useGroupAvatarsForProjects'} .value=${false} .description=${'When enabled, projects without their own avatar inherit the group avatar.'}></dees-input-checkbox>
</div>
`,
menuOptions: [
{ name: 'Cancel', action: async (modal: any) => { modal.destroy(); } },
@@ -299,7 +303,7 @@ export class GitopsViewSync extends DeesElement {
for (const input of inputs) {
if (input.key === 'sourceConnectionId' || input.key === 'targetConnectionId') {
data[input.key] = input.selectedOption?.key || '';
} else if (input.key === 'enforceDelete' || input.key === 'enforceGroupDelete' || input.key === 'addMirrorHint') {
} else if (input.key === 'enforceDelete' || input.key === 'enforceGroupDelete' || input.key === 'addMirrorHint' || input.key === 'useGroupAvatarsForProjects') {
data[input.key] = input.getValue();
} else {
data[input.key] = input.value || '';
@@ -314,6 +318,7 @@ export class GitopsViewSync extends DeesElement {
enforceDelete: !!data.enforceDelete,
enforceGroupDelete: !!data.enforceGroupDelete,
addMirrorHint: !!data.addMirrorHint,
useGroupAvatarsForProjects: !!data.useGroupAvatarsForProjects,
});
modal.destroy();
},
@@ -345,6 +350,9 @@ export class GitopsViewSync extends DeesElement {
<div class="form-row">
<dees-input-checkbox .label=${'Add Mirror Hint'} .key=${'addMirrorHint'} .value=${!!item.addMirrorHint} .description=${'When enabled, target descriptions get "(This is a mirror of ...)" appended.'}></dees-input-checkbox>
</div>
<div class="form-row">
<dees-input-checkbox .label=${'Group Avatars for Projects'} .key=${'useGroupAvatarsForProjects'} .value=${!!item.useGroupAvatarsForProjects} .description=${'When enabled, projects without their own avatar inherit the group avatar.'}></dees-input-checkbox>
</div>
`,
menuOptions: [
{ name: 'Cancel', action: async (modal: any) => { modal.destroy(); } },
@@ -354,7 +362,7 @@ export class GitopsViewSync extends DeesElement {
const inputs = modal.shadowRoot.querySelectorAll('dees-input-text, dees-input-checkbox');
const data: any = {};
for (const input of inputs) {
if (input.key === 'enforceDelete' || input.key === 'enforceGroupDelete' || input.key === 'addMirrorHint') {
if (input.key === 'enforceDelete' || input.key === 'enforceGroupDelete' || input.key === 'addMirrorHint' || input.key === 'useGroupAvatarsForProjects') {
data[input.key] = input.getValue();
} else {
data[input.key] = input.value || '';
@@ -368,6 +376,7 @@ export class GitopsViewSync extends DeesElement {
enforceDelete: !!data.enforceDelete,
enforceGroupDelete: !!data.enforceGroupDelete,
addMirrorHint: !!data.addMirrorHint,
useGroupAvatarsForProjects: !!data.useGroupAvatarsForProjects,
});
modal.destroy();
},