From 966c626d36ed4f71d871e9d60510ce4f9f358be9 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Thu, 28 May 2026 16:13:06 +0000 Subject: [PATCH] fix(services): clean up dependent resources during service deletion --- changelog.md | 10 + test/test.registrycleanup.node.ts | 86 +++++++ test/test.servicecleanup.node.ts | 137 +++++++++++ .../classes.appstoremanager.ts | 8 + ts/manager.backup/classes.backupmanager.ts | 12 + .../classes.replicationtarget.ts | 27 +++ ts/manager.image/classes.imagemanager.ts | 31 +++ .../classes.registrymanager.ts | 220 ++++++++++++++++++ ts/manager.service/classes.servicemanager.ts | 168 ++++++++++++- 9 files changed, 691 insertions(+), 8 deletions(-) create mode 100644 test/test.registrycleanup.node.ts create mode 100644 test/test.servicecleanup.node.ts diff --git a/changelog.md b/changelog.md index 7086702..b86bcc4 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,16 @@ ## Pending +### Fixes + +- make service deletion clean up dependent control-plane state (services) + - Delete appstore operations, deployments, DNS entries, platform bindings, backups, registry repositories, owned secrets, and unreferenced App Store images before removing the service row. + - Preserve shared OCI blobs while removing service-owned registry repositories. + - Keep the service row when required cleanup fails so deletion remains retryable. +- clean up dependent resources during service deletion (services) + - Remove dependent deployments, DNS entries, platform bindings, backups, registry repositories, app store operations, owned secret bundles, and unreferenced App Store images before deleting the service row. + - Delete service-owned backup prefixes and OCI registry repositories while preserving shared OCI blobs. + - Keep the service row when required cleanup fails so deletion remains retryable. ## 2026-05-27 - 6.4.1 diff --git a/test/test.registrycleanup.node.ts b/test/test.registrycleanup.node.ts new file mode 100644 index 0000000..b907de6 --- /dev/null +++ b/test/test.registrycleanup.node.ts @@ -0,0 +1,86 @@ +import { tap, expect } from '@git.zone/tstest/tapbundle'; + +import { CloudlyRegistryManager } from '../ts/manager.registry/classes.registrymanager.js'; + +const digest = (fillArg: string): string => `sha256:${fillArg.repeat(64)}`; + +class FakeRegistryStorage { + public objects = new Map(); + + public async getObject(pathArg: string): Promise { + return this.objects.get(pathArg) || null; + } + + public async putObject(pathArg: string, dataArg: Buffer): Promise { + this.objects.set(pathArg, dataArg); + } + + public async deleteObject(pathArg: string): Promise { + this.objects.delete(pathArg); + } + + public async objectExists(pathArg: string): Promise { + return this.objects.has(pathArg); + } + + public async listObjects(prefixArg: string): Promise { + return Array.from(this.objects.keys()).filter((pathArg) => pathArg.startsWith(prefixArg)); + } + + public async getOciManifest(repositoryArg: string, digestArg: string): Promise { + return this.getObject(`oci/manifests/${repositoryArg}/${digestArg.slice('sha256:'.length)}`); + } +} + +const putJson = (storageArg: FakeRegistryStorage, pathArg: string, dataArg: unknown): void => { + storageArg.objects.set(pathArg, Buffer.from(JSON.stringify(dataArg))); +}; + +tap.test('should delete Cloudly service-owned OCI repository without deleting shared blobs', async () => { + const storage = new FakeRegistryStorage(); + const manager = Object.create(CloudlyRegistryManager.prototype) as any; + const service = { + id: 'service-1', + data: { + registryTarget: { repository: 'workloads/ghost-service-1' }, + }, + }; + manager.started = true; + manager.smartRegistry = { getStorage: () => storage }; + manager.recordedTagDigests = new Map([ + ['workloads/ghost-service-1:latest', digest('a')], + ]); + manager.cloudlyRef = { + serviceManager: { + CService: { getInstances: async () => [service] }, + }, + }; + + const targetDigest = digest('a'); + const otherDigest = digest('b'); + const sharedLayerDigest = digest('c'); + const targetOnlyLayerDigest = digest('d'); + const otherOnlyLayerDigest = digest('e'); + putJson(storage, 'oci/tags/workloads/ghost-service-1/tags.json', { latest: targetDigest }); + putJson(storage, 'oci/tags/workloads/other-service/tags.json', { latest: otherDigest }); + putJson(storage, `oci/manifests/workloads/ghost-service-1/${targetDigest.slice('sha256:'.length)}`, { + layers: [{ digest: sharedLayerDigest }, { digest: targetOnlyLayerDigest }], + }); + putJson(storage, `oci/manifests/workloads/other-service/${otherDigest.slice('sha256:'.length)}`, { + layers: [{ digest: sharedLayerDigest }, { digest: otherOnlyLayerDigest }], + }); + for (const blobDigest of [sharedLayerDigest, targetOnlyLayerDigest, otherOnlyLayerDigest]) { + storage.objects.set(`oci/blobs/sha256/${blobDigest.slice('sha256:'.length)}`, Buffer.from(blobDigest)); + } + + await manager.deleteServiceRepository(service); + + expect(storage.objects.has('oci/tags/workloads/ghost-service-1/tags.json')).toBeFalse(); + expect(storage.objects.has(`oci/manifests/workloads/ghost-service-1/${targetDigest.slice('sha256:'.length)}`)).toBeFalse(); + expect(storage.objects.has(`oci/blobs/sha256/${targetOnlyLayerDigest.slice('sha256:'.length)}`)).toBeFalse(); + expect(storage.objects.has(`oci/blobs/sha256/${sharedLayerDigest.slice('sha256:'.length)}`)).toBeTrue(); + expect(storage.objects.has(`oci/blobs/sha256/${otherOnlyLayerDigest.slice('sha256:'.length)}`)).toBeTrue(); + expect(manager.recordedTagDigests.has('workloads/ghost-service-1:latest')).toBeFalse(); +}); + +export default tap.start(); diff --git a/test/test.servicecleanup.node.ts b/test/test.servicecleanup.node.ts new file mode 100644 index 0000000..6b08bbd --- /dev/null +++ b/test/test.servicecleanup.node.ts @@ -0,0 +1,137 @@ +import { tap, expect } from '@git.zone/tstest/tapbundle'; + +import { ServiceManager } from '../ts/manager.service/classes.servicemanager.js'; + +const createDeleteable = (labelArg: string, callsArg: string[]) => ({ + id: labelArg, + delete: async () => callsArg.push(`delete:${labelArg}`), +}); + +const createManager = (optionsArg: { + calls: string[]; + failBackups?: boolean; +}) => { + const calls = optionsArg.calls; + const service = { + id: 'service-1', + data: { + name: 'ghost', + imageId: 'image-1', + appTemplateId: 'ghost', + secretBundleId: 'bundle-1', + registryTarget: { repository: 'workloads/ghost-service-1' }, + domains: [{ name: 'ghost.example.com' }], + }, + removeDnsEntries: async () => calls.push('delete:dns'), + delete: async () => calls.push('delete:service'), + }; + const manager = Object.create(ServiceManager.prototype) as any; + manager.CService = { + getInstance: async () => service, + }; + manager.cloudlyRef = { + appStoreManager: { + clearOperationsForService: (serviceIdArg: string) => calls.push(`clear-upgrades:${serviceIdArg}`), + }, + deploymentManager: { + CDeployment: { + getInstances: async () => [createDeleteable('deployment-1', calls)], + }, + }, + platformManager: { + CPlatformBinding: { + getInstances: async (queryArg: { serviceId: string }) => queryArg.serviceId === 'service-1' + ? [createDeleteable('platform-binding-1', calls)] + : [], + }, + }, + backupManager: { + deleteBackupsForService: async (serviceIdArg: string) => { + calls.push(`delete-backups:${serviceIdArg}`); + if (optionsArg.failBackups) { + throw new Error('backup cleanup failed'); + } + }, + }, + registryManager: { + deleteServiceRepository: async () => calls.push('delete:registry-repository'), + }, + secretManager: { + CSecretBundle: { + getInstance: async () => ({ + id: 'bundle-1', + data: { + serviceId: 'service-1', + includedSecretGroupIds: ['group-1'], + }, + delete: async () => calls.push('delete:secret-bundle'), + }), + getInstances: async () => [], + }, + CSecretGroup: { + getInstance: async () => createDeleteable('secret-group-1', calls), + }, + }, + imageManager: { + deleteImageIfUnreferenced: async (imageIdArg: string, serviceIdArg: string) => { + calls.push(`delete-image:${imageIdArg}:${serviceIdArg}`); + return true; + }, + }, + settingsManager: { + getSettings: async () => ({ + dcrouterGatewayUrl: 'https://dcrouter.example.com', + dcrouterGatewayApiToken: 'token', + dcrouterWorkHosterId: 'cloudly-main', + }), + }, + clusterManager: { + getAllClusters: async () => [], + }, + coreflowManager: { + pushClusterConfigToConnectedCoreflows: async () => calls.push('push:coreflow'), + }, + }; + manager.fireDcRouterRequest = async (_url: string, methodArg: string, payloadArg: any) => { + calls.push(`${methodArg}:${payloadArg.ownership.hostname}:${payloadArg.ownership.workHosterId}`); + return { success: true, action: 'deleted' }; + }; + return { manager, service }; +}; + +tap.test('should delete Cloudly service-owned resources before deleting the service row', async () => { + const calls: string[] = []; + const { manager } = createManager({ calls }); + + await manager.deleteServiceById('service-1'); + + expect(calls).toContain('syncWorkAppRoute:ghost.example.com:cloudly-main'); + expect(calls).toContain('clear-upgrades:service-1'); + expect(calls).toContain('delete:deployment-1'); + expect(calls).toContain('delete:dns'); + expect(calls).toContain('delete:platform-binding-1'); + expect(calls).toContain('delete-backups:service-1'); + expect(calls).toContain('delete:registry-repository'); + expect(calls).toContain('delete:secret-bundle'); + expect(calls).toContain('delete:secret-group-1'); + expect(calls).toContain('delete-image:image-1:service-1'); + expect(calls.at(-2)).toEqual('delete:service'); + expect(calls.at(-1)).toEqual('push:coreflow'); +}); + +tap.test('should keep Cloudly service row when required cleanup fails', async () => { + const calls: string[] = []; + const { manager } = createManager({ calls, failBackups: true }); + + let errorMessage = ''; + try { + await manager.deleteServiceById('service-1'); + } catch (error) { + errorMessage = (error as Error).message; + } + expect(errorMessage).toEqual('backup cleanup failed'); + expect(calls).not.toContain('delete:service'); + expect(calls).not.toContain('push:coreflow'); +}); + +export default tap.start(); diff --git a/ts/manager.appstore/classes.appstoremanager.ts b/ts/manager.appstore/classes.appstoremanager.ts index a88fcb6..10dbd85 100644 --- a/ts/manager.appstore/classes.appstoremanager.ts +++ b/ts/manager.appstore/classes.appstoremanager.ts @@ -294,6 +294,14 @@ export class CloudlyAppStoreManager { .slice(0, 25); } + public clearOperationsForService(serviceIdArg: string): void { + for (const [operationId, operation] of this.upgradeOperations.entries()) { + if (operation.serviceId === serviceIdArg) { + this.upgradeOperations.delete(operationId); + } + } + } + public async startHostedAppUpgrade(serviceIdArg: string, targetVersionArg: string): Promise { const operation = await this.createUpgradeOperation(serviceIdArg, targetVersionArg); void this.performUpgrade(operation.id).catch(() => {}); diff --git a/ts/manager.backup/classes.backupmanager.ts b/ts/manager.backup/classes.backupmanager.ts index 9f4967f..5e92c32 100644 --- a/ts/manager.backup/classes.backupmanager.ts +++ b/ts/manager.backup/classes.backupmanager.ts @@ -198,6 +198,18 @@ export class CloudlyBackupManager { return await backup.createSavableObject(); } + public async deleteBackupsForService(serviceIdArg: string): Promise { + const backups = await this.CBackupRecord.getInstances({ + serviceId: serviceIdArg, + }); + for (const backup of backups) { + if (backup.replication?.targetPath) { + await this.getBackupTargetWriter().deletePrefix(backup.replication.targetPath); + } + await backup.delete(); + } + } + public async backupAllServices() { const services = await this.cloudlyRef.serviceManager.CService.getInstances({}); const results: Array<{ serviceId: string; backupId?: string; errorText?: string }> = []; diff --git a/ts/manager.backup/classes.replicationtarget.ts b/ts/manager.backup/classes.replicationtarget.ts index 9bf9b37..eeaed69 100644 --- a/ts/manager.backup/classes.replicationtarget.ts +++ b/ts/manager.backup/classes.replicationtarget.ts @@ -12,6 +12,7 @@ export interface IBackupTargetWriter { hasObject(pathArg: string, objectArg: TArchiveObject): Promise; putObject(pathArg: string, objectArg: TArchiveObject, contentsArg: Buffer): Promise; readObject(pathArg: string): Promise; + deletePrefix(pathPrefixArg: string): Promise; } const requiredEnv = (nameArg: string) => { @@ -103,6 +104,14 @@ class S3BackupTargetWriter implements IBackupTargetWriter { const bucket = await this.getBucket(); return await bucket.fastGet({ path: normalizeRemotePath(pathArg) }); } + + public async deletePrefix(pathPrefixArg: string): Promise { + const bucket = await this.getBucket(); + const prefix = normalizeRemotePath(pathPrefixArg).replace(/\/+$/, ''); + for await (const objectPath of bucket.listAllObjects(prefix)) { + await bucket.fastRemove({ path: objectPath }); + } + } } class SmbBackupTargetWriter implements IBackupTargetWriter { @@ -183,6 +192,24 @@ class SmbBackupTargetWriter implements IBackupTargetWriter { public async readObject(pathArg: string) { return await (await this.getClient()).readFile(this.getShare(), normalizeRemotePath(pathArg)); } + + public async deletePrefix(pathPrefixArg: string): Promise { + const client = await this.getClient(); + const share = this.getShare(); + const rootPath = normalizeRemotePath(pathPrefixArg).replace(/\/+$/, ''); + const deleteDirectoryFiles = async (pathArg: string): Promise => { + const entries = await client.listDirectory(share, pathArg).catch(() => []); + for (const entry of entries) { + const childPath = `${pathArg}/${entry.name}`.replace(/^\/+/, ''); + if (entry.isDirectory) { + await deleteDirectoryFiles(childPath); + } else { + await client.deleteFile(share, childPath).catch(() => {}); + } + } + }; + await deleteDirectoryFiles(rootPath); + } } export const createBackupTargetWriterFromEnv = (): IBackupTargetWriter => { diff --git a/ts/manager.image/classes.imagemanager.ts b/ts/manager.image/classes.imagemanager.ts index 7f2baee..a4804b9 100644 --- a/ts/manager.image/classes.imagemanager.ts +++ b/ts/manager.image/classes.imagemanager.ts @@ -212,4 +212,35 @@ export class ImageManager { bucketDir: this.imageDir, }); } + + public async deleteImageIfUnreferenced(imageIdArg: string, ownerServiceIdArg?: string): Promise { + if (!imageIdArg) return false; + + const referencingServices = await this.cloudlyRef.serviceManager.CService.getInstances({}); + const referencedByOtherService = referencingServices.some((serviceArg) => { + return serviceArg.id !== ownerServiceIdArg && serviceArg.data?.imageId === imageIdArg; + }); + if (referencedByOtherService) return false; + + const image = await this.CImage.getInstance({ + id: imageIdArg, + }).catch(() => null); + if (!image) return false; + + for (const version of image.data.versions || []) { + const storagePath = version.storagePath || await image.getStoragePath(version.versionString); + if (!storagePath || !this.imageDir) continue; + await this.imageDir.fastRemove({ + path: `${storagePath}.tar`, + }).catch((errorArg) => { + const message = (errorArg as Error).message.toLowerCase(); + if (!message.includes('not found') && !message.includes('no such')) { + throw errorArg; + } + }); + } + + await image.delete(); + return true; + } } diff --git a/ts/manager.registry/classes.registrymanager.ts b/ts/manager.registry/classes.registrymanager.ts index 03e3fd2..4b7daf1 100644 --- a/ts/manager.registry/classes.registrymanager.ts +++ b/ts/manager.registry/classes.registrymanager.ts @@ -9,6 +9,18 @@ type TAuthenticatedRegistryUser = { canWrite: boolean; }; +type TOciTags = Record; + +interface IOciDescriptor { + digest?: unknown; +} + +interface IOciManifestDocument { + config?: IOciDescriptor; + layers?: IOciDescriptor[]; + manifests?: IOciDescriptor[]; +} + export class CloudlyRegistryManager { private cloudlyRef: Cloudly; private smartRegistry!: plugins.smartregistry.SmartRegistry; @@ -123,6 +135,214 @@ export class CloudlyRegistryManager { }; } + public async deleteServiceRepository(serviceArg: Service): Promise { + const repository = serviceArg.data.registryTarget?.repository; + if (!repository) return; + + const services = await this.cloudlyRef.serviceManager.CService.getInstances({}); + const referencedByOtherService = services.some((candidateArg) => { + return candidateArg.id !== serviceArg.id && candidateArg.data.registryTarget?.repository === repository; + }); + if (referencedByOtherService) return; + + await this.deleteOciRepository(repository); + for (const tagKey of Array.from(this.recordedTagDigests.keys())) { + if (tagKey.startsWith(`${repository}:`)) { + this.recordedTagDigests.delete(tagKey); + } + } + } + + private getRegistryStorage(): any { + if (!this.started || !this.smartRegistry) { + throw new Error('Cloudly registry is not started'); + } + return this.smartRegistry.getStorage(); + } + + private getOciTagsPath(repositoryArg: string): string { + return `oci/tags/${repositoryArg}/tags.json`; + } + + private normalizeOciDigest(digestArg: string | null | undefined): string | null { + if (typeof digestArg !== 'string') return null; + const normalizedDigest = digestArg.trim().toLowerCase(); + return /^sha256:[a-f0-9]{64}$/.test(normalizedDigest) ? normalizedDigest : null; + } + + private getSha256HashFromDigest(digestArg: string): string { + const normalizedDigest = this.normalizeOciDigest(digestArg); + if (!normalizedDigest) { + throw new Error(`Invalid OCI digest: ${digestArg}`); + } + return normalizedDigest.slice('sha256:'.length); + } + + private getOciManifestPath(repositoryArg: string, digestArg: string): string { + return `oci/manifests/${repositoryArg}/${this.getSha256HashFromDigest(digestArg)}`; + } + + private getOciBlobPath(digestArg: string): string { + return `oci/blobs/sha256/${this.getSha256HashFromDigest(digestArg)}`; + } + + private async readOciTags(repositoryArg: string, storageArg = this.getRegistryStorage()): Promise { + const tagsBuffer = await storageArg.getObject(this.getOciTagsPath(repositoryArg)); + if (!tagsBuffer) return {}; + + const parsedTags = JSON.parse(tagsBuffer.toString('utf8')); + if (!parsedTags || typeof parsedTags !== 'object' || Array.isArray(parsedTags)) { + throw new Error(`Invalid OCI tags document for ${repositoryArg}`); + } + + const tags: TOciTags = {}; + for (const [tagName, digestValue] of Object.entries(parsedTags)) { + const digest = typeof digestValue === 'string' ? this.normalizeOciDigest(digestValue) : null; + if (!digest) { + throw new Error(`Invalid OCI digest for ${repositoryArg}:${tagName}`); + } + tags[tagName] = digest; + } + return tags; + } + + private async readOciManifest( + storageArg: any, + repositoryArg: string, + digestArg: string, + ): Promise { + const manifestBuffer = await storageArg.getOciManifest(repositoryArg, digestArg); + if (!manifestBuffer) return null; + try { + const manifest = JSON.parse(manifestBuffer.toString('utf8')); + return manifest && typeof manifest === 'object' ? manifest : null; + } catch (error) { + logger.log('warn', `failed to parse OCI manifest ${repositoryArg}@${digestArg}: ${(error as Error).message}`); + return null; + } + } + + private getDescriptorDigest(descriptorArg: IOciDescriptor | undefined): string | null { + return typeof descriptorArg?.digest === 'string' ? this.normalizeOciDigest(descriptorArg.digest) : null; + } + + private collectOciManifestReferences(manifestArg: IOciManifestDocument): { + blobDigests: string[]; + manifestDigests: string[]; + } { + const blobDigests = [ + this.getDescriptorDigest(manifestArg.config), + ...(manifestArg.layers || []).map((descriptorArg) => this.getDescriptorDigest(descriptorArg)), + ].filter((digestArg): digestArg is string => Boolean(digestArg)); + + const manifestDigests = (manifestArg.manifests || []) + .map((descriptorArg) => this.getDescriptorDigest(descriptorArg)) + .filter((digestArg): digestArg is string => Boolean(digestArg)); + + return { blobDigests, manifestDigests }; + } + + private async collectReferencedOciObjects( + storageArg: any, + repositoryArg: string, + rootDigestsArg: string[], + ): Promise<{ manifestDigests: Set; blobDigests: Set }> { + const manifestDigests = new Set(); + const blobDigests = new Set(); + const pendingManifestDigests = rootDigestsArg + .map((digestArg) => this.normalizeOciDigest(digestArg)) + .filter((digestArg): digestArg is string => Boolean(digestArg)); + + while (pendingManifestDigests.length > 0) { + const manifestDigest = pendingManifestDigests.shift()!; + if (manifestDigests.has(manifestDigest)) continue; + manifestDigests.add(manifestDigest); + + const manifest = await this.readOciManifest(storageArg, repositoryArg, manifestDigest); + if (!manifest) continue; + const references = this.collectOciManifestReferences(manifest); + for (const blobDigest of references.blobDigests) { + blobDigests.add(blobDigest); + } + for (const childManifestDigest of references.manifestDigests) { + if (!manifestDigests.has(childManifestDigest)) { + pendingManifestDigests.push(childManifestDigest); + } + } + } + + return { manifestDigests, blobDigests }; + } + + private async listRepositoryManifestDigests(storageArg: any, repositoryArg: string): Promise { + const manifestPrefix = `oci/manifests/${repositoryArg}/`; + const paths = await storageArg.listObjects(manifestPrefix); + return paths + .filter((pathArg: string) => !pathArg.endsWith('.type')) + .map((pathArg: string) => pathArg.slice(manifestPrefix.length)) + .filter((hashArg: string) => /^[a-f0-9]{64}$/.test(hashArg)) + .map((hashArg: string) => `sha256:${hashArg}`); + } + + private async collectAllTaggedOciObjectsExceptRepository(storageArg: any, repositoryArg: string) { + const protectedObjects = { + manifestDigests: new Set(), + blobDigests: new Set(), + }; + const tagPaths = await storageArg.listObjects('oci/tags/'); + for (const tagPath of tagPaths) { + const match = tagPath.match(/^oci\/tags\/(.+)\/tags\.json$/); + if (!match || match[1] === repositoryArg) continue; + const tags = await this.readOciTags(match[1], storageArg); + const references = await this.collectReferencedOciObjects(storageArg, match[1], Object.values(tags)); + for (const digest of references.manifestDigests) { + protectedObjects.manifestDigests.add(digest); + } + for (const digest of references.blobDigests) { + protectedObjects.blobDigests.add(digest); + } + } + return protectedObjects; + } + + private async deleteObjectIfExists(storageArg: any, pathArg: string): Promise { + if (typeof storageArg.objectExists === 'function' && !(await storageArg.objectExists(pathArg))) { + return; + } + await storageArg.deleteObject(pathArg); + } + + private async deleteOciRepository(repositoryArg: string): Promise { + const storage = this.getRegistryStorage(); + const tags = await this.readOciTags(repositoryArg, storage); + const repositoryManifestDigests = await this.listRepositoryManifestDigests(storage, repositoryArg); + const rootDigests = Array.from(new Set([...Object.values(tags), ...repositoryManifestDigests])); + if (rootDigests.length === 0) { + await this.deleteObjectIfExists(storage, this.getOciTagsPath(repositoryArg)); + return; + } + + const targetObjects = await this.collectReferencedOciObjects(storage, repositoryArg, rootDigests); + const protectedObjects = await this.collectAllTaggedOciObjectsExceptRepository(storage, repositoryArg); + + for (const blobDigest of targetObjects.blobDigests) { + if (!protectedObjects.blobDigests.has(blobDigest)) { + await this.deleteObjectIfExists(storage, this.getOciBlobPath(blobDigest)); + } + } + + for (const manifestDigest of targetObjects.manifestDigests) { + if (!protectedObjects.manifestDigests.has(manifestDigest)) { + const manifestPath = this.getOciManifestPath(repositoryArg, manifestDigest); + await this.deleteObjectIfExists(storage, manifestPath); + await this.deleteObjectIfExists(storage, `${manifestPath}.type`); + } + } + + await this.deleteObjectIfExists(storage, this.getOciTagsPath(repositoryArg)); + logger.log('info', `deleted Cloudly registry repository ${repositoryArg}`); + } + private async handleRegistryStorageAfterPut( contextArg: plugins.smartregistry.IStorageHookContext, ) { diff --git a/ts/manager.service/classes.servicemanager.ts b/ts/manager.service/classes.servicemanager.ts index 503c56f..71d4a99 100644 --- a/ts/manager.service/classes.servicemanager.ts +++ b/ts/manager.service/classes.servicemanager.ts @@ -2,6 +2,20 @@ import type { Cloudly } from '../classes.cloudly.js'; import * as plugins from '../plugins.js'; import { Service } from './classes.service.js'; +type TServiceWithDomains = Service & { + data: Service['data'] & { + appTemplateId?: string; + domains?: Array<{ name?: string }>; + }; +}; + +interface IWorkAppRouteSyncResult { + success: boolean; + action?: 'created' | 'updated' | 'deleted' | 'unchanged'; + routeId?: string; + message?: string; +} + export class ServiceManager { public typedrouter = new plugins.typedrequest.TypedRouter(); public cloudlyRef: Cloudly; @@ -145,14 +159,7 @@ export class ServiceManager { await plugins.smartguard.passGuardsOrReject(dataArg, [ this.cloudlyRef.authManager.adminIdentityGuard, ]); - const service = await Service.getInstance({ - id: dataArg.serviceId, - }); - - // Remove DNS entries before deleting the service - await service.removeDnsEntries(); - - await service.delete(); + await this.deleteServiceById(dataArg.serviceId); return { success: true, }; @@ -170,4 +177,149 @@ export class ServiceManager { // Cleanup if needed console.log('ServiceManager stopped'); } + + public async deleteServiceById(serviceIdArg: string): Promise { + const service = await this.CService.getInstance({ + id: serviceIdArg, + }); + if (!service) { + throw new plugins.typedrequest.TypedResponseError(`Service not found: ${serviceIdArg}`); + } + + await this.deleteExternalGatewayRoutes(service as TServiceWithDomains); + this.cloudlyRef.appStoreManager?.clearOperationsForService?.(service.id); + await this.deleteDeploymentsForService(service.id); + await service.removeDnsEntries(); + await this.deletePlatformBindingsForService(service.id, service.data.name); + await this.cloudlyRef.backupManager?.deleteBackupsForService?.(service.id); + await this.cloudlyRef.registryManager?.deleteServiceRepository?.(service); + await this.deleteServiceOwnedSecretBundles(service); + await this.deleteServiceOwnedImage(service as TServiceWithDomains); + await service.delete(); + await this.cloudlyRef.coreflowManager.pushClusterConfigToConnectedCoreflows(); + } + + private async deleteDeploymentsForService(serviceIdArg: string): Promise { + const deployments = await this.cloudlyRef.deploymentManager.CDeployment.getInstances({ + serviceId: serviceIdArg, + }); + for (const deployment of deployments) { + await deployment.delete(); + } + } + + private async deletePlatformBindingsForService( + serviceIdArg: string, + serviceNameArg: string, + ): Promise { + const bindingsById = await this.cloudlyRef.platformManager.CPlatformBinding.getInstances({ + serviceId: serviceIdArg, + }); + const bindingsByName = serviceNameArg + ? await this.cloudlyRef.platformManager.CPlatformBinding.getInstances({ serviceId: serviceNameArg }) + : []; + const bindings = new Map(); + for (const binding of [...bindingsById, ...bindingsByName]) { + bindings.set(binding.id, binding); + } + for (const binding of bindings.values()) { + await binding.delete(); + } + } + + private async deleteServiceOwnedSecretBundles(serviceArg: Service): Promise { + const secretBundleIds = [serviceArg.data.secretBundleId] + .filter((secretBundleIdArg): secretBundleIdArg is string => Boolean(secretBundleIdArg)); + if (secretBundleIds.length === 0) return; + + for (const secretBundleId of secretBundleIds) { + const secretBundle = await this.cloudlyRef.secretManager.CSecretBundle.getInstance({ + id: secretBundleId, + }).catch(() => null); + if (!secretBundle || (secretBundle.data as { serviceId?: string }).serviceId !== serviceArg.id) { + continue; + } + + const secretGroupIds = [...(secretBundle.data.includedSecretGroupIds || [])]; + await secretBundle.delete(); + await this.deleteUnreferencedSecretGroups(secretGroupIds); + } + } + + private async deleteUnreferencedSecretGroups(secretGroupIdsArg: string[]): Promise { + if (secretGroupIdsArg.length === 0) return; + const remainingBundles = await this.cloudlyRef.secretManager.CSecretBundle.getInstances({}); + for (const secretGroupId of secretGroupIdsArg) { + const stillReferenced = remainingBundles.some((bundleArg) => { + return (bundleArg.data.includedSecretGroupIds || []).includes(secretGroupId); + }); + if (stillReferenced) continue; + const secretGroup = await this.cloudlyRef.secretManager.CSecretGroup.getInstance({ + id: secretGroupId, + }).catch(() => null); + if (secretGroup) { + await secretGroup.delete(); + } + } + } + + private async deleteServiceOwnedImage(serviceArg: TServiceWithDomains): Promise { + if (!serviceArg.data.appTemplateId || !serviceArg.data.imageId) return; + await this.cloudlyRef.imageManager.deleteImageIfUnreferenced(serviceArg.data.imageId, serviceArg.id); + } + + private async deleteExternalGatewayRoutes(serviceArg: TServiceWithDomains): Promise { + const domains = (serviceArg.data.domains || []) + .map((domainArg) => domainArg.name?.trim().toLowerCase()) + .filter((domainArg): domainArg is string => Boolean(domainArg)); + if (domains.length === 0) return; + + const settings = await this.cloudlyRef.settingsManager.getSettings().catch(() => undefined); + if (!settings?.dcrouterGatewayUrl || !settings.dcrouterGatewayApiToken) return; + + const clusters = await this.cloudlyRef.clusterManager.getAllClusters().catch(() => []); + const workHosterIds = new Set(); + if (settings.dcrouterWorkHosterId) { + workHosterIds.add(settings.dcrouterWorkHosterId); + } else { + for (const cluster of clusters) { + workHosterIds.add(cluster.id); + } + } + if (workHosterIds.size === 0) return; + + for (const domain of domains) { + for (const workHosterId of workHosterIds) { + const result = await this.fireDcRouterRequest( + settings.dcrouterGatewayUrl, + 'syncWorkAppRoute', + { + apiToken: settings.dcrouterGatewayApiToken, + ownership: { + workHosterType: 'cloudly', + workHosterId, + workAppId: serviceArg.id || serviceArg.data.name, + hostname: domain, + }, + delete: true, + }, + ); + if (!result.success) { + throw new Error(result.message || `dcrouter route delete failed for ${domain}`); + } + } + } + } + + private async fireDcRouterRequest( + gatewayUrlArg: string, + methodArg: string, + requestDataArg: Record, + ): Promise { + const typedRequest = new plugins.typedrequest.TypedRequest( + `${gatewayUrlArg.replace(/\/+$/, '')}/typedrequest`, + methodArg, + ); + return await typedRequest.fire(requestDataArg) as TResponse; + } }