fix(services): clean up dependent resources during service deletion

changelog.md

@@ -2,6 +2,16 @@
 
 ## Pending
 
+### Fixes
+
+- make service deletion clean up dependent control-plane state (services)
+  - Delete appstore operations, deployments, DNS entries, platform bindings, backups, registry repositories, owned secrets, and unreferenced App Store images before removing the service row.
+  - Preserve shared OCI blobs while removing service-owned registry repositories.
+  - Keep the service row when required cleanup fails so deletion remains retryable.
+- clean up dependent resources during service deletion (services)
+  - Remove dependent deployments, DNS entries, platform bindings, backups, registry repositories, app store operations, owned secret bundles, and unreferenced App Store images before deleting the service row.
+  - Delete service-owned backup prefixes and OCI registry repositories while preserving shared OCI blobs.
+  - Keep the service row when required cleanup fails so deletion remains retryable.
 
 ## 2026-05-27 - 6.4.1
 

test/test.registrycleanup.node.ts

@@ -0,0 +1,86 @@
+import { tap, expect } from '@git.zone/tstest/tapbundle';
+
+import { CloudlyRegistryManager } from '../ts/manager.registry/classes.registrymanager.js';
+
+const digest = (fillArg: string): string => `sha256:${fillArg.repeat(64)}`;
+
+class FakeRegistryStorage {
+  public objects = new Map<string, Buffer>();
+
+  public async getObject(pathArg: string): Promise<Buffer | null> {
+    return this.objects.get(pathArg) || null;
+  }
+
+  public async putObject(pathArg: string, dataArg: Buffer): Promise<void> {
+    this.objects.set(pathArg, dataArg);
+  }
+
+  public async deleteObject(pathArg: string): Promise<void> {
+    this.objects.delete(pathArg);
+  }
+
+  public async objectExists(pathArg: string): Promise<boolean> {
+    return this.objects.has(pathArg);
+  }
+
+  public async listObjects(prefixArg: string): Promise<string[]> {
+    return Array.from(this.objects.keys()).filter((pathArg) => pathArg.startsWith(prefixArg));
+  }
+
+  public async getOciManifest(repositoryArg: string, digestArg: string): Promise<Buffer | null> {
+    return this.getObject(`oci/manifests/${repositoryArg}/${digestArg.slice('sha256:'.length)}`);
+  }
+}
+
+const putJson = (storageArg: FakeRegistryStorage, pathArg: string, dataArg: unknown): void => {
+  storageArg.objects.set(pathArg, Buffer.from(JSON.stringify(dataArg)));
+};
+
+tap.test('should delete Cloudly service-owned OCI repository without deleting shared blobs', async () => {
+  const storage = new FakeRegistryStorage();
+  const manager = Object.create(CloudlyRegistryManager.prototype) as any;
+  const service = {
+    id: 'service-1',
+    data: {
+      registryTarget: { repository: 'workloads/ghost-service-1' },
+    },
+  };
+  manager.started = true;
+  manager.smartRegistry = { getStorage: () => storage };
+  manager.recordedTagDigests = new Map([
+    ['workloads/ghost-service-1:latest', digest('a')],
+  ]);
+  manager.cloudlyRef = {
+    serviceManager: {
+      CService: { getInstances: async () => [service] },
+    },
+  };
+
+  const targetDigest = digest('a');
+  const otherDigest = digest('b');
+  const sharedLayerDigest = digest('c');
+  const targetOnlyLayerDigest = digest('d');
+  const otherOnlyLayerDigest = digest('e');
+  putJson(storage, 'oci/tags/workloads/ghost-service-1/tags.json', { latest: targetDigest });
+  putJson(storage, 'oci/tags/workloads/other-service/tags.json', { latest: otherDigest });
+  putJson(storage, `oci/manifests/workloads/ghost-service-1/${targetDigest.slice('sha256:'.length)}`, {
+    layers: [{ digest: sharedLayerDigest }, { digest: targetOnlyLayerDigest }],
+  });
+  putJson(storage, `oci/manifests/workloads/other-service/${otherDigest.slice('sha256:'.length)}`, {
+    layers: [{ digest: sharedLayerDigest }, { digest: otherOnlyLayerDigest }],
+  });
+  for (const blobDigest of [sharedLayerDigest, targetOnlyLayerDigest, otherOnlyLayerDigest]) {
+    storage.objects.set(`oci/blobs/sha256/${blobDigest.slice('sha256:'.length)}`, Buffer.from(blobDigest));
+  }
+
+  await manager.deleteServiceRepository(service);
+
+  expect(storage.objects.has('oci/tags/workloads/ghost-service-1/tags.json')).toBeFalse();
+  expect(storage.objects.has(`oci/manifests/workloads/ghost-service-1/${targetDigest.slice('sha256:'.length)}`)).toBeFalse();
+  expect(storage.objects.has(`oci/blobs/sha256/${targetOnlyLayerDigest.slice('sha256:'.length)}`)).toBeFalse();
+  expect(storage.objects.has(`oci/blobs/sha256/${sharedLayerDigest.slice('sha256:'.length)}`)).toBeTrue();
+  expect(storage.objects.has(`oci/blobs/sha256/${otherOnlyLayerDigest.slice('sha256:'.length)}`)).toBeTrue();
+  expect(manager.recordedTagDigests.has('workloads/ghost-service-1:latest')).toBeFalse();
+});
+
+export default tap.start();

test/test.servicecleanup.node.ts

@@ -0,0 +1,137 @@
+import { tap, expect } from '@git.zone/tstest/tapbundle';
+
+import { ServiceManager } from '../ts/manager.service/classes.servicemanager.js';
+
+const createDeleteable = (labelArg: string, callsArg: string[]) => ({
+  id: labelArg,
+  delete: async () => callsArg.push(`delete:${labelArg}`),
+});
+
+const createManager = (optionsArg: {
+  calls: string[];
+  failBackups?: boolean;
+}) => {
+  const calls = optionsArg.calls;
+  const service = {
+    id: 'service-1',
+    data: {
+      name: 'ghost',
+      imageId: 'image-1',
+      appTemplateId: 'ghost',
+      secretBundleId: 'bundle-1',
+      registryTarget: { repository: 'workloads/ghost-service-1' },
+      domains: [{ name: 'ghost.example.com' }],
+    },
+    removeDnsEntries: async () => calls.push('delete:dns'),
+    delete: async () => calls.push('delete:service'),
+  };
+  const manager = Object.create(ServiceManager.prototype) as any;
+  manager.CService = {
+    getInstance: async () => service,
+  };
+  manager.cloudlyRef = {
+    appStoreManager: {
+      clearOperationsForService: (serviceIdArg: string) => calls.push(`clear-upgrades:${serviceIdArg}`),
+    },
+    deploymentManager: {
+      CDeployment: {
+        getInstances: async () => [createDeleteable('deployment-1', calls)],
+      },
+    },
+    platformManager: {
+      CPlatformBinding: {
+        getInstances: async (queryArg: { serviceId: string }) => queryArg.serviceId === 'service-1'
+          ? [createDeleteable('platform-binding-1', calls)]
+          : [],
+      },
+    },
+    backupManager: {
+      deleteBackupsForService: async (serviceIdArg: string) => {
+        calls.push(`delete-backups:${serviceIdArg}`);
+        if (optionsArg.failBackups) {
+          throw new Error('backup cleanup failed');
+        }
+      },
+    },
+    registryManager: {
+      deleteServiceRepository: async () => calls.push('delete:registry-repository'),
+    },
+    secretManager: {
+      CSecretBundle: {
+        getInstance: async () => ({
+          id: 'bundle-1',
+          data: {
+            serviceId: 'service-1',
+            includedSecretGroupIds: ['group-1'],
+          },
+          delete: async () => calls.push('delete:secret-bundle'),
+        }),
+        getInstances: async () => [],
+      },
+      CSecretGroup: {
+        getInstance: async () => createDeleteable('secret-group-1', calls),
+      },
+    },
+    imageManager: {
+      deleteImageIfUnreferenced: async (imageIdArg: string, serviceIdArg: string) => {
+        calls.push(`delete-image:${imageIdArg}:${serviceIdArg}`);
+        return true;
+      },
+    },
+    settingsManager: {
+      getSettings: async () => ({
+        dcrouterGatewayUrl: 'https://dcrouter.example.com',
+        dcrouterGatewayApiToken: 'token',
+        dcrouterWorkHosterId: 'cloudly-main',
+      }),
+    },
+    clusterManager: {
+      getAllClusters: async () => [],
+    },
+    coreflowManager: {
+      pushClusterConfigToConnectedCoreflows: async () => calls.push('push:coreflow'),
+    },
+  };
+  manager.fireDcRouterRequest = async (_url: string, methodArg: string, payloadArg: any) => {
+    calls.push(`${methodArg}:${payloadArg.ownership.hostname}:${payloadArg.ownership.workHosterId}`);
+    return { success: true, action: 'deleted' };
+  };
+  return { manager, service };
+};
+
+tap.test('should delete Cloudly service-owned resources before deleting the service row', async () => {
+  const calls: string[] = [];
+  const { manager } = createManager({ calls });
+
+  await manager.deleteServiceById('service-1');
+
+  expect(calls).toContain('syncWorkAppRoute:ghost.example.com:cloudly-main');
+  expect(calls).toContain('clear-upgrades:service-1');
+  expect(calls).toContain('delete:deployment-1');
+  expect(calls).toContain('delete:dns');
+  expect(calls).toContain('delete:platform-binding-1');
+  expect(calls).toContain('delete-backups:service-1');
+  expect(calls).toContain('delete:registry-repository');
+  expect(calls).toContain('delete:secret-bundle');
+  expect(calls).toContain('delete:secret-group-1');
+  expect(calls).toContain('delete-image:image-1:service-1');
+  expect(calls.at(-2)).toEqual('delete:service');
+  expect(calls.at(-1)).toEqual('push:coreflow');
+});
+
+tap.test('should keep Cloudly service row when required cleanup fails', async () => {
+  const calls: string[] = [];
+  const { manager } = createManager({ calls, failBackups: true });
+
+  let errorMessage = '';
+  try {
+    await manager.deleteServiceById('service-1');
+  } catch (error) {
+    errorMessage = (error as Error).message;
+  }
+  expect(errorMessage).toEqual('backup cleanup failed');
+  expect(calls).not.toContain('delete:service');
+  expect(calls).not.toContain('push:coreflow');
+});
+
+export default tap.start();

ts/manager.appstore/classes.appstoremanager.ts

@@ -294,6 +294,14 @@ export class CloudlyAppStoreManager {
       .slice(0, 25);
   }
 
+  public clearOperationsForService(serviceIdArg: string): void {
+    for (const [operationId, operation] of this.upgradeOperations.entries()) {
+      if (operation.serviceId === serviceIdArg) {
+        this.upgradeOperations.delete(operationId);
+      }
+    }
+  }
+
   public async startHostedAppUpgrade(serviceIdArg: string, targetVersionArg: string): Promise<IAppStoreUpgradeOperation> {
     const operation = await this.createUpgradeOperation(serviceIdArg, targetVersionArg);
     void this.performUpgrade(operation.id).catch(() => {});

ts/manager.backup/classes.backupmanager.ts

@@ -198,6 +198,18 @@ export class CloudlyBackupManager {
     return await backup.createSavableObject();
   }
 
+  public async deleteBackupsForService(serviceIdArg: string): Promise<void> {
+    const backups = await this.CBackupRecord.getInstances({
+      serviceId: serviceIdArg,
+    });
+    for (const backup of backups) {
+      if (backup.replication?.targetPath) {
+        await this.getBackupTargetWriter().deletePrefix(backup.replication.targetPath);
+      }
+      await backup.delete();
+    }
+  }
+
   public async backupAllServices() {
     const services = await this.cloudlyRef.serviceManager.CService.getInstances({});
     const results: Array<{ serviceId: string; backupId?: string; errorText?: string }> = [];

ts/manager.backup/classes.replicationtarget.ts

@@ -12,6 +12,7 @@ export interface IBackupTargetWriter {
   hasObject(pathArg: string, objectArg: TArchiveObject): Promise<boolean>;
   putObject(pathArg: string, objectArg: TArchiveObject, contentsArg: Buffer): Promise<void>;
   readObject(pathArg: string): Promise<Buffer>;
+  deletePrefix(pathPrefixArg: string): Promise<void>;
 }
 
 const requiredEnv = (nameArg: string) => {
@@ -103,6 +104,14 @@ class S3BackupTargetWriter implements IBackupTargetWriter {
     const bucket = await this.getBucket();
     return await bucket.fastGet({ path: normalizeRemotePath(pathArg) });
   }
+
+  public async deletePrefix(pathPrefixArg: string): Promise<void> {
+    const bucket = await this.getBucket();
+    const prefix = normalizeRemotePath(pathPrefixArg).replace(/\/+$/, '');
+    for await (const objectPath of bucket.listAllObjects(prefix)) {
+      await bucket.fastRemove({ path: objectPath });
+    }
+  }
 }
 
 class SmbBackupTargetWriter implements IBackupTargetWriter {
@@ -183,6 +192,24 @@ class SmbBackupTargetWriter implements IBackupTargetWriter {
   public async readObject(pathArg: string) {
     return await (await this.getClient()).readFile(this.getShare(), normalizeRemotePath(pathArg));
   }
+
+  public async deletePrefix(pathPrefixArg: string): Promise<void> {
+    const client = await this.getClient();
+    const share = this.getShare();
+    const rootPath = normalizeRemotePath(pathPrefixArg).replace(/\/+$/, '');
+    const deleteDirectoryFiles = async (pathArg: string): Promise<void> => {
+      const entries = await client.listDirectory(share, pathArg).catch(() => []);
+      for (const entry of entries) {
+        const childPath = `${pathArg}/${entry.name}`.replace(/^\/+/, '');
+        if (entry.isDirectory) {
+          await deleteDirectoryFiles(childPath);
+        } else {
+          await client.deleteFile(share, childPath).catch(() => {});
+        }
+      }
+    };
+    await deleteDirectoryFiles(rootPath);
+  }
 }
 
 export const createBackupTargetWriterFromEnv = (): IBackupTargetWriter => {

ts/manager.image/classes.imagemanager.ts

@@ -212,4 +212,35 @@ export class ImageManager {
       bucketDir: this.imageDir,
     });
   }
+
+  public async deleteImageIfUnreferenced(imageIdArg: string, ownerServiceIdArg?: string): Promise<boolean> {
+    if (!imageIdArg) return false;
+
+    const referencingServices = await this.cloudlyRef.serviceManager.CService.getInstances({});
+    const referencedByOtherService = referencingServices.some((serviceArg) => {
+      return serviceArg.id !== ownerServiceIdArg && serviceArg.data?.imageId === imageIdArg;
+    });
+    if (referencedByOtherService) return false;
+
+    const image = await this.CImage.getInstance({
+      id: imageIdArg,
+    }).catch(() => null);
+    if (!image) return false;
+
+    for (const version of image.data.versions || []) {
+      const storagePath = version.storagePath || await image.getStoragePath(version.versionString);
+      if (!storagePath || !this.imageDir) continue;
+      await this.imageDir.fastRemove({
+        path: `${storagePath}.tar`,
+      }).catch((errorArg) => {
+        const message = (errorArg as Error).message.toLowerCase();
+        if (!message.includes('not found') && !message.includes('no such')) {
+          throw errorArg;
+        }
+      });
+    }
+
+    await image.delete();
+    return true;
+  }
 }

ts/manager.registry/classes.registrymanager.ts

@@ -9,6 +9,18 @@ type TAuthenticatedRegistryUser = {
   canWrite: boolean;
 };
 
+type TOciTags = Record<string, string>;
+
+interface IOciDescriptor {
+  digest?: unknown;
+}
+
+interface IOciManifestDocument {
+  config?: IOciDescriptor;
+  layers?: IOciDescriptor[];
+  manifests?: IOciDescriptor[];
+}
+
 export class CloudlyRegistryManager {
   private cloudlyRef: Cloudly;
   private smartRegistry!: plugins.smartregistry.SmartRegistry;
@@ -123,6 +135,214 @@ export class CloudlyRegistryManager {
     };
   }
 
+  public async deleteServiceRepository(serviceArg: Service): Promise<void> {
+    const repository = serviceArg.data.registryTarget?.repository;
+    if (!repository) return;
+
+    const services = await this.cloudlyRef.serviceManager.CService.getInstances({});
+    const referencedByOtherService = services.some((candidateArg) => {
+      return candidateArg.id !== serviceArg.id && candidateArg.data.registryTarget?.repository === repository;
+    });
+    if (referencedByOtherService) return;
+
+    await this.deleteOciRepository(repository);
+    for (const tagKey of Array.from(this.recordedTagDigests.keys())) {
+      if (tagKey.startsWith(`${repository}:`)) {
+        this.recordedTagDigests.delete(tagKey);
+      }
+    }
+  }
+
+  private getRegistryStorage(): any {
+    if (!this.started || !this.smartRegistry) {
+      throw new Error('Cloudly registry is not started');
+    }
+    return this.smartRegistry.getStorage();
+  }
+
+  private getOciTagsPath(repositoryArg: string): string {
+    return `oci/tags/${repositoryArg}/tags.json`;
+  }
+
+  private normalizeOciDigest(digestArg: string | null | undefined): string | null {
+    if (typeof digestArg !== 'string') return null;
+    const normalizedDigest = digestArg.trim().toLowerCase();
+    return /^sha256:[a-f0-9]{64}$/.test(normalizedDigest) ? normalizedDigest : null;
+  }
+
+  private getSha256HashFromDigest(digestArg: string): string {
+    const normalizedDigest = this.normalizeOciDigest(digestArg);
+    if (!normalizedDigest) {
+      throw new Error(`Invalid OCI digest: ${digestArg}`);
+    }
+    return normalizedDigest.slice('sha256:'.length);
+  }
+
+  private getOciManifestPath(repositoryArg: string, digestArg: string): string {
+    return `oci/manifests/${repositoryArg}/${this.getSha256HashFromDigest(digestArg)}`;
+  }
+
+  private getOciBlobPath(digestArg: string): string {
+    return `oci/blobs/sha256/${this.getSha256HashFromDigest(digestArg)}`;
+  }
+
+  private async readOciTags(repositoryArg: string, storageArg = this.getRegistryStorage()): Promise<TOciTags> {
+    const tagsBuffer = await storageArg.getObject(this.getOciTagsPath(repositoryArg));
+    if (!tagsBuffer) return {};
+
+    const parsedTags = JSON.parse(tagsBuffer.toString('utf8'));
+    if (!parsedTags || typeof parsedTags !== 'object' || Array.isArray(parsedTags)) {
+      throw new Error(`Invalid OCI tags document for ${repositoryArg}`);
+    }
+
+    const tags: TOciTags = {};
+    for (const [tagName, digestValue] of Object.entries(parsedTags)) {
+      const digest = typeof digestValue === 'string' ? this.normalizeOciDigest(digestValue) : null;
+      if (!digest) {
+        throw new Error(`Invalid OCI digest for ${repositoryArg}:${tagName}`);
+      }
+      tags[tagName] = digest;
+    }
+    return tags;
+  }
+
+  private async readOciManifest(
+    storageArg: any,
+    repositoryArg: string,
+    digestArg: string,
+  ): Promise<IOciManifestDocument | null> {
+    const manifestBuffer = await storageArg.getOciManifest(repositoryArg, digestArg);
+    if (!manifestBuffer) return null;
+    try {
+      const manifest = JSON.parse(manifestBuffer.toString('utf8'));
+      return manifest && typeof manifest === 'object' ? manifest : null;
+    } catch (error) {
+      logger.log('warn', `failed to parse OCI manifest ${repositoryArg}@${digestArg}: ${(error as Error).message}`);
+      return null;
+    }
+  }
+
+  private getDescriptorDigest(descriptorArg: IOciDescriptor | undefined): string | null {
+    return typeof descriptorArg?.digest === 'string' ? this.normalizeOciDigest(descriptorArg.digest) : null;
+  }
+
+  private collectOciManifestReferences(manifestArg: IOciManifestDocument): {
+    blobDigests: string[];
+    manifestDigests: string[];
+  } {
+    const blobDigests = [
+      this.getDescriptorDigest(manifestArg.config),
+      ...(manifestArg.layers || []).map((descriptorArg) => this.getDescriptorDigest(descriptorArg)),
+    ].filter((digestArg): digestArg is string => Boolean(digestArg));
+
+    const manifestDigests = (manifestArg.manifests || [])
+      .map((descriptorArg) => this.getDescriptorDigest(descriptorArg))
+      .filter((digestArg): digestArg is string => Boolean(digestArg));
+
+    return { blobDigests, manifestDigests };
+  }
+
+  private async collectReferencedOciObjects(
+    storageArg: any,
+    repositoryArg: string,
+    rootDigestsArg: string[],
+  ): Promise<{ manifestDigests: Set<string>; blobDigests: Set<string> }> {
+    const manifestDigests = new Set<string>();
+    const blobDigests = new Set<string>();
+    const pendingManifestDigests = rootDigestsArg
+      .map((digestArg) => this.normalizeOciDigest(digestArg))
+      .filter((digestArg): digestArg is string => Boolean(digestArg));
+
+    while (pendingManifestDigests.length > 0) {
+      const manifestDigest = pendingManifestDigests.shift()!;
+      if (manifestDigests.has(manifestDigest)) continue;
+      manifestDigests.add(manifestDigest);
+
+      const manifest = await this.readOciManifest(storageArg, repositoryArg, manifestDigest);
+      if (!manifest) continue;
+      const references = this.collectOciManifestReferences(manifest);
+      for (const blobDigest of references.blobDigests) {
+        blobDigests.add(blobDigest);
+      }
+      for (const childManifestDigest of references.manifestDigests) {
+        if (!manifestDigests.has(childManifestDigest)) {
+          pendingManifestDigests.push(childManifestDigest);
+        }
+      }
+    }
+
+    return { manifestDigests, blobDigests };
+  }
+
+  private async listRepositoryManifestDigests(storageArg: any, repositoryArg: string): Promise<string[]> {
+    const manifestPrefix = `oci/manifests/${repositoryArg}/`;
+    const paths = await storageArg.listObjects(manifestPrefix);
+    return paths
+      .filter((pathArg: string) => !pathArg.endsWith('.type'))
+      .map((pathArg: string) => pathArg.slice(manifestPrefix.length))
+      .filter((hashArg: string) => /^[a-f0-9]{64}$/.test(hashArg))
+      .map((hashArg: string) => `sha256:${hashArg}`);
+  }
+
+  private async collectAllTaggedOciObjectsExceptRepository(storageArg: any, repositoryArg: string) {
+    const protectedObjects = {
+      manifestDigests: new Set<string>(),
+      blobDigests: new Set<string>(),
+    };
+    const tagPaths = await storageArg.listObjects('oci/tags/');
+    for (const tagPath of tagPaths) {
+      const match = tagPath.match(/^oci\/tags\/(.+)\/tags\.json$/);
+      if (!match || match[1] === repositoryArg) continue;
+      const tags = await this.readOciTags(match[1], storageArg);
+      const references = await this.collectReferencedOciObjects(storageArg, match[1], Object.values(tags));
+      for (const digest of references.manifestDigests) {
+        protectedObjects.manifestDigests.add(digest);
+      }
+      for (const digest of references.blobDigests) {
+        protectedObjects.blobDigests.add(digest);
+      }
+    }
+    return protectedObjects;
+  }
+
+  private async deleteObjectIfExists(storageArg: any, pathArg: string): Promise<void> {
+    if (typeof storageArg.objectExists === 'function' && !(await storageArg.objectExists(pathArg))) {
+      return;
+    }
+    await storageArg.deleteObject(pathArg);
+  }
+
+  private async deleteOciRepository(repositoryArg: string): Promise<void> {
+    const storage = this.getRegistryStorage();
+    const tags = await this.readOciTags(repositoryArg, storage);
+    const repositoryManifestDigests = await this.listRepositoryManifestDigests(storage, repositoryArg);
+    const rootDigests = Array.from(new Set([...Object.values(tags), ...repositoryManifestDigests]));
+    if (rootDigests.length === 0) {
+      await this.deleteObjectIfExists(storage, this.getOciTagsPath(repositoryArg));
+      return;
+    }
+
+    const targetObjects = await this.collectReferencedOciObjects(storage, repositoryArg, rootDigests);
+    const protectedObjects = await this.collectAllTaggedOciObjectsExceptRepository(storage, repositoryArg);
+
+    for (const blobDigest of targetObjects.blobDigests) {
+      if (!protectedObjects.blobDigests.has(blobDigest)) {
+        await this.deleteObjectIfExists(storage, this.getOciBlobPath(blobDigest));
+      }
+    }
+
+    for (const manifestDigest of targetObjects.manifestDigests) {
+      if (!protectedObjects.manifestDigests.has(manifestDigest)) {
+        const manifestPath = this.getOciManifestPath(repositoryArg, manifestDigest);
+        await this.deleteObjectIfExists(storage, manifestPath);
+        await this.deleteObjectIfExists(storage, `${manifestPath}.type`);
+      }
+    }
+
+    await this.deleteObjectIfExists(storage, this.getOciTagsPath(repositoryArg));
+    logger.log('info', `deleted Cloudly registry repository ${repositoryArg}`);
+  }
+
   private async handleRegistryStorageAfterPut(
     contextArg: plugins.smartregistry.IStorageHookContext,
   ) {

ts/manager.service/classes.servicemanager.ts

@@ -2,6 +2,20 @@ import type { Cloudly } from '../classes.cloudly.js';
 import * as plugins from '../plugins.js';
 import { Service } from './classes.service.js';
 
+type TServiceWithDomains = Service & {
+  data: Service['data'] & {
+    appTemplateId?: string;
+    domains?: Array<{ name?: string }>;
+  };
+};
+
+interface IWorkAppRouteSyncResult {
+  success: boolean;
+  action?: 'created' | 'updated' | 'deleted' | 'unchanged';
+  routeId?: string;
+  message?: string;
+}
+
 export class ServiceManager {
   public typedrouter = new plugins.typedrequest.TypedRouter();
   public cloudlyRef: Cloudly;
@@ -145,14 +159,7 @@ export class ServiceManager {
           await plugins.smartguard.passGuardsOrReject(dataArg, [
             this.cloudlyRef.authManager.adminIdentityGuard,
           ]);
-          const service = await Service.getInstance({
+          await this.deleteServiceById(dataArg.serviceId);
-            id: dataArg.serviceId,
-          });
-          
-          // Remove DNS entries before deleting the service
-          await service.removeDnsEntries();
-          
-          await service.delete();
           return {
             success: true,
           };
@@ -170,4 +177,149 @@ export class ServiceManager {
     // Cleanup if needed
     console.log('ServiceManager stopped');
   }
+
+  public async deleteServiceById(serviceIdArg: string): Promise<void> {
+    const service = await this.CService.getInstance({
+      id: serviceIdArg,
+    });
+    if (!service) {
+      throw new plugins.typedrequest.TypedResponseError(`Service not found: ${serviceIdArg}`);
+    }
+
+    await this.deleteExternalGatewayRoutes(service as TServiceWithDomains);
+    this.cloudlyRef.appStoreManager?.clearOperationsForService?.(service.id);
+    await this.deleteDeploymentsForService(service.id);
+    await service.removeDnsEntries();
+    await this.deletePlatformBindingsForService(service.id, service.data.name);
+    await this.cloudlyRef.backupManager?.deleteBackupsForService?.(service.id);
+    await this.cloudlyRef.registryManager?.deleteServiceRepository?.(service);
+    await this.deleteServiceOwnedSecretBundles(service);
+    await this.deleteServiceOwnedImage(service as TServiceWithDomains);
+    await service.delete();
+    await this.cloudlyRef.coreflowManager.pushClusterConfigToConnectedCoreflows();
+  }
+
+  private async deleteDeploymentsForService(serviceIdArg: string): Promise<void> {
+    const deployments = await this.cloudlyRef.deploymentManager.CDeployment.getInstances({
+      serviceId: serviceIdArg,
+    });
+    for (const deployment of deployments) {
+      await deployment.delete();
+    }
+  }
+
+  private async deletePlatformBindingsForService(
+    serviceIdArg: string,
+    serviceNameArg: string,
+  ): Promise<void> {
+    const bindingsById = await this.cloudlyRef.platformManager.CPlatformBinding.getInstances({
+      serviceId: serviceIdArg,
+    });
+    const bindingsByName = serviceNameArg
+      ? await this.cloudlyRef.platformManager.CPlatformBinding.getInstances({ serviceId: serviceNameArg })
+      : [];
+    const bindings = new Map<string, typeof bindingsById[number]>();
+    for (const binding of [...bindingsById, ...bindingsByName]) {
+      bindings.set(binding.id, binding);
+    }
+    for (const binding of bindings.values()) {
+      await binding.delete();
+    }
+  }
+
+  private async deleteServiceOwnedSecretBundles(serviceArg: Service): Promise<void> {
+    const secretBundleIds = [serviceArg.data.secretBundleId]
+      .filter((secretBundleIdArg): secretBundleIdArg is string => Boolean(secretBundleIdArg));
+    if (secretBundleIds.length === 0) return;
+
+    for (const secretBundleId of secretBundleIds) {
+      const secretBundle = await this.cloudlyRef.secretManager.CSecretBundle.getInstance({
+        id: secretBundleId,
+      }).catch(() => null);
+      if (!secretBundle || (secretBundle.data as { serviceId?: string }).serviceId !== serviceArg.id) {
+        continue;
+      }
+
+      const secretGroupIds = [...(secretBundle.data.includedSecretGroupIds || [])];
+      await secretBundle.delete();
+      await this.deleteUnreferencedSecretGroups(secretGroupIds);
+    }
+  }
+
+  private async deleteUnreferencedSecretGroups(secretGroupIdsArg: string[]): Promise<void> {
+    if (secretGroupIdsArg.length === 0) return;
+    const remainingBundles = await this.cloudlyRef.secretManager.CSecretBundle.getInstances({});
+    for (const secretGroupId of secretGroupIdsArg) {
+      const stillReferenced = remainingBundles.some((bundleArg) => {
+        return (bundleArg.data.includedSecretGroupIds || []).includes(secretGroupId);
+      });
+      if (stillReferenced) continue;
+      const secretGroup = await this.cloudlyRef.secretManager.CSecretGroup.getInstance({
+        id: secretGroupId,
+      }).catch(() => null);
+      if (secretGroup) {
+        await secretGroup.delete();
+      }
+    }
+  }
+
+  private async deleteServiceOwnedImage(serviceArg: TServiceWithDomains): Promise<void> {
+    if (!serviceArg.data.appTemplateId || !serviceArg.data.imageId) return;
+    await this.cloudlyRef.imageManager.deleteImageIfUnreferenced(serviceArg.data.imageId, serviceArg.id);
+  }
+
+  private async deleteExternalGatewayRoutes(serviceArg: TServiceWithDomains): Promise<void> {
+    const domains = (serviceArg.data.domains || [])
+      .map((domainArg) => domainArg.name?.trim().toLowerCase())
+      .filter((domainArg): domainArg is string => Boolean(domainArg));
+    if (domains.length === 0) return;
+
+    const settings = await this.cloudlyRef.settingsManager.getSettings().catch(() => undefined);
+    if (!settings?.dcrouterGatewayUrl || !settings.dcrouterGatewayApiToken) return;
+
+    const clusters = await this.cloudlyRef.clusterManager.getAllClusters().catch(() => []);
+    const workHosterIds = new Set<string>();
+    if (settings.dcrouterWorkHosterId) {
+      workHosterIds.add(settings.dcrouterWorkHosterId);
+    } else {
+      for (const cluster of clusters) {
+        workHosterIds.add(cluster.id);
+      }
+    }
+    if (workHosterIds.size === 0) return;
+
+    for (const domain of domains) {
+      for (const workHosterId of workHosterIds) {
+        const result = await this.fireDcRouterRequest<IWorkAppRouteSyncResult>(
+          settings.dcrouterGatewayUrl,
+          'syncWorkAppRoute',
+          {
+            apiToken: settings.dcrouterGatewayApiToken,
+            ownership: {
+              workHosterType: 'cloudly',
+              workHosterId,
+              workAppId: serviceArg.id || serviceArg.data.name,
+              hostname: domain,
+            },
+            delete: true,
+          },
+        );
+        if (!result.success) {
+          throw new Error(result.message || `dcrouter route delete failed for ${domain}`);
+        }
+      }
+    }
+  }
+
+  private async fireDcRouterRequest<TResponse>(
+    gatewayUrlArg: string,
+    methodArg: string,
+    requestDataArg: Record<string, unknown>,
+  ): Promise<TResponse> {
+    const typedRequest = new plugins.typedrequest.TypedRequest<any>(
+      `${gatewayUrlArg.replace(/\/+$/, '')}/typedrequest`,
+      methodArg,
+    );
+    return await typedRequest.fire(requestDataArg) as TResponse;
+  }
 }