cloudly/ts/manager.baseos/classes.baseosmanager.ts

import * as plugins from '../plugins.js';
import * as crypto from 'node:crypto';
import * as nodeStream from 'node:stream';
import type { Cloudly } from '../classes.cloudly.js';
import { logger } from '../logger.js';
import {
  BaseOsNode,
  type IBaseOsDesiredState,
  type IBaseOsNodePublic,
  type IBaseOsRuntimeInfo,
} from './classes.baseosnode.js';
import {
  BaseOsImageBuild,
  type IBaseOsImageArtifact,
  type IBaseOsImageBuildPublic,
  type TBaseOsImageArchitecture,
  type TBaseOsImageKind,
  type TBaseOsImageSourcePreset,
} from './classes.baseosimagebuild.js';

interface IBalenaSourcePreset {
  preset: TBaseOsImageSourcePreset;
  architecture: TBaseOsImageArchitecture;
}

const balenaSourcePresets: IBalenaSourcePreset[] = [
  {
    preset: 'balena-generic-amd64',
    architecture: 'amd64',
  },
  {
    preset: 'balena-generic-aarch64',
    architecture: 'arm64',
  },
  {
    preset: 'balena-raspberrypi4-64',
    architecture: 'rpi',
  },
];

interface IBaseOsRegisterRequest {
  joinToken?: string;
  nodeToken?: string;
  status?: IBaseOsRuntimeInfo;
}

interface IBaseOsRegisterResponse {
  nodeId?: string;
  nodeToken?: string;
  accepted: boolean;
  message?: string;
  desiredState?: IBaseOsDesiredState;
}

interface IBaseOsHeartbeatRequest {
  nodeToken?: string;
  status?: IBaseOsRuntimeInfo;
}

interface IBaseOsHeartbeatResponse {
  accepted: boolean;
  message?: string;
  desiredState?: IBaseOsDesiredState;
}

interface IRequestGetBaseOsNodes {
  method: 'getBaseOsNodes';
  request: {
    identity: plugins.servezoneInterfaces.data.IIdentity;
  };
  response: {
    nodes: IBaseOsNodePublic[];
  };
}

interface IRequestSetBaseOsNodeDesiredState {
  method: 'setBaseOsNodeDesiredState';
  request: {
    identity: plugins.servezoneInterfaces.data.IIdentity;
    nodeId: string;
    desiredState: IBaseOsDesiredState;
  };
  response: {
    node: IBaseOsNodePublic;
  };
}

interface IBaseOsImageBuildRequest {
  architecture: TBaseOsImageArchitecture;
  imageKind?: TBaseOsImageKind;
  cloudlyUrl?: string;
  sourceImageUrl?: string;
  sourceImagePreset?: TBaseOsImageSourcePreset;
  balenaOsVersion?: string;
  ubuntuVersion?: string;
  hostname?: string;
  wifi?: {
    ssid: string;
    password?: string;
  };
  sshPublicKey?: string;
  artifactRetentionMs?: number;
}

interface IRequestCreateBaseOsImageBuild {
  method: 'createBaseOsImageBuild';
  request: {
    identity: plugins.servezoneInterfaces.data.IIdentity;
    build: IBaseOsImageBuildRequest;
  };
  response: {
    build: IBaseOsImageBuildPublic;
  };
}

interface IRequestGetBaseOsImageBuilds {
  method: 'getBaseOsImageBuilds';
  request: {
    identity: plugins.servezoneInterfaces.data.IIdentity;
  };
  response: {
    builds: IBaseOsImageBuildPublic[];
  };
}

interface IRequestGetBaseOsImageBuildById {
  method: 'getBaseOsImageBuildById';
  request: {
    identity: plugins.servezoneInterfaces.data.IIdentity;
    buildId: string;
  };
  response: {
    build: IBaseOsImageBuildPublic;
  };
}

interface IRequestCreateBaseOsImageDownloadUrl {
  method: 'createBaseOsImageDownloadUrl';
  request: {
    identity: plugins.servezoneInterfaces.data.IIdentity;
    buildId: string;
  };
  response: {
    url: string;
    expiresAt: number;
  };
}

interface ICoreBuildBaseOsImageResponse {
  success: boolean;
  artifact?: IBaseOsImageArtifact;
  logs: string[];
  errorText?: string;
}

interface ICoreBuildCapabilitiesResponse {
  workerId: string;
  supportedBuildTypes: string[];
  supportedArchitectures: TBaseOsImageArchitecture[];
  supportedImageKinds?: TBaseOsImageKind[];
  supportedSourcePresets?: TBaseOsImageSourcePreset[];
}

interface ICoreBuildWorkerSetting {
  id?: string;
  url: string;
  token?: string;
}

interface ISelectedCoreBuildWorker extends ICoreBuildWorkerSetting {
  capabilities: ICoreBuildCapabilitiesResponse;
}

export class CloudlyBaseOsManager {
  public cloudlyRef: Cloudly;
  public typedRouter = new plugins.typedrequest.TypedRouter();
  private cleanupInterval?: ReturnType<typeof setInterval>;

  public get db() {
    return this.cloudlyRef.mongodbConnector.smartdataDb;
  }

  public CBaseOsNode = plugins.smartdata.setDefaultManagerForDoc(this, BaseOsNode);
  public CBaseOsImageBuild = plugins.smartdata.setDefaultManagerForDoc(this, BaseOsImageBuild);

  constructor(cloudlyRefArg: Cloudly) {
    this.cloudlyRef = cloudlyRefArg;
    this.cloudlyRef.typedrouter.addTypedRouter(this.typedRouter);

    this.typedRouter.addTypedHandler(
      new plugins.typedrequest.TypedHandler<IRequestGetBaseOsNodes>(
        'getBaseOsNodes',
        async (requestDataArg) => {
          await plugins.smartguard.passGuardsOrReject(
            { identity: requestDataArg.identity },
            [this.cloudlyRef.authManager.adminIdentityGuard],
          );
          return {
            nodes: await this.getPublicNodes(),
          };
        },
      ),
    );

    this.typedRouter.addTypedHandler(
      new plugins.typedrequest.TypedHandler<IRequestCreateBaseOsImageBuild>(
        'createBaseOsImageBuild',
        async (requestDataArg) => {
          await plugins.smartguard.passGuardsOrReject(
            { identity: requestDataArg.identity },
            [this.cloudlyRef.authManager.adminIdentityGuard],
          );
          return {
            build: await this.createImageBuild(requestDataArg.build),
          };
        },
      ),
    );

    this.typedRouter.addTypedHandler(
      new plugins.typedrequest.TypedHandler<IRequestSetBaseOsNodeDesiredState>(
        'setBaseOsNodeDesiredState',
        async (requestDataArg) => {
          await plugins.smartguard.passGuardsOrReject(
            { identity: requestDataArg.identity },
            [this.cloudlyRef.authManager.adminIdentityGuard],
          );
          return {
            node: (await this.setNodeDesiredState(
              requestDataArg.nodeId,
              requestDataArg.desiredState,
            )).toPublicNode(),
          };
        },
      ),
    );

    this.typedRouter.addTypedHandler(
      new plugins.typedrequest.TypedHandler<IRequestGetBaseOsImageBuilds>(
        'getBaseOsImageBuilds',
        async (requestDataArg) => {
          await plugins.smartguard.passGuardsOrReject(
            { identity: requestDataArg.identity },
            [this.cloudlyRef.authManager.adminIdentityGuard],
          );
          return {
            builds: await this.getPublicImageBuilds(),
          };
        },
      ),
    );

    this.typedRouter.addTypedHandler(
      new plugins.typedrequest.TypedHandler<IRequestGetBaseOsImageBuildById>(
        'getBaseOsImageBuildById',
        async (requestDataArg) => {
          await plugins.smartguard.passGuardsOrReject(
            { identity: requestDataArg.identity },
            [this.cloudlyRef.authManager.adminIdentityGuard],
          );
          return {
            build: (await this.getImageBuildById(requestDataArg.buildId)).toPublicBuild(),
          };
        },
      ),
    );

    this.typedRouter.addTypedHandler(
      new plugins.typedrequest.TypedHandler<IRequestCreateBaseOsImageDownloadUrl>(
        'createBaseOsImageDownloadUrl',
        async (requestDataArg) => {
          await plugins.smartguard.passGuardsOrReject(
            { identity: requestDataArg.identity },
            [this.cloudlyRef.authManager.adminIdentityGuard],
          );
          return await this.createImageDownloadUrl(requestDataArg.buildId);
        },
      ),
    );
  }

  public async start() {
    await this.cleanupExpiredImageBuilds().catch((error) => {
      logger.log('error', `BaseOS image cleanup failed: ${(error as Error).message}`);
    });
    const cleanupIntervalMs = Number(
      process.env.CLOUDLY_BASEOS_IMAGE_CLEANUP_INTERVAL_MS || 1000 * 60 * 60 * 12,
    );
    if (Number.isFinite(cleanupIntervalMs) && cleanupIntervalMs > 0) {
      this.cleanupInterval = setInterval(() => {
        this.cleanupExpiredImageBuilds().catch((error) => {
          logger.log('error', `BaseOS image cleanup failed: ${(error as Error).message}`);
        });
      }, cleanupIntervalMs);
    }
    logger.log('info', 'BaseOS manager started');
  }

  public async stop() {
    if (this.cleanupInterval) {
      clearInterval(this.cleanupInterval);
      this.cleanupInterval = undefined;
    }
    logger.log('info', 'BaseOS manager stopped');
  }

  public async handleRegisterHttpRequest(
    ctxArg: plugins.typedserver.IRequestContext,
  ): Promise<Response> {
    try {
      const requestData = await this.readJsonBody<IBaseOsRegisterRequest>(ctxArg);
      const response = await this.registerNode(requestData);
      return this.createJsonResponse(200, response);
    } catch (error) {
      return this.createJsonResponse(400, {
        accepted: false,
        message: `BaseOS registration failed: ${(error as Error).message}`,
      } satisfies IBaseOsRegisterResponse);
    }
  }

  public async handleHeartbeatHttpRequest(
    ctxArg: plugins.typedserver.IRequestContext,
  ): Promise<Response> {
    try {
      const requestData = await this.readJsonBody<IBaseOsHeartbeatRequest>(ctxArg);
      const response = await this.acceptHeartbeat(requestData);
      return this.createJsonResponse(200, response);
    } catch (error) {
      return this.createJsonResponse(400, {
        accepted: false,
        message: `BaseOS heartbeat failed: ${(error as Error).message}`,
      } satisfies IBaseOsHeartbeatResponse);
    }
  }

  public async handleImageDownloadHttpRequest(
    ctxArg: plugins.typedserver.IRequestContext,
  ): Promise<Response> {
    try {
      const buildId = ctxArg.params.buildId || ctxArg.url.pathname.split('/').at(-2);
      const token = ctxArg.url.searchParams.get('token');
      if (!buildId || !token) {
        return this.createJsonResponse(400, { errorText: 'build id or download token missing' });
      }
      const build = await this.getImageBuildById(buildId);
      if (build.downloadTokenHash !== this.hashSecret(token) || (build.downloadTokenExpiresAt || 0) < Date.now()) {
        return this.createJsonResponse(403, { errorText: 'download token is invalid or expired' });
      }
      if (build.data.status !== 'ready' || !build.data.artifact) {
        return this.createJsonResponse(409, { errorText: 'image build is not ready' });
      }

      const artifact = build.data.artifact;
      const bucket = await this.getArtifactBucket(artifact.bucketName);
      const artifactStream = await bucket.fastGetStream({ path: artifact.key }, 'nodestream');
      return new Response(nodeStream.Readable.toWeb(artifactStream as nodeStream.Readable) as ReadableStream, {
        status: 200,
        headers: {
          'Content-Type': artifact.contentType || 'application/octet-stream',
          'Content-Length': String(artifact.size),
          'Content-Disposition': `attachment; filename="${artifact.filename}"`,
        },
      });
    } catch (error) {
      return this.createJsonResponse(500, {
        errorText: `BaseOS image download failed: ${(error as Error).message}`,
      });
    }
  }

  public async registerNode(
    requestDataArg: IBaseOsRegisterRequest,
  ): Promise<IBaseOsRegisterResponse> {
    if (!this.isRuntimeInfo(requestDataArg.status)) {
      return {
        accepted: false,
        message: 'BaseOS runtime status is missing or invalid',
      };
    }

    if (requestDataArg.nodeToken) {
      const existingNode = await this.getNodeByToken(requestDataArg.nodeToken);
      if (existingNode) {
        await this.updateNodeRuntimeInfo(existingNode, requestDataArg.status, true);
        return {
          accepted: true,
          nodeId: existingNode.id,
          desiredState: existingNode.data.desiredState || {},
        };
      }
    }

    const acceptedBuild = await this.consumeProvisioningToken(requestDataArg.joinToken);
    if (acceptedBuild) {
      const nodeToken = await this.cloudlyRef.authManager.createNewSecureToken();
      const node = await this.upsertNode(requestDataArg.status, nodeToken);
      return {
        accepted: true,
        nodeId: node.id,
        nodeToken,
        desiredState: node.data.desiredState || {},
      };
    }

    const configuredJoinToken = await this.cloudlyRef.settingsManager.getSetting('baseosJoinToken');
    if (!configuredJoinToken) {
      return {
        accepted: false,
        message: 'BaseOS join token is not configured in Cloudly settings',
      };
    }

    if (!requestDataArg.joinToken || requestDataArg.joinToken !== configuredJoinToken) {
      return {
        accepted: false,
        message: 'BaseOS join token is invalid',
      };
    }

    const nodeToken = await this.cloudlyRef.authManager.createNewSecureToken();
    const node = await this.upsertNode(requestDataArg.status, nodeToken);
    return {
      accepted: true,
      nodeId: node.id,
      nodeToken,
      desiredState: node.data.desiredState || {},
    };
  }

  public async acceptHeartbeat(
    requestDataArg: IBaseOsHeartbeatRequest,
  ): Promise<IBaseOsHeartbeatResponse> {
    if (!requestDataArg.nodeToken) {
      return {
        accepted: false,
        message: 'BaseOS node token is missing',
      };
    }
    if (!this.isRuntimeInfo(requestDataArg.status)) {
      return {
        accepted: false,
        message: 'BaseOS runtime status is missing or invalid',
      };
    }

    const node = await this.getNodeByToken(requestDataArg.nodeToken);
    if (!node) {
      return {
        accepted: false,
        message: 'BaseOS node token is invalid',
      };
    }

    await this.updateNodeRuntimeInfo(node, requestDataArg.status, true);
    return {
      accepted: true,
      desiredState: node.data.desiredState || {},
    };
  }

  public async getPublicNodes(): Promise<IBaseOsNodePublic[]> {
    const nodes = await this.CBaseOsNode.getInstances({});
    return nodes.map((nodeArg) => nodeArg.toPublicNode());
  }

  public async setNodeDesiredState(nodeIdArg: string, desiredStateArg: IBaseOsDesiredState) {
    const node = await this.CBaseOsNode.getInstance({ id: nodeIdArg });
    if (!node) {
      throw new plugins.typedrequest.TypedResponseError(`BaseOS node ${nodeIdArg} not found`);
    }
    node.data = {
      ...node.data,
      desiredState: {
        ...desiredStateArg,
        updatedAt: desiredStateArg.updatedAt || Date.now(),
      },
      updatedAt: Date.now(),
    };
    await node.save();
    return node;
  }

  public async createImageBuild(buildRequestArg: IBaseOsImageBuildRequest) {
    const s3Descriptor = this.cloudlyRef.config.data.s3Descriptor;
    if (!s3Descriptor?.bucketName) {
      throw new plugins.typedrequest.TypedResponseError('Cloudly S3 storage is required for BaseOS image builds');
    }

    const imageKind = this.getImageKind(buildRequestArg);
    if (buildRequestArg.architecture === 'rpi' && imageKind === 'ubuntu-iso') {
      throw new plugins.typedrequest.TypedResponseError('Raspberry Pi BaseOS images require balena-raw image builds');
    }
    const sourceImagePreset = this.getSourceImagePreset(buildRequestArg, imageKind);
    const balenaOsVersion = imageKind === 'balena-raw' && !buildRequestArg.sourceImageUrl
      ? buildRequestArg.balenaOsVersion?.trim() || 'latest'
      : undefined;
    const worker = await this.selectCoreBuildWorker(buildRequestArg.architecture, imageKind, sourceImagePreset);

    const now = Date.now();
    const buildId = await this.CBaseOsImageBuild.getNewId();
    const provisioningToken = await this.cloudlyRef.authManager.createNewSecureToken();
    const artifactRetentionMs = buildRequestArg.artifactRetentionMs || 1000 * 60 * 60 * 24 * 7;
    const build = new this.CBaseOsImageBuild({
      id: buildId,
      provisioningTokenHash: this.hashSecret(provisioningToken),
      data: {
        status: 'queued',
        architecture: buildRequestArg.architecture,
        imageKind,
        cloudlyUrl: buildRequestArg.cloudlyUrl || this.getPublicCloudlyUrl(),
        sourceImageUrl: buildRequestArg.sourceImageUrl,
        sourceImagePreset,
        balenaOsVersion,
        ubuntuVersion: buildRequestArg.ubuntuVersion || '24.04',
        hostname: buildRequestArg.hostname,
        wifiSsid: buildRequestArg.wifi?.ssid,
        sshPublicKey: buildRequestArg.sshPublicKey,
        logs: [],
        createdAt: now,
        updatedAt: now,
        expiresAt: now + artifactRetentionMs,
      },
    });
    await build.save();

    this.executeImageBuild(build, provisioningToken, buildRequestArg, worker).catch(async (error) => {
      build.data.status = 'failed';
      build.data.errorText = (error as Error).message;
      build.data.updatedAt = Date.now();
      build.data.completedAt = Date.now();
      await build.save();
    });

    return build.toPublicBuild();
  }

  public async getPublicImageBuilds() {
    const builds = await this.CBaseOsImageBuild.getInstances({});
    return builds
      .sort((a, b) => b.data.createdAt - a.data.createdAt)
      .map((buildArg) => buildArg.toPublicBuild());
  }

  public async createImageDownloadUrl(buildIdArg: string) {
    const build = await this.getImageBuildById(buildIdArg);
    if (build.data.status !== 'ready' || !build.data.artifact) {
      throw new plugins.typedrequest.TypedResponseError('BaseOS image build is not ready');
    }
    const token = await this.cloudlyRef.authManager.createNewSecureToken();
    const expiresAt = Date.now() + 1000 * 60 * 15;
    build.downloadTokenHash = this.hashSecret(token);
    build.downloadTokenExpiresAt = expiresAt;
    build.data.updatedAt = Date.now();
    await build.save();
    return {
      url: `/baseos/v1/images/${build.id}/download?token=${encodeURIComponent(token)}`,
      expiresAt,
    };
  }

  public async cleanupExpiredImageBuilds() {
    const now = Date.now();
    const builds = await this.CBaseOsImageBuild.getInstances({});
    let deletedBuilds = 0;
    let deletedArtifacts = 0;
    let clearedDownloadTokens = 0;

    for (const build of builds) {
      let shouldSave = false;
      if (build.downloadTokenExpiresAt && build.downloadTokenExpiresAt <= now) {
        build.downloadTokenHash = undefined;
        build.downloadTokenExpiresAt = undefined;
        clearedDownloadTokens++;
        shouldSave = true;
      }

      const isExpired = Boolean(build.data.expiresAt && build.data.expiresAt <= now);
      const canDeleteBuild = build.data.status === 'ready'
        || build.data.status === 'failed'
        || build.data.status === 'cancelled';
      if (isExpired && canDeleteBuild) {
        if (build.data.artifact) {
          await this.deleteImageArtifact(build.data.artifact);
          deletedArtifacts++;
        }
        await build.delete();
        deletedBuilds++;
        continue;
      }

      if (shouldSave) {
        build.data.updatedAt = now;
        await build.save();
      }
    }

    if (deletedBuilds > 0 || deletedArtifacts > 0 || clearedDownloadTokens > 0) {
      logger.log(
        'info',
        `BaseOS image cleanup completed: ${deletedBuilds} builds, ${deletedArtifacts} artifacts, ${clearedDownloadTokens} download tokens`,
      );
    }

    return {
      deletedBuilds,
      deletedArtifacts,
      clearedDownloadTokens,
    };
  }

  private async upsertNode(statusArg: IBaseOsRuntimeInfo, nodeTokenArg: string) {
    const now = Date.now();
    let node = await this.CBaseOsNode.getInstance({
      id: statusArg.nodeId,
    }).catch(() => null);

    if (!node) {
      node = new this.CBaseOsNode({
        id: statusArg.nodeId,
        nodeToken: nodeTokenArg,
        data: {
          runtimeInfo: statusArg,
          createdAt: now,
          updatedAt: now,
          lastHeartbeatAt: now,
        },
      });
    } else {
      node.nodeToken = nodeTokenArg;
      node.data = {
        ...node.data,
        runtimeInfo: statusArg,
        updatedAt: now,
        lastHeartbeatAt: now,
      };
    }

    await node.save();
    return node;
  }

  private async executeImageBuild(
    buildArg: BaseOsImageBuild,
    provisioningTokenArg: string,
    buildRequestArg: IBaseOsImageBuildRequest,
    workerArg: ISelectedCoreBuildWorker,
  ) {
    buildArg.data.status = 'building';
    buildArg.data.startedAt = Date.now();
    buildArg.data.updatedAt = Date.now();
    await buildArg.save();

    const artifactFilename = this.getArtifactFilename(
      buildArg.data.architecture,
      buildArg.data.imageKind || 'ubuntu-iso',
    );
    const artifactKey = `corebuild/baseos/${buildArg.id}/${artifactFilename}`;
    const response = await fetch(
      this.getCoreBuildUrl(workerArg.url, '/corebuild/v1/jobs/baseos-image'),
      {
        method: 'POST',
        headers: {
          'content-type': 'application/json',
          ...(workerArg.token ? { authorization: `Bearer ${workerArg.token}` } : {}),
        },
        body: JSON.stringify({
          apiToken: workerArg.token,
          job: {
            id: buildArg.id,
            architecture: buildArg.data.architecture,
            imageKind: buildArg.data.imageKind,
            cloudlyUrl: buildArg.data.cloudlyUrl,
            provisioningToken: provisioningTokenArg,
            sourceImageUrl: buildArg.data.sourceImageUrl,
            sourceImagePreset: buildArg.data.sourceImagePreset,
            balenaOsVersion: buildArg.data.balenaOsVersion,
            ubuntuVersion: buildArg.data.ubuntuVersion,
            hostname: buildArg.data.hostname,
            wifi: buildArg.data.wifiSsid
              ? {
                  ssid: buildArg.data.wifiSsid,
                  password: buildRequestArg.wifi?.password,
                }
              : undefined,
            sshPublicKey: buildArg.data.sshPublicKey,
            s3Descriptor: this.cloudlyRef.config.data.s3Descriptor,
            artifactKey,
          },
        }),
      },
    );
    const responseBody = await response.json() as ICoreBuildBaseOsImageResponse;
    buildArg.data.logs = responseBody.logs || [];
    if (!response.ok || !responseBody.success || !responseBody.artifact) {
      buildArg.data.status = 'failed';
      buildArg.data.errorText = responseBody.errorText || `CoreBuild failed with HTTP ${response.status}`;
      buildArg.data.updatedAt = Date.now();
      buildArg.data.completedAt = Date.now();
      await buildArg.save();
      return;
    }

    buildArg.data.status = 'ready';
    buildArg.data.artifact = responseBody.artifact;
    buildArg.data.updatedAt = Date.now();
    buildArg.data.completedAt = Date.now();
    await buildArg.save();
  }

  private async selectCoreBuildWorker(
    architectureArg: TBaseOsImageArchitecture,
    imageKindArg: TBaseOsImageKind,
    sourceImagePresetArg?: TBaseOsImageSourcePreset,
  ): Promise<ISelectedCoreBuildWorker> {
    const workers = await this.getConfiguredCoreBuildWorkers();
    if (workers.length === 0) {
      throw new plugins.typedrequest.TypedResponseError('No CoreBuild workers are configured in Cloudly settings');
    }

    const rejectionReasons: string[] = [];
    for (const worker of workers) {
      try {
        const capabilities = await this.fetchWorkerCapabilities(worker);
        const workerLabel = capabilities.workerId || worker.id || worker.url;
        const supportedImageKinds = capabilities.supportedImageKinds || ['ubuntu-iso'];
        if (!capabilities.supportedBuildTypes?.includes('baseos-image')) {
          rejectionReasons.push(`${workerLabel}: missing baseos-image support`);
          continue;
        }
        if (!capabilities.supportedArchitectures?.includes(architectureArg)) {
          rejectionReasons.push(`${workerLabel}: missing ${architectureArg} support`);
          continue;
        }
        if (!supportedImageKinds.includes(imageKindArg)) {
          rejectionReasons.push(`${workerLabel}: missing ${imageKindArg} support`);
          continue;
        }
        if (sourceImagePresetArg && !capabilities.supportedSourcePresets?.includes(sourceImagePresetArg)) {
          rejectionReasons.push(`${workerLabel}: missing ${sourceImagePresetArg} source preset support`);
          continue;
        }
        return {
          ...worker,
          capabilities,
        };
      } catch (error) {
        rejectionReasons.push(`${worker.id || worker.url}: ${(error as Error).message}`);
      }
    }

    throw new plugins.typedrequest.TypedResponseError(
      `No CoreBuild worker supports BaseOS ${architectureArg} ${imageKindArg} builds. ${rejectionReasons.join('; ')}`,
    );
  }

  private async getConfiguredCoreBuildWorkers(): Promise<ICoreBuildWorkerSetting[]> {
    const workers: ICoreBuildWorkerSetting[] = [];
    const workersJson = await this.cloudlyRef.settingsManager.getSetting('corebuildWorkersJson');
    if (workersJson) {
      try {
        const parsedWorkers = JSON.parse(workersJson) as unknown;
        if (!Array.isArray(parsedWorkers)) {
          throw new Error('corebuildWorkersJson must be a JSON array');
        }
        for (const worker of parsedWorkers) {
          if (typeof worker === 'string' && worker) {
            workers.push({ url: worker });
          } else if (this.isCoreBuildWorkerSetting(worker)) {
            workers.push(worker);
          } else {
            throw new Error('Each CoreBuild worker must be a URL string or object with a url field');
          }
        }
      } catch (error) {
        throw new plugins.typedrequest.TypedResponseError(
          `corebuildWorkersJson is invalid: ${(error as Error).message}`,
        );
      }
    }

    const legacyWorkerUrl = await this.cloudlyRef.settingsManager.getSetting('corebuildWorkerUrl');
    const legacyWorkerToken = await this.cloudlyRef.settingsManager.getSetting('corebuildWorkerToken');
    if (legacyWorkerUrl) {
      workers.push({
        id: 'default',
        url: legacyWorkerUrl,
        token: legacyWorkerToken,
      });
    }

    const seenUrls = new Set<string>();
    return workers.filter((workerArg) => {
      if (seenUrls.has(workerArg.url)) {
        return false;
      }
      seenUrls.add(workerArg.url);
      return true;
    });
  }

  private async fetchWorkerCapabilities(workerArg: ICoreBuildWorkerSetting) {
    const response = await fetch(
      this.getCoreBuildUrl(workerArg.url, '/corebuild/v1/capabilities'),
      {
        method: 'GET',
        headers: workerArg.token ? { authorization: `Bearer ${workerArg.token}` } : {},
      },
    );
    if (!response.ok) {
      throw new plugins.typedrequest.TypedResponseError(
        `CoreBuild capabilities request failed with HTTP ${response.status}`,
      );
    }
    return await response.json() as ICoreBuildCapabilitiesResponse;
  }

  private getImageKind(buildRequestArg: IBaseOsImageBuildRequest): TBaseOsImageKind {
    if (buildRequestArg.imageKind) {
      return buildRequestArg.imageKind;
    }
    if (buildRequestArg.architecture === 'rpi') {
      return 'balena-raw';
    }
    if (buildRequestArg.sourceImagePreset || buildRequestArg.balenaOsVersion) {
      return 'balena-raw';
    }
    if (buildRequestArg.sourceImageUrl && this.isRawImageUrl(buildRequestArg.sourceImageUrl)) {
      return 'balena-raw';
    }
    return 'ubuntu-iso';
  }

  private getSourceImagePreset(
    buildRequestArg: IBaseOsImageBuildRequest,
    imageKindArg: TBaseOsImageKind,
  ) {
    if (imageKindArg === 'ubuntu-iso') {
      if (buildRequestArg.sourceImagePreset || buildRequestArg.balenaOsVersion) {
        throw new plugins.typedrequest.TypedResponseError('balenaOS source presets only apply to balena-raw builds');
      }
      return undefined;
    }

    if (buildRequestArg.sourceImageUrl) {
      return undefined;
    }

    const preset = buildRequestArg.sourceImagePreset
      ? balenaSourcePresets.find((presetArg) => presetArg.preset === buildRequestArg.sourceImagePreset)
      : balenaSourcePresets.find((presetArg) => presetArg.architecture === buildRequestArg.architecture);
    if (!preset) {
      throw new plugins.typedrequest.TypedResponseError(
        `No balenaOS source preset is available for ${buildRequestArg.architecture}`,
      );
    }
    if (preset.architecture !== buildRequestArg.architecture) {
      throw new plugins.typedrequest.TypedResponseError(
        `${preset.preset} is only valid for ${preset.architecture} BaseOS images`,
      );
    }
    return preset.preset;
  }

  private isRawImageUrl(sourceImageUrlArg: string) {
    if (/\.(img|img\.xz|zip)(\?|$)/i.test(sourceImageUrlArg)) {
      return true;
    }
    try {
      const sourceUrl = new URL(sourceImageUrlArg);
      return sourceUrl.searchParams.get('fileType') === '.zip';
    } catch {
      return false;
    }
  }

  private getArtifactFilename(
    architectureArg: TBaseOsImageArchitecture,
    imageKindArg: TBaseOsImageKind,
  ) {
    const architectureSuffix = architectureArg === 'amd64' ? '' : `-${architectureArg}`;
    if (imageKindArg === 'balena-raw') {
      return `baseos${architectureSuffix}.img.xz`;
    }
    return `baseos${architectureSuffix}.iso`;
  }

  private isCoreBuildWorkerSetting(valueArg: unknown): valueArg is ICoreBuildWorkerSetting {
    return Boolean(valueArg)
      && typeof valueArg === 'object'
      && typeof (valueArg as ICoreBuildWorkerSetting).url === 'string'
      && (!(valueArg as ICoreBuildWorkerSetting).token || typeof (valueArg as ICoreBuildWorkerSetting).token === 'string')
      && (!(valueArg as ICoreBuildWorkerSetting).id || typeof (valueArg as ICoreBuildWorkerSetting).id === 'string');
  }

  private getCoreBuildUrl(workerUrlArg: string, pathArg: string) {
    return new URL(pathArg, workerUrlArg.endsWith('/') ? workerUrlArg : `${workerUrlArg}/`);
  }

  private async deleteImageArtifact(artifactArg: IBaseOsImageArtifact) {
    const bucket = await this.getArtifactBucket(artifactArg.bucketName);
    await bucket.fastRemove({ path: artifactArg.key });
  }

  private async getArtifactBucket(bucketNameArg: string) {
    const smartbucket = new plugins.smartbucket.SmartBucket({
      ...this.cloudlyRef.config.data.s3Descriptor,
      port: Number((this.cloudlyRef.config.data.s3Descriptor as any).port || 443),
    } as any);
    return await smartbucket.getBucketByName(bucketNameArg);
  }

  private async getImageBuildById(buildIdArg: string) {
    const build = await this.CBaseOsImageBuild.getInstance({ id: buildIdArg });
    if (!build) {
      throw new plugins.typedrequest.TypedResponseError(`BaseOS image build ${buildIdArg} not found`);
    }
    return build;
  }

  private async consumeProvisioningToken(joinTokenArg?: string) {
    if (!joinTokenArg) {
      return null;
    }
    const tokenHash = this.hashSecret(joinTokenArg);
    const builds = await this.CBaseOsImageBuild.getInstances({
      provisioningTokenHash: tokenHash,
    });
    const build = builds.find((buildArg) => {
      return !buildArg.provisioningTokenConsumedAt && (!buildArg.data.expiresAt || buildArg.data.expiresAt > Date.now());
    });
    if (!build) {
      return null;
    }
    build.provisioningTokenConsumedAt = Date.now();
    build.data.updatedAt = Date.now();
    await build.save();
    return build;
  }

  private hashSecret(secretArg: string) {
    return crypto.createHash('sha256').update(secretArg).digest('hex');
  }

  private getPublicCloudlyUrl() {
    const sslMode = this.cloudlyRef.config.data.sslMode;
    const protocol = sslMode === 'none' ? 'http' : 'https';
    const port = this.cloudlyRef.config.data.publicPort;
    const includePort = sslMode === 'none' && port && !['80', '443'].includes(port);
    return `${protocol}://${this.cloudlyRef.config.data.publicUrl}${includePort ? `:${port}` : ''}`;
  }

  private async updateNodeRuntimeInfo(
    nodeArg: BaseOsNode,
    statusArg: IBaseOsRuntimeInfo,
    heartbeatArg = false,
  ) {
    nodeArg.data = {
      ...nodeArg.data,
      runtimeInfo: statusArg,
      updatedAt: Date.now(),
      ...(heartbeatArg ? { lastHeartbeatAt: Date.now() } : {}),
    };
    await nodeArg.save();
  }

  private async getNodeByToken(nodeTokenArg: string) {
    const nodes = await this.CBaseOsNode.getInstances({
      nodeToken: nodeTokenArg,
    });
    return nodes[0] || null;
  }

  private isRuntimeInfo(valueArg: unknown): valueArg is IBaseOsRuntimeInfo {
    if (!valueArg || typeof valueArg !== 'object') {
      return false;
    }
    const runtimeInfo = valueArg as Partial<IBaseOsRuntimeInfo>;
    return runtimeInfo.runtime === 'baseos'
      && typeof runtimeInfo.nodeId === 'string'
      && runtimeInfo.nodeId.length > 0
      && typeof runtimeInfo.checkedAt === 'number';
  }

  private async readJsonBody<T>(ctxArg: plugins.typedserver.IRequestContext): Promise<T> {
    const bodyString = (await ctxArg.text()).trim();
    return bodyString ? JSON.parse(bodyString) as T : {} as T;
  }

  private createJsonResponse(
    statusCodeArg: number,
    bodyArg: object,
  ): Response {
    return new Response(JSON.stringify(bodyArg), {
      status: statusCodeArg,
      headers: {
        'Content-Type': 'application/json',
      },
    });
  }
}