feat: add baseos image builds

ts/manager.baseos/classes.baseosimagebuild.ts

@@ -0,0 +1,79 @@
+import * as plugins from '../plugins.js';
+
+export type TBaseOsImageArchitecture = 'amd64' | 'arm64' | 'rpi';
+export type TBaseOsImageBuildStatus = 'queued' | 'building' | 'ready' | 'failed' | 'cancelled';
+
+export interface IBaseOsImageArtifact {
+  bucketName: string;
+  key: string;
+  filename: string;
+  contentType: string;
+  size: number;
+  sha256: string;
+  createdAt: number;
+}
+
+export interface IBaseOsImageBuildPublic {
+  id: string;
+  data: {
+    status: TBaseOsImageBuildStatus;
+    architecture: TBaseOsImageArchitecture;
+    cloudlyUrl: string;
+    sourceImageUrl?: string;
+    ubuntuVersion?: string;
+    hostname?: string;
+    wifiSsid?: string;
+    sshPublicKey?: string;
+    artifact?: IBaseOsImageArtifact;
+    errorText?: string;
+    logs: string[];
+    createdAt: number;
+    updatedAt: number;
+    startedAt?: number;
+    completedAt?: number;
+    expiresAt?: number;
+  };
+}
+
+@plugins.smartdata.managed()
+export class BaseOsImageBuild extends plugins.smartdata.SmartDataDbDoc<
+  BaseOsImageBuild,
+  IBaseOsImageBuildPublic
+> {
+  constructor(optionsArg?: IBaseOsImageBuildPublic & {
+    provisioningTokenHash?: string;
+    provisioningTokenConsumedAt?: number;
+    downloadTokenHash?: string;
+    downloadTokenExpiresAt?: number;
+  }) {
+    super();
+    if (optionsArg) {
+      Object.assign(this, optionsArg);
+    }
+  }
+
+  @plugins.smartdata.unI()
+  public id!: string;
+
+  @plugins.smartdata.svDb()
+  public provisioningTokenHash!: string;
+
+  @plugins.smartdata.svDb()
+  public provisioningTokenConsumedAt?: number;
+
+  @plugins.smartdata.svDb()
+  public downloadTokenHash?: string;
+
+  @plugins.smartdata.svDb()
+  public downloadTokenExpiresAt?: number;
+
+  @plugins.smartdata.svDb()
+  public data!: IBaseOsImageBuildPublic['data'];
+
+  public toPublicBuild(): IBaseOsImageBuildPublic {
+    return {
+      id: this.id,
+      data: this.data,
+    };
+  }
+}

ts/manager.baseos/classes.baseosmanager.ts

@@ -1,4 +1,6 @@
 import * as plugins from '../plugins.js';
+import * as crypto from 'node:crypto';
+import * as nodeStream from 'node:stream';
 import type { Cloudly } from '../classes.cloudly.js';
 import { logger } from '../logger.js';
 import {
@@ -7,6 +9,12 @@ import {
   type IBaseOsNodePublic,
   type IBaseOsRuntimeInfo,
 } from './classes.baseosnode.js';
+import {
+  BaseOsImageBuild,
+  type IBaseOsImageArtifact,
+  type IBaseOsImageBuildPublic,
+  type TBaseOsImageArchitecture,
+} from './classes.baseosimagebuild.js';
 
 interface IBaseOsRegisterRequest {
   joinToken?: string;
@@ -42,6 +50,71 @@ interface IRequestGetBaseOsNodes {
   };
 }
 
+interface IBaseOsImageBuildRequest {
+  architecture: TBaseOsImageArchitecture;
+  cloudlyUrl?: string;
+  sourceImageUrl?: string;
+  ubuntuVersion?: string;
+  hostname?: string;
+  wifi?: {
+    ssid: string;
+    password?: string;
+  };
+  sshPublicKey?: string;
+  artifactRetentionMs?: number;
+}
+
+interface IRequestCreateBaseOsImageBuild {
+  method: 'createBaseOsImageBuild';
+  request: {
+    identity: plugins.servezoneInterfaces.data.IIdentity;
+    build: IBaseOsImageBuildRequest;
+  };
+  response: {
+    build: IBaseOsImageBuildPublic;
+  };
+}
+
+interface IRequestGetBaseOsImageBuilds {
+  method: 'getBaseOsImageBuilds';
+  request: {
+    identity: plugins.servezoneInterfaces.data.IIdentity;
+  };
+  response: {
+    builds: IBaseOsImageBuildPublic[];
+  };
+}
+
+interface IRequestGetBaseOsImageBuildById {
+  method: 'getBaseOsImageBuildById';
+  request: {
+    identity: plugins.servezoneInterfaces.data.IIdentity;
+    buildId: string;
+  };
+  response: {
+    build: IBaseOsImageBuildPublic;
+  };
+}
+
+interface IRequestCreateBaseOsImageDownloadUrl {
+  method: 'createBaseOsImageDownloadUrl';
+  request: {
+    identity: plugins.servezoneInterfaces.data.IIdentity;
+    buildId: string;
+  };
+  response: {
+    url: string;
+    expiresAt: number;
+  };
+}
+
+interface ICoreBuildBaseOsImageResponse {
+  success: boolean;
+  artifact?: IBaseOsImageArtifact;
+  logs: string[];
+  errorText?: string;
+}
+
 export class CloudlyBaseOsManager {
   public cloudlyRef: Cloudly;
   public typedRouter = new plugins.typedrequest.TypedRouter();
@@ -51,6 +124,7 @@ export class CloudlyBaseOsManager {
   }
 
   public CBaseOsNode = plugins.smartdata.setDefaultManagerForDoc(this, BaseOsNode);
+  public CBaseOsImageBuild = plugins.smartdata.setDefaultManagerForDoc(this, BaseOsImageBuild);
 
   constructor(cloudlyRefArg: Cloudly) {
     this.cloudlyRef = cloudlyRefArg;
@@ -70,6 +144,64 @@ export class CloudlyBaseOsManager {
         },
       ),
     );
+
+    this.typedRouter.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<IRequestCreateBaseOsImageBuild>(
+        'createBaseOsImageBuild',
+        async (requestDataArg) => {
+          await plugins.smartguard.passGuardsOrReject(
+            { identity: requestDataArg.identity },
+            [this.cloudlyRef.authManager.adminIdentityGuard],
+          );
+          return {
+            build: await this.createImageBuild(requestDataArg.build),
+          };
+        },
+      ),
+    );
+
+    this.typedRouter.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<IRequestGetBaseOsImageBuilds>(
+        'getBaseOsImageBuilds',
+        async (requestDataArg) => {
+          await plugins.smartguard.passGuardsOrReject(
+            { identity: requestDataArg.identity },
+            [this.cloudlyRef.authManager.adminIdentityGuard],
+          );
+          return {
+            builds: await this.getPublicImageBuilds(),
+          };
+        },
+      ),
+    );
+
+    this.typedRouter.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<IRequestGetBaseOsImageBuildById>(
+        'getBaseOsImageBuildById',
+        async (requestDataArg) => {
+          await plugins.smartguard.passGuardsOrReject(
+            { identity: requestDataArg.identity },
+            [this.cloudlyRef.authManager.adminIdentityGuard],
+          );
+          return {
+            build: (await this.getImageBuildById(requestDataArg.buildId)).toPublicBuild(),
+          };
+        },
+      ),
+    );
+
+    this.typedRouter.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<IRequestCreateBaseOsImageDownloadUrl>(
+        'createBaseOsImageDownloadUrl',
+        async (requestDataArg) => {
+          await plugins.smartguard.passGuardsOrReject(
+            { identity: requestDataArg.identity },
+            [this.cloudlyRef.authManager.adminIdentityGuard],
+          );
+          return await this.createImageDownloadUrl(requestDataArg.buildId);
+        },
+      ),
+    );
   }
 
   public async start() {
@@ -112,6 +244,47 @@ export class CloudlyBaseOsManager {
     }
   }
 
+  public async handleImageDownloadHttpRequest(
+    reqArg: plugins.typedserver.Request,
+    resArg: plugins.typedserver.Response,
+  ) {
+    try {
+      const requestUrl = new URL((reqArg as any).originalUrl || reqArg.url || '/', 'http://localhost');
+      const buildId = requestUrl.pathname.split('/').at(-2);
+      const token = requestUrl.searchParams.get('token');
+      if (!buildId || !token) {
+        this.sendJson(resArg, 400, { errorText: 'build id or download token missing' });
+        return;
+      }
+      const build = await this.getImageBuildById(buildId);
+      if (build.downloadTokenHash !== this.hashSecret(token) || (build.downloadTokenExpiresAt || 0) < Date.now()) {
+        this.sendJson(resArg, 403, { errorText: 'download token is invalid or expired' });
+        return;
+      }
+      if (build.data.status !== 'ready' || !build.data.artifact) {
+        this.sendJson(resArg, 409, { errorText: 'image build is not ready' });
+        return;
+      }
+
+      const artifact = build.data.artifact;
+      const smartbucket = new plugins.smartbucket.SmartBucket({
+        ...this.cloudlyRef.config.data.s3Descriptor,
+        port: Number((this.cloudlyRef.config.data.s3Descriptor as any).port || 443),
+      } as any);
+      const bucket = await smartbucket.getBucketByName(artifact.bucketName);
+      const artifactStream = await bucket.fastGetStream({ path: artifact.key }, 'nodestream');
+      resArg.status(200);
+      resArg.setHeader('Content-Type', artifact.contentType || 'application/octet-stream');
+      resArg.setHeader('Content-Length', String(artifact.size));
+      resArg.setHeader('Content-Disposition', `attachment; filename="${artifact.filename}"`);
+      (artifactStream as nodeStream.Readable).pipe(resArg as any);
+    } catch (error) {
+      this.sendJson(resArg, 500, {
+        errorText: `BaseOS image download failed: ${(error as Error).message}`,
+      });
+    }
+  }
+
   public async registerNode(
     requestDataArg: IBaseOsRegisterRequest,
   ): Promise<IBaseOsRegisterResponse> {
@@ -133,6 +306,17 @@ export class CloudlyBaseOsManager {
       }
     }
 
+    const acceptedBuild = await this.consumeProvisioningToken(requestDataArg.joinToken);
+    if (acceptedBuild) {
+      const nodeToken = await this.cloudlyRef.authManager.createNewSecureToken();
+      const node = await this.upsertNode(requestDataArg.status, nodeToken);
+      return {
+        accepted: true,
+        nodeId: node.id,
+        nodeToken,
+      };
+    }
+
     const configuredJoinToken = await this.cloudlyRef.settingsManager.getSetting('baseosJoinToken');
     if (!configuredJoinToken) {
       return {
@@ -193,6 +377,77 @@ export class CloudlyBaseOsManager {
     return nodes.map((nodeArg) => nodeArg.toPublicNode());
   }
 
+  public async createImageBuild(buildRequestArg: IBaseOsImageBuildRequest) {
+    const s3Descriptor = this.cloudlyRef.config.data.s3Descriptor;
+    if (!s3Descriptor?.bucketName) {
+      throw new plugins.typedrequest.TypedResponseError('Cloudly S3 storage is required for BaseOS image builds');
+    }
+
+    const workerUrl = await this.cloudlyRef.settingsManager.getSetting('corebuildWorkerUrl');
+    const workerToken = await this.cloudlyRef.settingsManager.getSetting('corebuildWorkerToken');
+    if (!workerUrl) {
+      throw new plugins.typedrequest.TypedResponseError('corebuildWorkerUrl is not configured in Cloudly settings');
+    }
+
+    const now = Date.now();
+    const buildId = await this.CBaseOsImageBuild.getNewId();
+    const provisioningToken = await this.cloudlyRef.authManager.createNewSecureToken();
+    const artifactRetentionMs = buildRequestArg.artifactRetentionMs || 1000 * 60 * 60 * 24 * 7;
+    const build = new this.CBaseOsImageBuild({
+      id: buildId,
+      provisioningTokenHash: this.hashSecret(provisioningToken),
+      data: {
+        status: 'queued',
+        architecture: buildRequestArg.architecture,
+        cloudlyUrl: buildRequestArg.cloudlyUrl || this.getPublicCloudlyUrl(),
+        sourceImageUrl: buildRequestArg.sourceImageUrl,
+        ubuntuVersion: buildRequestArg.ubuntuVersion || '24.04',
+        hostname: buildRequestArg.hostname,
+        wifiSsid: buildRequestArg.wifi?.ssid,
+        sshPublicKey: buildRequestArg.sshPublicKey,
+        logs: [],
+        createdAt: now,
+        updatedAt: now,
+        expiresAt: now + artifactRetentionMs,
+      },
+    });
+    await build.save();
+
+    this.executeImageBuild(build, provisioningToken, buildRequestArg, workerUrl, workerToken).catch(async (error) => {
+      build.data.status = 'failed';
+      build.data.errorText = (error as Error).message;
+      build.data.updatedAt = Date.now();
+      build.data.completedAt = Date.now();
+      await build.save();
+    });
+
+    return build.toPublicBuild();
+  }
+
+  public async getPublicImageBuilds() {
+    const builds = await this.CBaseOsImageBuild.getInstances({});
+    return builds
+      .sort((a, b) => b.data.createdAt - a.data.createdAt)
+      .map((buildArg) => buildArg.toPublicBuild());
+  }
+
+  public async createImageDownloadUrl(buildIdArg: string) {
+    const build = await this.getImageBuildById(buildIdArg);
+    if (build.data.status !== 'ready' || !build.data.artifact) {
+      throw new plugins.typedrequest.TypedResponseError('BaseOS image build is not ready');
+    }
+    const token = await this.cloudlyRef.authManager.createNewSecureToken();
+    const expiresAt = Date.now() + 1000 * 60 * 15;
+    build.downloadTokenHash = this.hashSecret(token);
+    build.downloadTokenExpiresAt = expiresAt;
+    build.data.updatedAt = Date.now();
+    await build.save();
+    return {
+      url: `/baseos/v1/images/${build.id}/download?token=${encodeURIComponent(token)}`,
+      expiresAt,
+    };
+  }
+
   private async upsertNode(statusArg: IBaseOsRuntimeInfo, nodeTokenArg: string) {
     const now = Date.now();
     let node = await this.CBaseOsNode.getInstance({
@@ -224,6 +479,113 @@ export class CloudlyBaseOsManager {
     return node;
   }
 
+  private async executeImageBuild(
+    buildArg: BaseOsImageBuild,
+    provisioningTokenArg: string,
+    buildRequestArg: IBaseOsImageBuildRequest,
+    workerUrlArg: string,
+    workerTokenArg?: string,
+  ) {
+    buildArg.data.status = 'building';
+    buildArg.data.startedAt = Date.now();
+    buildArg.data.updatedAt = Date.now();
+    await buildArg.save();
+
+    const artifactFilename = buildArg.data.architecture === 'amd64'
+      ? 'baseos.iso'
+      : buildArg.data.architecture === 'arm64'
+      ? 'baseos-arm64.iso'
+      : 'baseos-rpi.img';
+    const artifactKey = `corebuild/baseos/${buildArg.id}/${artifactFilename}`;
+    const response = await fetch(
+      new URL('/corebuild/v1/jobs/baseos-image', workerUrlArg.endsWith('/') ? workerUrlArg : `${workerUrlArg}/`),
+      {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          ...(workerTokenArg ? { authorization: `Bearer ${workerTokenArg}` } : {}),
+        },
+        body: JSON.stringify({
+          apiToken: workerTokenArg,
+          job: {
+            id: buildArg.id,
+            architecture: buildArg.data.architecture,
+            cloudlyUrl: buildArg.data.cloudlyUrl,
+            provisioningToken: provisioningTokenArg,
+            sourceImageUrl: buildArg.data.sourceImageUrl,
+            ubuntuVersion: buildArg.data.ubuntuVersion,
+            hostname: buildArg.data.hostname,
+            wifi: buildArg.data.wifiSsid
+              ? {
+                  ssid: buildArg.data.wifiSsid,
+                  password: buildRequestArg.wifi?.password,
+                }
+              : undefined,
+            sshPublicKey: buildArg.data.sshPublicKey,
+            s3Descriptor: this.cloudlyRef.config.data.s3Descriptor,
+            artifactKey,
+          },
+        }),
+      },
+    );
+    const responseBody = await response.json() as ICoreBuildBaseOsImageResponse;
+    buildArg.data.logs = responseBody.logs || [];
+    if (!response.ok || !responseBody.success || !responseBody.artifact) {
+      buildArg.data.status = 'failed';
+      buildArg.data.errorText = responseBody.errorText || `CoreBuild failed with HTTP ${response.status}`;
+      buildArg.data.updatedAt = Date.now();
+      buildArg.data.completedAt = Date.now();
+      await buildArg.save();
+      return;
+    }
+
+    buildArg.data.status = 'ready';
+    buildArg.data.artifact = responseBody.artifact;
+    buildArg.data.updatedAt = Date.now();
+    buildArg.data.completedAt = Date.now();
+    await buildArg.save();
+  }
+
+  private async getImageBuildById(buildIdArg: string) {
+    const build = await this.CBaseOsImageBuild.getInstance({ id: buildIdArg });
+    if (!build) {
+      throw new plugins.typedrequest.TypedResponseError(`BaseOS image build ${buildIdArg} not found`);
+    }
+    return build;
+  }
+
+  private async consumeProvisioningToken(joinTokenArg?: string) {
+    if (!joinTokenArg) {
+      return null;
+    }
+    const tokenHash = this.hashSecret(joinTokenArg);
+    const builds = await this.CBaseOsImageBuild.getInstances({
+      provisioningTokenHash: tokenHash,
+    });
+    const build = builds.find((buildArg) => {
+      return !buildArg.provisioningTokenConsumedAt && (!buildArg.data.expiresAt || buildArg.data.expiresAt > Date.now());
+    });
+    if (!build) {
+      return null;
+    }
+    build.provisioningTokenConsumedAt = Date.now();
+    build.data.updatedAt = Date.now();
+    await build.save();
+    return build;
+  }
+
+  private hashSecret(secretArg: string) {
+    return crypto.createHash('sha256').update(secretArg).digest('hex');
+  }
+
+  private getPublicCloudlyUrl() {
+    const sslMode = this.cloudlyRef.config.data.sslMode;
+    const protocol = sslMode === 'none' ? 'http' : 'https';
+    const port = this.cloudlyRef.config.data.publicPort;
+    const includePort = sslMode === 'none' && port && !['80', '443'].includes(port);
+    return `${protocol}://${this.cloudlyRef.config.data.publicUrl}${includePort ? `:${port}` : ''}`;
+  }
+
   private async updateNodeRuntimeInfo(
     nodeArg: BaseOsNode,
     statusArg: IBaseOsRuntimeInfo,