feat: add balena source presets

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serve.zone/corebuild",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "private": false,
   "description": "Build worker for serve.zone image and ISO artifact generation.",
   "type": "module",
@@ -14,6 +14,7 @@
     "test": "pnpm run build"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1044.0",
     "@push.rocks/smartbucket": "^3.3.10",
     "@tsclass/tsclass": "^9.2.0"
   },

pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     dependencies:
+      '@aws-sdk/client-s3':
+        specifier: ^3.1044.0
+        version: 3.1044.0
       '@push.rocks/smartbucket':
         specifier: ^3.3.10
         version: 3.3.10

ts/classes.baseosimagebuilder.ts

@@ -9,9 +9,35 @@ import * as plugins from './plugins.js';
 import type {
   IBaseOsImageArtifactResult,
   IBaseOsImageJob,
+  IS3Descriptor,
   TBaseOsImageKind,
+  TBaseOsImageSourcePreset,
 } from './types.js';
 
+interface IBalenaSourcePreset {
+  preset: TBaseOsImageSourcePreset;
+  architecture: IBaseOsImageJob['architecture'];
+  deviceType: string;
+}
+
+const balenaSourcePresets: IBalenaSourcePreset[] = [
+  {
+    preset: 'balena-generic-amd64',
+    architecture: 'amd64',
+    deviceType: 'generic-amd64',
+  },
+  {
+    preset: 'balena-generic-aarch64',
+    architecture: 'arm64',
+    deviceType: 'generic-aarch64',
+  },
+  {
+    preset: 'balena-raspberrypi4-64',
+    architecture: 'rpi',
+    deviceType: 'raspberrypi4-64',
+  },
+];
+
 export interface IBaseOsImageBuilderOptions {
   workdir: string;
   isoCreatorCommand: string;
@@ -35,7 +61,7 @@ export class BaseOsImageBuilder {
     const outputPath = path.join(outputDir, filename);
     const configPath = path.join(jobDir, 'isocreator.config.json');
     const isoCreatorConfig = imageKind === 'balena-raw'
-      ? this.createRawImageConfig(jobArg, outputDir, filename)
+      ? await this.createRawImageConfig(jobArg, outputDir, filename, logs)
       : this.createIsoCreatorConfig(jobArg, outputDir, filename);
     await fsp.writeFile(configPath, `${JSON.stringify(isoCreatorConfig, null, 2)}\n`);
 
@@ -44,7 +70,13 @@ export class BaseOsImageBuilder {
 
     const stat = await fsp.stat(outputPath);
     const sha256 = await this.sha256File(outputPath);
-    await this.uploadArtifact(jobArg, outputPath, logs);
+    await this.uploadArtifact(
+      jobArg,
+      outputPath,
+      stat.size,
+      this.getContentType(filename, imageKind),
+      logs,
+    );
 
     await fsp.rm(jobDir, { recursive: true, force: true }).catch(() => undefined);
     return {
@@ -138,18 +170,18 @@ export class BaseOsImageBuilder {
     };
   }
 
-  private createRawImageConfig(jobArg: IBaseOsImageJob, outputDirArg: string, filenameArg: string) {
-    if (!jobArg.sourceImageUrl) {
-      throw new Error('sourceImageUrl is required for balena-raw BaseOS image builds');
-    }
+  private async createRawImageConfig(
+    jobArg: IBaseOsImageJob,
+    outputDirArg: string,
+    filenameArg: string,
+    logsArg: string[],
+  ) {
+    const source = await this.resolveRawImageSource(jobArg, logsArg);
 
     return {
       version: '1.0',
       imageKind: 'raw-image',
-      source: {
-        type: 'url',
-        url: jobArg.sourceImageUrl,
-      },
+      source,
       output: {
         filename: filenameArg,
         path: outputDirArg,
@@ -182,6 +214,7 @@ export class BaseOsImageBuilder {
           BASEOS_CLOUDLY_URL: jobArg.cloudlyUrl,
           BASEOS_JOIN_TOKEN: jobArg.provisioningToken,
           BASEOS_STATE_PATH: '/data/baseos/state.json',
+          BASEOS_PRELOAD_TARGET_STATE_PATH: '/data/baseos/preload-target-state.json',
           BASEOS_HEARTBEAT_INTERVAL_MS: '60000',
           SERVEZONE_RUNTIME: 'baseos',
         },
@@ -196,12 +229,110 @@ export class BaseOsImageBuilder {
     if (jobArg.architecture === 'rpi') {
       return 'balena-raw';
     }
-    if (jobArg.sourceImageUrl && /\.(img|img\.xz|zip)(\?|$)/i.test(jobArg.sourceImageUrl)) {
+    if (jobArg.sourceImagePreset || jobArg.balenaOsVersion) {
+      return 'balena-raw';
+    }
+    if (jobArg.sourceImageUrl && this.isRawImageUrl(jobArg.sourceImageUrl)) {
       return 'balena-raw';
     }
     return 'ubuntu-iso';
   }
 
+  private async resolveRawImageSource(jobArg: IBaseOsImageJob, logsArg: string[]) {
+    if (jobArg.sourceImageUrl) {
+      const filename = this.getSourceFilenameFromUrl(jobArg.sourceImageUrl);
+      return {
+        type: 'url' as const,
+        url: jobArg.sourceImageUrl,
+        ...(filename ? { filename } : {}),
+      };
+    }
+
+    const preset = this.getBalenaSourcePreset(jobArg);
+    const version = await this.resolveBalenaOsVersion(preset, jobArg.balenaOsVersion, logsArg);
+    const downloadUrl = new URL('https://api.balena-cloud.com/download');
+    downloadUrl.searchParams.set('deviceType', preset.deviceType);
+    downloadUrl.searchParams.set('version', version);
+    downloadUrl.searchParams.set('fileType', '.zip');
+    return {
+      type: 'url' as const,
+      url: downloadUrl.toString(),
+      filename: `balenaos-${preset.deviceType}-${version}.img.zip`,
+    };
+  }
+
+  private getBalenaSourcePreset(jobArg: IBaseOsImageJob) {
+    const preset = jobArg.sourceImagePreset
+      ? balenaSourcePresets.find((presetArg) => presetArg.preset === jobArg.sourceImagePreset)
+      : balenaSourcePresets.find((presetArg) => presetArg.architecture === jobArg.architecture);
+    if (!preset) {
+      throw new Error(`No balenaOS source preset is available for ${jobArg.architecture}`);
+    }
+    if (preset.architecture !== jobArg.architecture) {
+      throw new Error(`${preset.preset} is only valid for ${preset.architecture} BaseOS images`);
+    }
+    return preset;
+  }
+
+  private async resolveBalenaOsVersion(
+    presetArg: IBalenaSourcePreset,
+    versionArg: string | undefined,
+    logsArg: string[],
+  ) {
+    const requestedVersion = versionArg?.trim() || 'latest';
+    if (requestedVersion !== 'latest') {
+      return requestedVersion;
+    }
+
+    const releaseUrl = new URL('https://api.balena-cloud.com/v7/release');
+    releaseUrl.searchParams.set('$select', 'raw_version');
+    releaseUrl.searchParams.set(
+      '$filter',
+      `(is_final eq true) and (is_invalidated eq false) and (status eq 'success') and (semver_major gt 0) and (belongs_to__application/any(bta:(bta/is_host eq true) and (bta/is_for__device_type/any(dt:dt/slug eq '${presetArg.deviceType}'))))`,
+    );
+    releaseUrl.searchParams.set('$orderby', 'semver_major desc,semver_minor desc,semver_patch desc,revision desc');
+    releaseUrl.searchParams.set('$top', '1');
+    const response = await fetch(releaseUrl);
+    if (!response.ok) {
+      throw new Error(`Failed to resolve latest balenaOS version for ${presetArg.deviceType}: HTTP ${response.status}`);
+    }
+    const responseBody = await response.json() as { d?: Array<{ raw_version?: string }> };
+    const latestVersion = responseBody.d?.[0]?.raw_version;
+    if (!latestVersion) {
+      throw new Error(`No balenaOS version found for ${presetArg.deviceType}`);
+    }
+    logsArg.push(`Resolved balenaOS ${presetArg.deviceType} latest version ${latestVersion}`);
+    return latestVersion;
+  }
+
+  private isRawImageUrl(sourceImageUrlArg: string) {
+    if (/\.(img|img\.xz|zip)(\?|$)/i.test(sourceImageUrlArg)) {
+      return true;
+    }
+    try {
+      const sourceUrl = new URL(sourceImageUrlArg);
+      return sourceUrl.searchParams.get('fileType') === '.zip';
+    } catch {
+      return false;
+    }
+  }
+
+  private getSourceFilenameFromUrl(sourceImageUrlArg: string) {
+    try {
+      const sourceUrl = new URL(sourceImageUrlArg);
+      const lowerPath = sourceUrl.pathname.toLowerCase();
+      if (lowerPath.endsWith('.img') || lowerPath.endsWith('.img.xz') || lowerPath.endsWith('.zip')) {
+        return undefined;
+      }
+      if (sourceUrl.searchParams.get('fileType') === '.zip') {
+        return 'source.img.zip';
+      }
+    } catch {
+      return undefined;
+    }
+    return undefined;
+  }
+
   private getArtifactFilename(jobArg: IBaseOsImageJob, imageKindArg: TBaseOsImageKind) {
     const architectureSuffix = jobArg.architecture === 'amd64' ? '' : `-${jobArg.architecture}`;
     if (imageKindArg === 'balena-raw') {
@@ -304,20 +435,49 @@ fi
     }
   }
 
-  private async uploadArtifact(jobArg: IBaseOsImageJob, outputPathArg: string, logsArg: string[]) {
+  private async uploadArtifact(
+    jobArg: IBaseOsImageJob,
+    outputPathArg: string,
+    contentLengthArg: number,
+    contentTypeArg: string,
+    logsArg: string[],
+  ) {
     logsArg.push(`Uploading artifact to ${jobArg.s3Descriptor.bucketName}/${jobArg.artifactKey}`);
-    const smartbucket = new plugins.smartbucket.SmartBucket({
-      ...jobArg.s3Descriptor,
-      port: Number(jobArg.s3Descriptor.port || 443),
-    } as any);
-    const bucket = await smartbucket.getBucketByName(jobArg.s3Descriptor.bucketName);
-    await bucket.fastPutStream({
-      path: jobArg.artifactKey,
-      readableStream: fs.createReadStream(outputPathArg),
-      overwrite: true,
+    const s3Client = this.createS3Client(jobArg.s3Descriptor);
+    await s3Client.send(new plugins.awsS3.PutObjectCommand({
+      Bucket: jobArg.s3Descriptor.bucketName,
+      Key: jobArg.artifactKey,
+      Body: fs.createReadStream(outputPathArg),
+      ContentLength: contentLengthArg,
+      ContentType: contentTypeArg,
+    }));
+  }
+
+  private createS3Client(s3DescriptorArg: IS3Descriptor) {
+    return new plugins.awsS3.S3Client({
+      endpoint: this.getS3Endpoint(s3DescriptorArg),
+      region: s3DescriptorArg.region || 'us-east-1',
+      credentials: {
+        accessKeyId: s3DescriptorArg.accessKey,
+        secretAccessKey: s3DescriptorArg.accessSecret,
+      },
+      forcePathStyle: true,
+      requestChecksumCalculation: 'WHEN_REQUIRED',
     });
   }
 
+  private getS3Endpoint(s3DescriptorArg: IS3Descriptor) {
+    const rawEndpoint = s3DescriptorArg.endpoint.trim();
+    const useSsl = s3DescriptorArg.useSsl !== false;
+    const endpointUrl = /^https?:\/\//i.test(rawEndpoint)
+      ? new URL(rawEndpoint)
+      : new URL(`${useSsl ? 'https' : 'http'}://${rawEndpoint}`);
+    if (s3DescriptorArg.port) {
+      endpointUrl.port = String(s3DescriptorArg.port);
+    }
+    return endpointUrl.origin;
+  }
+
   private async sha256File(filePathArg: string) {
     return await new Promise<string>((resolve, reject) => {
       const hash = crypto.createHash('sha256');

ts/classes.corebuildserver.ts

@@ -106,6 +106,7 @@ export class CoreBuildServer {
       supportedBuildTypes: ['baseos-image'],
       supportedArchitectures: ['amd64', 'arm64', 'rpi'],
       supportedImageKinds: ['ubuntu-iso', 'balena-raw'],
+      supportedSourcePresets: ['balena-generic-amd64', 'balena-generic-aarch64', 'balena-raspberrypi4-64'],
       cpuCores: os.cpus().length,
       memoryGb: Math.round(os.totalmem() / 1024 / 1024 / 1024),
       workdir: this.options.workdir,

ts/plugins.ts

@@ -1,3 +1,4 @@
 import * as smartbucket from '@push.rocks/smartbucket';
+import * as awsS3 from '@aws-sdk/client-s3';
 
-export { smartbucket };
+export { smartbucket, awsS3 };

ts/types.ts

@@ -2,6 +2,10 @@ import type { Readable } from 'node:stream';
 
 export type TBaseOsImageArchitecture = 'amd64' | 'arm64' | 'rpi';
 export type TBaseOsImageKind = 'ubuntu-iso' | 'balena-raw';
+export type TBaseOsImageSourcePreset =
+  | 'balena-generic-amd64'
+  | 'balena-generic-aarch64'
+  | 'balena-raspberrypi4-64';
 
 export interface IS3Descriptor {
   endpoint: string;
@@ -20,6 +24,8 @@ export interface IBaseOsImageJob {
   cloudlyUrl: string;
   provisioningToken: string;
   sourceImageUrl?: string;
+  sourceImagePreset?: TBaseOsImageSourcePreset;
+  balenaOsVersion?: string;
   ubuntuVersion?: string;
   hostname?: string;
   wifi?: {
@@ -58,6 +64,7 @@ export interface ICoreBuildCapabilities {
   supportedBuildTypes: string[];
   supportedArchitectures: TBaseOsImageArchitecture[];
   supportedImageKinds: TBaseOsImageKind[];
+  supportedSourcePresets: TBaseOsImageSourcePreset[];
   cpuCores: number;
   memoryGb?: number;
   workdir: string;