diff --git a/package.json b/package.json index 2fa7e89..8cb94a0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@serve.zone/corebuild", - "version": "0.1.0", + "version": "0.2.0", "private": false, "description": "Build worker for serve.zone image and ISO artifact generation.", "type": "module", @@ -14,6 +14,7 @@ "test": "pnpm run build" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1044.0", "@push.rocks/smartbucket": "^3.3.10", "@tsclass/tsclass": "^9.2.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index df47e35..0567fe8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1044.0 + version: 3.1044.0 '@push.rocks/smartbucket': specifier: ^3.3.10 version: 3.3.10 diff --git a/ts/classes.baseosimagebuilder.ts b/ts/classes.baseosimagebuilder.ts index 36ec81e..36437c3 100644 --- a/ts/classes.baseosimagebuilder.ts +++ b/ts/classes.baseosimagebuilder.ts @@ -9,9 +9,35 @@ import * as plugins from './plugins.js'; import type { IBaseOsImageArtifactResult, IBaseOsImageJob, + IS3Descriptor, TBaseOsImageKind, + TBaseOsImageSourcePreset, } from './types.js'; +interface IBalenaSourcePreset { + preset: TBaseOsImageSourcePreset; + architecture: IBaseOsImageJob['architecture']; + deviceType: string; +} + +const balenaSourcePresets: IBalenaSourcePreset[] = [ + { + preset: 'balena-generic-amd64', + architecture: 'amd64', + deviceType: 'generic-amd64', + }, + { + preset: 'balena-generic-aarch64', + architecture: 'arm64', + deviceType: 'generic-aarch64', + }, + { + preset: 'balena-raspberrypi4-64', + architecture: 'rpi', + deviceType: 'raspberrypi4-64', + }, +]; + export interface IBaseOsImageBuilderOptions { workdir: string; isoCreatorCommand: string; @@ -35,7 +61,7 @@ export class BaseOsImageBuilder { const outputPath = path.join(outputDir, filename); const configPath = path.join(jobDir, 'isocreator.config.json'); const isoCreatorConfig = imageKind === 'balena-raw' - ? this.createRawImageConfig(jobArg, outputDir, filename) + ? await this.createRawImageConfig(jobArg, outputDir, filename, logs) : this.createIsoCreatorConfig(jobArg, outputDir, filename); await fsp.writeFile(configPath, `${JSON.stringify(isoCreatorConfig, null, 2)}\n`); @@ -44,7 +70,13 @@ export class BaseOsImageBuilder { const stat = await fsp.stat(outputPath); const sha256 = await this.sha256File(outputPath); - await this.uploadArtifact(jobArg, outputPath, logs); + await this.uploadArtifact( + jobArg, + outputPath, + stat.size, + this.getContentType(filename, imageKind), + logs, + ); await fsp.rm(jobDir, { recursive: true, force: true }).catch(() => undefined); return { @@ -138,18 +170,18 @@ export class BaseOsImageBuilder { }; } - private createRawImageConfig(jobArg: IBaseOsImageJob, outputDirArg: string, filenameArg: string) { - if (!jobArg.sourceImageUrl) { - throw new Error('sourceImageUrl is required for balena-raw BaseOS image builds'); - } + private async createRawImageConfig( + jobArg: IBaseOsImageJob, + outputDirArg: string, + filenameArg: string, + logsArg: string[], + ) { + const source = await this.resolveRawImageSource(jobArg, logsArg); return { version: '1.0', imageKind: 'raw-image', - source: { - type: 'url', - url: jobArg.sourceImageUrl, - }, + source, output: { filename: filenameArg, path: outputDirArg, @@ -182,6 +214,7 @@ export class BaseOsImageBuilder { BASEOS_CLOUDLY_URL: jobArg.cloudlyUrl, BASEOS_JOIN_TOKEN: jobArg.provisioningToken, BASEOS_STATE_PATH: '/data/baseos/state.json', + BASEOS_PRELOAD_TARGET_STATE_PATH: '/data/baseos/preload-target-state.json', BASEOS_HEARTBEAT_INTERVAL_MS: '60000', SERVEZONE_RUNTIME: 'baseos', }, @@ -196,12 +229,110 @@ export class BaseOsImageBuilder { if (jobArg.architecture === 'rpi') { return 'balena-raw'; } - if (jobArg.sourceImageUrl && /\.(img|img\.xz|zip)(\?|$)/i.test(jobArg.sourceImageUrl)) { + if (jobArg.sourceImagePreset || jobArg.balenaOsVersion) { + return 'balena-raw'; + } + if (jobArg.sourceImageUrl && this.isRawImageUrl(jobArg.sourceImageUrl)) { return 'balena-raw'; } return 'ubuntu-iso'; } + private async resolveRawImageSource(jobArg: IBaseOsImageJob, logsArg: string[]) { + if (jobArg.sourceImageUrl) { + const filename = this.getSourceFilenameFromUrl(jobArg.sourceImageUrl); + return { + type: 'url' as const, + url: jobArg.sourceImageUrl, + ...(filename ? { filename } : {}), + }; + } + + const preset = this.getBalenaSourcePreset(jobArg); + const version = await this.resolveBalenaOsVersion(preset, jobArg.balenaOsVersion, logsArg); + const downloadUrl = new URL('https://api.balena-cloud.com/download'); + downloadUrl.searchParams.set('deviceType', preset.deviceType); + downloadUrl.searchParams.set('version', version); + downloadUrl.searchParams.set('fileType', '.zip'); + return { + type: 'url' as const, + url: downloadUrl.toString(), + filename: `balenaos-${preset.deviceType}-${version}.img.zip`, + }; + } + + private getBalenaSourcePreset(jobArg: IBaseOsImageJob) { + const preset = jobArg.sourceImagePreset + ? balenaSourcePresets.find((presetArg) => presetArg.preset === jobArg.sourceImagePreset) + : balenaSourcePresets.find((presetArg) => presetArg.architecture === jobArg.architecture); + if (!preset) { + throw new Error(`No balenaOS source preset is available for ${jobArg.architecture}`); + } + if (preset.architecture !== jobArg.architecture) { + throw new Error(`${preset.preset} is only valid for ${preset.architecture} BaseOS images`); + } + return preset; + } + + private async resolveBalenaOsVersion( + presetArg: IBalenaSourcePreset, + versionArg: string | undefined, + logsArg: string[], + ) { + const requestedVersion = versionArg?.trim() || 'latest'; + if (requestedVersion !== 'latest') { + return requestedVersion; + } + + const releaseUrl = new URL('https://api.balena-cloud.com/v7/release'); + releaseUrl.searchParams.set('$select', 'raw_version'); + releaseUrl.searchParams.set( + '$filter', + `(is_final eq true) and (is_invalidated eq false) and (status eq 'success') and (semver_major gt 0) and (belongs_to__application/any(bta:(bta/is_host eq true) and (bta/is_for__device_type/any(dt:dt/slug eq '${presetArg.deviceType}'))))`, + ); + releaseUrl.searchParams.set('$orderby', 'semver_major desc,semver_minor desc,semver_patch desc,revision desc'); + releaseUrl.searchParams.set('$top', '1'); + const response = await fetch(releaseUrl); + if (!response.ok) { + throw new Error(`Failed to resolve latest balenaOS version for ${presetArg.deviceType}: HTTP ${response.status}`); + } + const responseBody = await response.json() as { d?: Array<{ raw_version?: string }> }; + const latestVersion = responseBody.d?.[0]?.raw_version; + if (!latestVersion) { + throw new Error(`No balenaOS version found for ${presetArg.deviceType}`); + } + logsArg.push(`Resolved balenaOS ${presetArg.deviceType} latest version ${latestVersion}`); + return latestVersion; + } + + private isRawImageUrl(sourceImageUrlArg: string) { + if (/\.(img|img\.xz|zip)(\?|$)/i.test(sourceImageUrlArg)) { + return true; + } + try { + const sourceUrl = new URL(sourceImageUrlArg); + return sourceUrl.searchParams.get('fileType') === '.zip'; + } catch { + return false; + } + } + + private getSourceFilenameFromUrl(sourceImageUrlArg: string) { + try { + const sourceUrl = new URL(sourceImageUrlArg); + const lowerPath = sourceUrl.pathname.toLowerCase(); + if (lowerPath.endsWith('.img') || lowerPath.endsWith('.img.xz') || lowerPath.endsWith('.zip')) { + return undefined; + } + if (sourceUrl.searchParams.get('fileType') === '.zip') { + return 'source.img.zip'; + } + } catch { + return undefined; + } + return undefined; + } + private getArtifactFilename(jobArg: IBaseOsImageJob, imageKindArg: TBaseOsImageKind) { const architectureSuffix = jobArg.architecture === 'amd64' ? '' : `-${jobArg.architecture}`; if (imageKindArg === 'balena-raw') { @@ -304,20 +435,49 @@ fi } } - private async uploadArtifact(jobArg: IBaseOsImageJob, outputPathArg: string, logsArg: string[]) { + private async uploadArtifact( + jobArg: IBaseOsImageJob, + outputPathArg: string, + contentLengthArg: number, + contentTypeArg: string, + logsArg: string[], + ) { logsArg.push(`Uploading artifact to ${jobArg.s3Descriptor.bucketName}/${jobArg.artifactKey}`); - const smartbucket = new plugins.smartbucket.SmartBucket({ - ...jobArg.s3Descriptor, - port: Number(jobArg.s3Descriptor.port || 443), - } as any); - const bucket = await smartbucket.getBucketByName(jobArg.s3Descriptor.bucketName); - await bucket.fastPutStream({ - path: jobArg.artifactKey, - readableStream: fs.createReadStream(outputPathArg), - overwrite: true, + const s3Client = this.createS3Client(jobArg.s3Descriptor); + await s3Client.send(new plugins.awsS3.PutObjectCommand({ + Bucket: jobArg.s3Descriptor.bucketName, + Key: jobArg.artifactKey, + Body: fs.createReadStream(outputPathArg), + ContentLength: contentLengthArg, + ContentType: contentTypeArg, + })); + } + + private createS3Client(s3DescriptorArg: IS3Descriptor) { + return new plugins.awsS3.S3Client({ + endpoint: this.getS3Endpoint(s3DescriptorArg), + region: s3DescriptorArg.region || 'us-east-1', + credentials: { + accessKeyId: s3DescriptorArg.accessKey, + secretAccessKey: s3DescriptorArg.accessSecret, + }, + forcePathStyle: true, + requestChecksumCalculation: 'WHEN_REQUIRED', }); } + private getS3Endpoint(s3DescriptorArg: IS3Descriptor) { + const rawEndpoint = s3DescriptorArg.endpoint.trim(); + const useSsl = s3DescriptorArg.useSsl !== false; + const endpointUrl = /^https?:\/\//i.test(rawEndpoint) + ? new URL(rawEndpoint) + : new URL(`${useSsl ? 'https' : 'http'}://${rawEndpoint}`); + if (s3DescriptorArg.port) { + endpointUrl.port = String(s3DescriptorArg.port); + } + return endpointUrl.origin; + } + private async sha256File(filePathArg: string) { return await new Promise((resolve, reject) => { const hash = crypto.createHash('sha256'); diff --git a/ts/classes.corebuildserver.ts b/ts/classes.corebuildserver.ts index 23ac41b..2455ed3 100644 --- a/ts/classes.corebuildserver.ts +++ b/ts/classes.corebuildserver.ts @@ -106,6 +106,7 @@ export class CoreBuildServer { supportedBuildTypes: ['baseos-image'], supportedArchitectures: ['amd64', 'arm64', 'rpi'], supportedImageKinds: ['ubuntu-iso', 'balena-raw'], + supportedSourcePresets: ['balena-generic-amd64', 'balena-generic-aarch64', 'balena-raspberrypi4-64'], cpuCores: os.cpus().length, memoryGb: Math.round(os.totalmem() / 1024 / 1024 / 1024), workdir: this.options.workdir, diff --git a/ts/plugins.ts b/ts/plugins.ts index 7867c18..e6acce6 100644 --- a/ts/plugins.ts +++ b/ts/plugins.ts @@ -1,3 +1,4 @@ import * as smartbucket from '@push.rocks/smartbucket'; +import * as awsS3 from '@aws-sdk/client-s3'; -export { smartbucket }; +export { smartbucket, awsS3 }; diff --git a/ts/types.ts b/ts/types.ts index 7aae849..aa7709e 100644 --- a/ts/types.ts +++ b/ts/types.ts @@ -2,6 +2,10 @@ import type { Readable } from 'node:stream'; export type TBaseOsImageArchitecture = 'amd64' | 'arm64' | 'rpi'; export type TBaseOsImageKind = 'ubuntu-iso' | 'balena-raw'; +export type TBaseOsImageSourcePreset = + | 'balena-generic-amd64' + | 'balena-generic-aarch64' + | 'balena-raspberrypi4-64'; export interface IS3Descriptor { endpoint: string; @@ -20,6 +24,8 @@ export interface IBaseOsImageJob { cloudlyUrl: string; provisioningToken: string; sourceImageUrl?: string; + sourceImagePreset?: TBaseOsImageSourcePreset; + balenaOsVersion?: string; ubuntuVersion?: string; hostname?: string; wifi?: { @@ -58,6 +64,7 @@ export interface ICoreBuildCapabilities { supportedBuildTypes: string[]; supportedArchitectures: TBaseOsImageArchitecture[]; supportedImageKinds: TBaseOsImageKind[]; + supportedSourcePresets: TBaseOsImageSourcePreset[]; cpuCores: number; memoryGb?: number; workdir: string;