feat(base-images): add managed base image bundles with cache retention, hosted manifests, and opt-in integration boot testing
This commit is contained in:
@@ -0,0 +1,713 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import type {
|
||||
IBaseImageArtifactManifest,
|
||||
IBaseImageBundle,
|
||||
IBaseImageHostedManifest,
|
||||
IBaseImageManagerOptions,
|
||||
IEnsureBaseImageOptions,
|
||||
TBaseImagePreset,
|
||||
TBaseImageRootfsType,
|
||||
TFirecrackerArch,
|
||||
} from './interfaces/index.js';
|
||||
import { SmartVMError } from './interfaces/index.js';
|
||||
|
||||
const FIRECRACKER_CI_BUCKET_URL = 'https://s3.amazonaws.com/spec.ccfc.min';
|
||||
const DEFAULT_MAX_STORED_BASE_IMAGES = 2;
|
||||
const LTS_CI_VERSION = 'v1.7';
|
||||
const LTS_FIRECRACKER_VERSION = 'v1.7.0';
|
||||
|
||||
type TShellExecResult = Awaited<ReturnType<InstanceType<typeof plugins.smartshell.Smartshell>['execSpawn']>>;
|
||||
|
||||
interface IResolvedBaseImageSource {
|
||||
preset: TBaseImagePreset;
|
||||
arch: TFirecrackerArch;
|
||||
ciVersion: string;
|
||||
firecrackerVersion: string;
|
||||
kernelKey?: string;
|
||||
rootfsKey?: string;
|
||||
kernelUrl?: string;
|
||||
rootfsUrl?: string;
|
||||
kernelSourcePath?: string;
|
||||
rootfsSourcePath?: string;
|
||||
kernelFileName?: string;
|
||||
rootfsFileName?: string;
|
||||
expectedKernelSha256?: string;
|
||||
expectedRootfsSha256?: string;
|
||||
expectedKernelBytes?: number;
|
||||
expectedRootfsBytes?: number;
|
||||
rootfsType: TBaseImageRootfsType;
|
||||
rootfsIsReadOnly: boolean;
|
||||
bundleId: string;
|
||||
bootArgs: string;
|
||||
source: IBaseImageBundle['source'];
|
||||
}
|
||||
|
||||
function getErrorMessage(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and retains Firecracker CI base images for integration testing.
|
||||
*/
|
||||
export class BaseImageManager {
|
||||
private arch: TFirecrackerArch;
|
||||
private cacheDir: string;
|
||||
private maxStoredBaseImages: number;
|
||||
private hostedManifestUrl?: string;
|
||||
private hostedManifestPath?: string;
|
||||
private shell: InstanceType<typeof plugins.smartshell.Smartshell>;
|
||||
|
||||
constructor(options: IBaseImageManagerOptions = {}) {
|
||||
this.arch = options.arch || 'x86_64';
|
||||
this.cacheDir = options.cacheDir || plugins.path.join(plugins.os.tmpdir(), '.smartvm', 'base-images');
|
||||
this.maxStoredBaseImages = options.maxStoredBaseImages ?? DEFAULT_MAX_STORED_BASE_IMAGES;
|
||||
this.hostedManifestUrl = options.hostedManifestUrl;
|
||||
this.hostedManifestPath = options.hostedManifestPath;
|
||||
if (!Number.isInteger(this.maxStoredBaseImages) || this.maxStoredBaseImages < 1) {
|
||||
throw new SmartVMError(
|
||||
'maxStoredBaseImages must be a positive integer',
|
||||
'INVALID_BASE_IMAGE_CACHE_LIMIT',
|
||||
);
|
||||
}
|
||||
this.shell = new plugins.smartshell.Smartshell({ executor: 'bash' });
|
||||
}
|
||||
|
||||
public getCacheDir(): string {
|
||||
return this.cacheDir;
|
||||
}
|
||||
|
||||
public getMaxStoredBaseImages(): number {
|
||||
return this.maxStoredBaseImages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a base image bundle exists locally and return its paths.
|
||||
*/
|
||||
public async ensureBaseImage(options: IEnsureBaseImageOptions = {}): Promise<IBaseImageBundle> {
|
||||
const source = await this.resolveBaseImageSource(options);
|
||||
const bundleDir = plugins.path.join(this.cacheDir, source.bundleId);
|
||||
const manifestPath = this.getManifestPath(bundleDir);
|
||||
|
||||
const cachedBundle = options.forceDownload ? undefined : await this.readCompleteBundle(bundleDir);
|
||||
if (cachedBundle) {
|
||||
const updatedBundle = {
|
||||
...cachedBundle,
|
||||
lastAccessedAt: new Date().toISOString(),
|
||||
};
|
||||
await this.writeBundleManifest(updatedBundle);
|
||||
await this.pruneBaseImageCache(updatedBundle.bundleId);
|
||||
return updatedBundle;
|
||||
}
|
||||
|
||||
await plugins.fs.promises.mkdir(bundleDir, { recursive: true });
|
||||
|
||||
const kernelFileName = source.kernelFileName || this.getSourceFileName(source.kernelUrl || source.kernelSourcePath || source.kernelKey!, 'vmlinux');
|
||||
const rootfsFileName = source.rootfsFileName || this.getSourceFileName(source.rootfsUrl || source.rootfsSourcePath || source.rootfsKey!, `rootfs.${source.rootfsType}`);
|
||||
const kernelPath = this.resolveBundleFilePath(bundleDir, kernelFileName);
|
||||
const rootfsPath = this.resolveBundleFilePath(bundleDir, rootfsFileName);
|
||||
|
||||
try {
|
||||
await this.prepareArtifact({
|
||||
url: source.kernelUrl || (source.kernelKey ? this.keyToUrl(source.kernelKey) : undefined),
|
||||
sourcePath: source.kernelSourcePath,
|
||||
targetPath: kernelPath,
|
||||
expectedSha256: source.expectedKernelSha256,
|
||||
expectedBytes: source.expectedKernelBytes,
|
||||
});
|
||||
await this.prepareArtifact({
|
||||
url: source.rootfsUrl || (source.rootfsKey ? this.keyToUrl(source.rootfsKey) : undefined),
|
||||
sourcePath: source.rootfsSourcePath,
|
||||
targetPath: rootfsPath,
|
||||
expectedSha256: source.expectedRootfsSha256,
|
||||
expectedBytes: source.expectedRootfsBytes,
|
||||
});
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const bundle: IBaseImageBundle = {
|
||||
preset: source.preset,
|
||||
arch: source.arch,
|
||||
ciVersion: source.ciVersion,
|
||||
firecrackerVersion: source.firecrackerVersion,
|
||||
bundleId: source.bundleId,
|
||||
bundleDir,
|
||||
kernelImagePath: kernelPath,
|
||||
rootfsPath,
|
||||
rootfsType: source.rootfsType,
|
||||
rootfsIsReadOnly: source.rootfsIsReadOnly,
|
||||
bootArgs: source.bootArgs,
|
||||
source: source.source,
|
||||
checksums: {
|
||||
kernelSha256: await this.sha256File(kernelPath),
|
||||
rootfsSha256: await this.sha256File(rootfsPath),
|
||||
},
|
||||
sizes: {
|
||||
kernelBytes: (await plugins.fs.promises.stat(kernelPath)).size,
|
||||
rootfsBytes: (await plugins.fs.promises.stat(rootfsPath)).size,
|
||||
},
|
||||
createdAt: now,
|
||||
lastAccessedAt: now,
|
||||
};
|
||||
|
||||
await this.writeBundleManifest(bundle);
|
||||
await this.pruneBaseImageCache(bundle.bundleId);
|
||||
return bundle;
|
||||
} catch (err) {
|
||||
await plugins.fs.promises.rm(bundleDir, { recursive: true, force: true });
|
||||
throw new SmartVMError(
|
||||
`Failed to prepare base image bundle ${source.bundleId}: ${getErrorMessage(err)}`,
|
||||
'BASE_IMAGE_PREPARE_FAILED',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune cached base image bundles according to the retention limit.
|
||||
*/
|
||||
public async pruneBaseImageCache(keepBundleId?: string): Promise<string[]> {
|
||||
await plugins.fs.promises.mkdir(this.cacheDir, { recursive: true });
|
||||
const bundles = await this.listCachedBundles();
|
||||
bundles.sort((a, b) => {
|
||||
if (keepBundleId) {
|
||||
if (a.bundleId === keepBundleId) return -1;
|
||||
if (b.bundleId === keepBundleId) return 1;
|
||||
}
|
||||
return Date.parse(b.lastAccessedAt) - Date.parse(a.lastAccessedAt);
|
||||
});
|
||||
|
||||
const evicted: string[] = [];
|
||||
for (const bundle of bundles.slice(this.maxStoredBaseImages)) {
|
||||
console.warn(
|
||||
`[smartvm] Base image cache stores at most ${this.maxStoredBaseImages} bundle(s). ` +
|
||||
`Evicting ${bundle.bundleId} from ${bundle.bundleDir}. Configure maxStoredBaseImages to change this behavior.`,
|
||||
);
|
||||
await plugins.fs.promises.rm(bundle.bundleDir, { recursive: true, force: true });
|
||||
evicted.push(bundle.bundleId);
|
||||
}
|
||||
return evicted;
|
||||
}
|
||||
|
||||
private async resolveBaseImageSource(options: IEnsureBaseImageOptions): Promise<IResolvedBaseImageSource> {
|
||||
const arch = options.arch || this.arch;
|
||||
const manifestUrl = options.manifestUrl || this.hostedManifestUrl;
|
||||
const manifestPath = options.manifestPath || this.hostedManifestPath;
|
||||
if (manifestUrl || manifestPath) {
|
||||
return this.resolveHostedManifestSource({ arch, manifestUrl, manifestPath });
|
||||
}
|
||||
|
||||
const preset = options.preset || 'latest';
|
||||
if (preset === 'hosted') {
|
||||
throw new SmartVMError(
|
||||
'The hosted base image preset requires manifestUrl, manifestPath, or a manager-level hosted manifest option',
|
||||
'BASE_IMAGE_MANIFEST_FAILED',
|
||||
);
|
||||
}
|
||||
const firecrackerVersion = preset === 'latest'
|
||||
? await this.getLatestFirecrackerVersion()
|
||||
: LTS_FIRECRACKER_VERSION;
|
||||
const ciVersion = preset === 'latest'
|
||||
? firecrackerVersion.split('.').slice(0, 2).join('.')
|
||||
: LTS_CI_VERSION;
|
||||
|
||||
const keys = await this.listCiKeys(ciVersion, arch);
|
||||
const kernelKey = this.selectKernelKey(keys);
|
||||
const rootfsKey = this.selectRootfsKey(keys);
|
||||
const rootfsType = rootfsKey.endsWith('.ext4') ? 'ext4' : 'squashfs';
|
||||
const bundleId = this.buildBundleId(preset, ciVersion, arch, kernelKey, rootfsKey);
|
||||
|
||||
return {
|
||||
preset,
|
||||
arch,
|
||||
ciVersion,
|
||||
firecrackerVersion,
|
||||
kernelKey,
|
||||
rootfsKey,
|
||||
rootfsType,
|
||||
rootfsIsReadOnly: rootfsType === 'squashfs',
|
||||
bundleId,
|
||||
bootArgs: this.buildBootArgs(arch, rootfsType),
|
||||
source: {
|
||||
type: 'firecracker-ci',
|
||||
bucketUrl: FIRECRACKER_CI_BUCKET_URL,
|
||||
kernelKey,
|
||||
rootfsKey,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async resolveHostedManifestSource(options: {
|
||||
arch: TFirecrackerArch;
|
||||
manifestUrl?: string;
|
||||
manifestPath?: string;
|
||||
}): Promise<IResolvedBaseImageSource> {
|
||||
const manifest = await this.loadHostedManifest(options);
|
||||
this.validateHostedManifest(manifest, options.arch);
|
||||
this.getArtifactSource(manifest.kernel, 'kernel');
|
||||
this.getArtifactSource(manifest.rootfs, 'rootfs');
|
||||
|
||||
return {
|
||||
preset: 'hosted',
|
||||
arch: manifest.arch,
|
||||
ciVersion: 'hosted',
|
||||
firecrackerVersion: manifest.firecrackerVersion,
|
||||
kernelUrl: manifest.kernel.url,
|
||||
rootfsUrl: manifest.rootfs.url,
|
||||
kernelSourcePath: manifest.kernel.path,
|
||||
rootfsSourcePath: manifest.rootfs.path,
|
||||
kernelFileName: manifest.kernel.fileName,
|
||||
rootfsFileName: manifest.rootfs.fileName,
|
||||
expectedKernelSha256: manifest.kernel.sha256,
|
||||
expectedRootfsSha256: manifest.rootfs.sha256,
|
||||
expectedKernelBytes: manifest.kernel.sizeBytes,
|
||||
expectedRootfsBytes: manifest.rootfs.sizeBytes,
|
||||
rootfsType: manifest.rootfsType,
|
||||
rootfsIsReadOnly: manifest.rootfsIsReadOnly ?? manifest.rootfsType === 'squashfs',
|
||||
bundleId: this.sanitizeBundleId(manifest.bundleId),
|
||||
bootArgs: manifest.bootArgs || this.buildBootArgs(manifest.arch, manifest.rootfsType),
|
||||
source: {
|
||||
type: 'hosted-manifest',
|
||||
manifestUrl: options.manifestUrl,
|
||||
manifestPath: options.manifestPath,
|
||||
kernelUrl: manifest.kernel.url,
|
||||
rootfsUrl: manifest.rootfs.url,
|
||||
kernelSourcePath: manifest.kernel.path,
|
||||
rootfsSourcePath: manifest.rootfs.path,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async getLatestFirecrackerVersion(): Promise<string> {
|
||||
try {
|
||||
const result = await this.shell.execSpawn('curl', [
|
||||
'-fsSLI',
|
||||
'-o',
|
||||
'/dev/null',
|
||||
'-w',
|
||||
'%{url_effective}',
|
||||
'https://github.com/firecracker-microvm/firecracker/releases/latest',
|
||||
], { silent: true });
|
||||
if (result.exitCode !== 0) {
|
||||
const output = (result.stderr || result.stdout || '').trim();
|
||||
throw new Error(`curl exited with code ${result.exitCode}${output ? `: ${output}` : ''}`);
|
||||
}
|
||||
|
||||
const match = result.stdout.trim().match(/\/releases\/tag\/(v\d+\.\d+\.\d+)$/);
|
||||
if (!match) {
|
||||
throw new Error(`Could not parse latest Firecracker release URL: ${result.stdout.trim()}`);
|
||||
}
|
||||
return match[1];
|
||||
} catch (err) {
|
||||
throw new SmartVMError(
|
||||
`Failed to resolve latest Firecracker version: ${getErrorMessage(err)}`,
|
||||
'VERSION_FETCH_FAILED',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async loadHostedManifest(options: {
|
||||
manifestUrl?: string;
|
||||
manifestPath?: string;
|
||||
}): Promise<IBaseImageHostedManifest> {
|
||||
try {
|
||||
let raw: string;
|
||||
if (options.manifestPath) {
|
||||
raw = await plugins.fs.promises.readFile(options.manifestPath, 'utf8');
|
||||
} else if (options.manifestUrl) {
|
||||
const response = await plugins.SmartRequest.create()
|
||||
.url(options.manifestUrl)
|
||||
.get();
|
||||
raw = await response.text();
|
||||
} else {
|
||||
throw new Error('manifestUrl or manifestPath is required');
|
||||
}
|
||||
return JSON.parse(raw) as IBaseImageHostedManifest;
|
||||
} catch (err) {
|
||||
throw new SmartVMError(
|
||||
`Failed to load hosted base image manifest: ${getErrorMessage(err)}`,
|
||||
'BASE_IMAGE_MANIFEST_FAILED',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private validateHostedManifest(manifest: IBaseImageHostedManifest, expectedArch: TFirecrackerArch): void {
|
||||
if (manifest.schemaVersion !== 1) {
|
||||
throw new SmartVMError(
|
||||
'Hosted base image manifest schemaVersion must be 1',
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
if (!manifest.bundleId || !/^[a-zA-Z0-9._-]+$/.test(manifest.bundleId)) {
|
||||
throw new SmartVMError(
|
||||
'Hosted base image manifest bundleId must use only letters, numbers, dot, underscore, and dash',
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
if (manifest.arch !== expectedArch) {
|
||||
throw new SmartVMError(
|
||||
`Hosted base image arch '${manifest.arch}' does not match requested arch '${expectedArch}'`,
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
if (!manifest.firecrackerVersion || !/^v\d+\.\d+\.\d+$/.test(manifest.firecrackerVersion)) {
|
||||
throw new SmartVMError(
|
||||
'Hosted base image manifest firecrackerVersion must look like v1.15.1',
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
if (manifest.rootfsType !== 'ext4' && manifest.rootfsType !== 'squashfs') {
|
||||
throw new SmartVMError(
|
||||
'Hosted base image manifest rootfsType must be ext4 or squashfs',
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
this.validateArtifactManifest(manifest.kernel, 'kernel');
|
||||
this.validateArtifactManifest(manifest.rootfs, 'rootfs');
|
||||
}
|
||||
|
||||
private validateArtifactManifest(artifact: IBaseImageArtifactManifest, label: string): void {
|
||||
this.getArtifactSource(artifact, label);
|
||||
if (artifact.fileName !== undefined) {
|
||||
this.validateArtifactFileName(artifact.fileName, label);
|
||||
}
|
||||
if (artifact.url && !artifact.sha256) {
|
||||
throw new SmartVMError(
|
||||
`Hosted base image manifest ${label} artifact with url requires sha256`,
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
if (artifact.sha256 !== undefined && !/^[a-fA-F0-9]{64}$/.test(artifact.sha256)) {
|
||||
throw new SmartVMError(
|
||||
`Hosted base image manifest ${label} artifact sha256 must be a 64 character hex string`,
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
if (
|
||||
artifact.sizeBytes !== undefined &&
|
||||
(!Number.isInteger(artifact.sizeBytes) || artifact.sizeBytes < 0)
|
||||
) {
|
||||
throw new SmartVMError(
|
||||
`Hosted base image manifest ${label} artifact sizeBytes must be a non-negative integer`,
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private validateArtifactFileName(fileName: string, label: string): void {
|
||||
if (
|
||||
!fileName ||
|
||||
fileName === '.' ||
|
||||
fileName === '..' ||
|
||||
fileName !== plugins.path.basename(fileName) ||
|
||||
!/^[a-zA-Z0-9._-]+$/.test(fileName)
|
||||
) {
|
||||
throw new SmartVMError(
|
||||
`Hosted base image manifest ${label} artifact fileName must be a plain file name using letters, numbers, dot, underscore, and dash`,
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private getArtifactSource(artifact: { url?: string; path?: string }, label: string): string {
|
||||
if (!artifact.url && !artifact.path) {
|
||||
throw new SmartVMError(
|
||||
`Hosted base image manifest ${label} artifact requires url or path`,
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
if (artifact.url && artifact.path) {
|
||||
throw new SmartVMError(
|
||||
`Hosted base image manifest ${label} artifact must not set both url and path`,
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
return artifact.url || artifact.path!;
|
||||
}
|
||||
|
||||
private getSourceFileName(source: string, fallback: string): string {
|
||||
let fileName: string;
|
||||
try {
|
||||
fileName = plugins.path.basename(new URL(source).pathname);
|
||||
} catch {
|
||||
fileName = plugins.path.basename(source);
|
||||
}
|
||||
return this.sanitizeFileName(fileName || fallback);
|
||||
}
|
||||
|
||||
private resolveBundleFilePath(bundleDir: string, fileName: string): string {
|
||||
const resolvedBundleDir = plugins.path.resolve(bundleDir);
|
||||
const resolvedFilePath = plugins.path.resolve(resolvedBundleDir, this.sanitizeFileName(fileName));
|
||||
if (!this.isPathInside(resolvedBundleDir, resolvedFilePath)) {
|
||||
throw new SmartVMError(
|
||||
`Resolved base image artifact path escapes bundle directory: ${fileName}`,
|
||||
'INVALID_BASE_IMAGE_MANIFEST',
|
||||
);
|
||||
}
|
||||
return resolvedFilePath;
|
||||
}
|
||||
|
||||
private sanitizeFileName(fileName: string): string {
|
||||
const sanitized = plugins.path.basename(fileName).replace(/[^a-zA-Z0-9._-]/g, '_');
|
||||
if (!sanitized || sanitized === '.' || sanitized === '..') {
|
||||
return 'artifact';
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
private sanitizeBundleId(bundleId: string): string {
|
||||
return bundleId.replace(/[^a-zA-Z0-9._-]/g, '_');
|
||||
}
|
||||
|
||||
private async listCiKeys(ciVersion: string, arch: TFirecrackerArch): Promise<string[]> {
|
||||
const prefix = `firecracker-ci/${ciVersion}/${arch}/`;
|
||||
try {
|
||||
const response = await plugins.SmartRequest.create()
|
||||
.url(`${FIRECRACKER_CI_BUCKET_URL}/?prefix=${encodeURIComponent(prefix)}&list-type=2`)
|
||||
.get();
|
||||
const body = await response.text();
|
||||
const keys = Array.from(body.matchAll(/<Key>([^<]+)<\/Key>/g)).map((match) => this.decodeXml(match[1]));
|
||||
if (keys.length === 0) {
|
||||
throw new Error(`No Firecracker CI artifacts found for ${ciVersion}/${arch}`);
|
||||
}
|
||||
return keys;
|
||||
} catch (err) {
|
||||
throw new SmartVMError(
|
||||
`Failed to list Firecracker CI artifacts for ${ciVersion}/${arch}: ${getErrorMessage(err)}`,
|
||||
'BASE_IMAGE_RESOLVE_FAILED',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private selectKernelKey(keys: string[]): string {
|
||||
const kernelKeys = keys.filter((key) => /\/vmlinux-\d+\.\d+\.\d+$/.test(key) && !key.includes('/debug/'));
|
||||
if (kernelKeys.length === 0) {
|
||||
throw new SmartVMError('No suitable Firecracker CI kernel image found', 'BASE_IMAGE_RESOLVE_FAILED');
|
||||
}
|
||||
return kernelKeys.sort((a, b) => this.compareKernelKeys(a, b)).at(-1)!;
|
||||
}
|
||||
|
||||
private selectRootfsKey(keys: string[]): string {
|
||||
const ext4Keys = keys.filter((key) => /\/ubuntu-[^/]+\.ext4$/.test(key));
|
||||
if (ext4Keys.length > 0) {
|
||||
return ext4Keys.sort((a, b) => a.localeCompare(b, undefined, { numeric: true })).at(-1)!;
|
||||
}
|
||||
const squashfsKeys = keys.filter((key) => /\/ubuntu-[^/]+\.squashfs$/.test(key));
|
||||
if (squashfsKeys.length > 0) {
|
||||
return squashfsKeys.sort((a, b) => a.localeCompare(b, undefined, { numeric: true })).at(-1)!;
|
||||
}
|
||||
throw new SmartVMError('No suitable Firecracker CI rootfs image found', 'BASE_IMAGE_RESOLVE_FAILED');
|
||||
}
|
||||
|
||||
private compareKernelKeys(a: string, b: string): number {
|
||||
const aParts = this.extractKernelVersion(a);
|
||||
const bParts = this.extractKernelVersion(b);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (aParts[i] !== bParts[i]) {
|
||||
return aParts[i] - bParts[i];
|
||||
}
|
||||
}
|
||||
return a.localeCompare(b);
|
||||
}
|
||||
|
||||
private extractKernelVersion(key: string): [number, number, number] {
|
||||
const match = key.match(/vmlinux-(\d+)\.(\d+)\.(\d+)$/);
|
||||
if (!match) {
|
||||
return [0, 0, 0];
|
||||
}
|
||||
return [Number(match[1]), Number(match[2]), Number(match[3])];
|
||||
}
|
||||
|
||||
private buildBundleId(
|
||||
preset: TBaseImagePreset,
|
||||
ciVersion: string,
|
||||
arch: TFirecrackerArch,
|
||||
kernelKey: string,
|
||||
rootfsKey: string,
|
||||
): string {
|
||||
const rawId = [
|
||||
preset,
|
||||
ciVersion,
|
||||
arch,
|
||||
plugins.path.basename(kernelKey),
|
||||
plugins.path.basename(rootfsKey),
|
||||
].join('-');
|
||||
return this.sanitizeBundleId(rawId);
|
||||
}
|
||||
|
||||
private buildBootArgs(arch: TFirecrackerArch, rootfsType: TBaseImageRootfsType): string {
|
||||
const args = ['console=ttyS0', 'reboot=k', 'panic=1', 'pci=off'];
|
||||
if (arch === 'aarch64') {
|
||||
args.unshift('keep_bootcon');
|
||||
}
|
||||
if (rootfsType === 'squashfs') {
|
||||
args.push('ro', 'rootfstype=squashfs');
|
||||
}
|
||||
return args.join(' ');
|
||||
}
|
||||
|
||||
private keyToUrl(key: string): string {
|
||||
return `${FIRECRACKER_CI_BUCKET_URL}/${key}`;
|
||||
}
|
||||
|
||||
private async prepareArtifact(options: {
|
||||
url?: string;
|
||||
sourcePath?: string;
|
||||
targetPath: string;
|
||||
expectedSha256?: string;
|
||||
expectedBytes?: number;
|
||||
}): Promise<void> {
|
||||
if (options.sourcePath) {
|
||||
await plugins.fs.promises.copyFile(options.sourcePath, options.targetPath);
|
||||
} else if (options.url) {
|
||||
await this.downloadFile(options.url, options.targetPath);
|
||||
} else {
|
||||
throw new Error('Artifact requires url or sourcePath');
|
||||
}
|
||||
|
||||
const stat = await plugins.fs.promises.stat(options.targetPath);
|
||||
if (options.expectedBytes !== undefined && stat.size !== options.expectedBytes) {
|
||||
throw new Error(
|
||||
`Artifact ${options.targetPath} size mismatch: expected ${options.expectedBytes}, got ${stat.size}`,
|
||||
);
|
||||
}
|
||||
if (options.expectedSha256) {
|
||||
const actualSha256 = await this.sha256File(options.targetPath);
|
||||
if (actualSha256.toLowerCase() !== options.expectedSha256.toLowerCase()) {
|
||||
throw new Error(
|
||||
`Artifact ${options.targetPath} SHA256 mismatch: expected ${options.expectedSha256}, got ${actualSha256}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadFile(url: string, targetPath: string): Promise<TShellExecResult> {
|
||||
await plugins.fs.promises.mkdir(plugins.path.dirname(targetPath), { recursive: true });
|
||||
const tempPath = `${targetPath}.download`;
|
||||
await plugins.fs.promises.rm(tempPath, { force: true });
|
||||
const result = await this.shell.execSpawn('curl', ['-fSL', '-o', tempPath, url], { silent: true });
|
||||
if (result.exitCode !== 0) {
|
||||
const output = (result.stderr || result.stdout || '').trim();
|
||||
throw new Error(`curl failed for ${url} with code ${result.exitCode}${output ? `: ${output}` : ''}`);
|
||||
}
|
||||
await plugins.fs.promises.rename(tempPath, targetPath);
|
||||
return result;
|
||||
}
|
||||
|
||||
private async sha256File(filePath: string): Promise<string> {
|
||||
const hash = plugins.crypto.createHash('sha256');
|
||||
const stream = plugins.fs.createReadStream(filePath);
|
||||
for await (const chunk of stream) {
|
||||
hash.update(chunk);
|
||||
}
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
private async readCompleteBundle(bundleDir: string): Promise<IBaseImageBundle | undefined> {
|
||||
const manifestPath = this.getManifestPath(bundleDir);
|
||||
try {
|
||||
const bundle = {
|
||||
...await this.readBundleManifest(manifestPath),
|
||||
bundleDir,
|
||||
};
|
||||
await this.verifyCachedBundle(bundle);
|
||||
return bundle;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private async verifyCachedBundle(bundle: IBaseImageBundle): Promise<void> {
|
||||
if (!this.isPathInside(bundle.bundleDir, bundle.kernelImagePath)) {
|
||||
throw new Error(`Cached kernel path escapes bundle directory: ${bundle.kernelImagePath}`);
|
||||
}
|
||||
if (!this.isPathInside(bundle.bundleDir, bundle.rootfsPath)) {
|
||||
throw new Error(`Cached rootfs path escapes bundle directory: ${bundle.rootfsPath}`);
|
||||
}
|
||||
if (!bundle.checksums?.kernelSha256 || !bundle.checksums?.rootfsSha256) {
|
||||
throw new Error(`Cached bundle ${bundle.bundleId} is missing checksums`);
|
||||
}
|
||||
if (bundle.sizes?.kernelBytes === undefined || bundle.sizes.rootfsBytes === undefined) {
|
||||
throw new Error(`Cached bundle ${bundle.bundleId} is missing sizes`);
|
||||
}
|
||||
|
||||
const [kernelStat, rootfsStat] = await Promise.all([
|
||||
plugins.fs.promises.stat(bundle.kernelImagePath),
|
||||
plugins.fs.promises.stat(bundle.rootfsPath),
|
||||
]);
|
||||
if (kernelStat.size !== bundle.sizes.kernelBytes) {
|
||||
throw new Error(`Cached kernel size mismatch for bundle ${bundle.bundleId}`);
|
||||
}
|
||||
if (rootfsStat.size !== bundle.sizes.rootfsBytes) {
|
||||
throw new Error(`Cached rootfs size mismatch for bundle ${bundle.bundleId}`);
|
||||
}
|
||||
|
||||
const [kernelSha256, rootfsSha256] = await Promise.all([
|
||||
this.sha256File(bundle.kernelImagePath),
|
||||
this.sha256File(bundle.rootfsPath),
|
||||
]);
|
||||
if (kernelSha256.toLowerCase() !== bundle.checksums.kernelSha256.toLowerCase()) {
|
||||
throw new Error(`Cached kernel SHA256 mismatch for bundle ${bundle.bundleId}`);
|
||||
}
|
||||
if (rootfsSha256.toLowerCase() !== bundle.checksums.rootfsSha256.toLowerCase()) {
|
||||
throw new Error(`Cached rootfs SHA256 mismatch for bundle ${bundle.bundleId}`);
|
||||
}
|
||||
}
|
||||
|
||||
private isPathInside(baseDir: string, candidatePath: string): boolean {
|
||||
const resolvedBase = plugins.path.resolve(baseDir);
|
||||
const resolvedCandidate = plugins.path.resolve(candidatePath);
|
||||
return resolvedCandidate === resolvedBase || resolvedCandidate.startsWith(`${resolvedBase}${plugins.path.sep}`);
|
||||
}
|
||||
|
||||
private getManifestPath(bundleDir: string): string {
|
||||
return plugins.path.join(bundleDir, 'manifest.json');
|
||||
}
|
||||
|
||||
private async readBundleManifest(manifestPath: string): Promise<IBaseImageBundle> {
|
||||
const raw = await plugins.fs.promises.readFile(manifestPath, 'utf8');
|
||||
return JSON.parse(raw) as IBaseImageBundle;
|
||||
}
|
||||
|
||||
private async writeBundleManifest(bundle: IBaseImageBundle): Promise<void> {
|
||||
await plugins.fs.promises.mkdir(bundle.bundleDir, { recursive: true });
|
||||
await plugins.fs.promises.writeFile(
|
||||
this.getManifestPath(bundle.bundleDir),
|
||||
`${JSON.stringify(bundle, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
private async listCachedBundles(): Promise<IBaseImageBundle[]> {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await plugins.fs.promises.readdir(this.cacheDir);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
const bundles: IBaseImageBundle[] = [];
|
||||
for (const entry of entries) {
|
||||
const bundleDir = plugins.path.join(this.cacheDir, entry);
|
||||
try {
|
||||
const stat = await plugins.fs.promises.stat(bundleDir);
|
||||
if (!stat.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
const bundle = await this.readBundleManifest(this.getManifestPath(bundleDir));
|
||||
bundles.push({
|
||||
...bundle,
|
||||
bundleDir,
|
||||
});
|
||||
} catch {
|
||||
// Ignore incomplete cache entries.
|
||||
}
|
||||
}
|
||||
return bundles;
|
||||
}
|
||||
|
||||
private decodeXml(value: string): string {
|
||||
return value
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user