diff --git a/ts/classes.server.ts b/ts/classes.server.ts index ddf0e19..07b5f7f 100644 --- a/ts/classes.server.ts +++ b/ts/classes.server.ts @@ -121,6 +121,12 @@ export class CloudlyServer { await this.cloudlyRef.baseOsManager.handleHeartbeatHttpRequest(req, res); }), ); + this.typedServer.server.addRoute( + '/baseos/v1/images/:buildId/download', + new plugins.typedserver.servertools.Handler('GET', async (req, res) => { + await this.cloudlyRef.baseOsManager.handleImageDownloadHttpRequest(req, res); + }), + ); await this.typedServer.start(); } diff --git a/ts/manager.baseos/classes.baseosimagebuild.ts b/ts/manager.baseos/classes.baseosimagebuild.ts new file mode 100644 index 0000000..7a05174 --- /dev/null +++ b/ts/manager.baseos/classes.baseosimagebuild.ts @@ -0,0 +1,79 @@ +import * as plugins from '../plugins.js'; + +export type TBaseOsImageArchitecture = 'amd64' | 'arm64' | 'rpi'; +export type TBaseOsImageBuildStatus = 'queued' | 'building' | 'ready' | 'failed' | 'cancelled'; + +export interface IBaseOsImageArtifact { + bucketName: string; + key: string; + filename: string; + contentType: string; + size: number; + sha256: string; + createdAt: number; +} + +export interface IBaseOsImageBuildPublic { + id: string; + data: { + status: TBaseOsImageBuildStatus; + architecture: TBaseOsImageArchitecture; + cloudlyUrl: string; + sourceImageUrl?: string; + ubuntuVersion?: string; + hostname?: string; + wifiSsid?: string; + sshPublicKey?: string; + artifact?: IBaseOsImageArtifact; + errorText?: string; + logs: string[]; + createdAt: number; + updatedAt: number; + startedAt?: number; + completedAt?: number; + expiresAt?: number; + }; +} + +@plugins.smartdata.managed() +export class BaseOsImageBuild extends plugins.smartdata.SmartDataDbDoc< + BaseOsImageBuild, + IBaseOsImageBuildPublic +> { + constructor(optionsArg?: IBaseOsImageBuildPublic & { + provisioningTokenHash?: string; + provisioningTokenConsumedAt?: number; + downloadTokenHash?: string; + downloadTokenExpiresAt?: number; + }) { + super(); + if (optionsArg) { + Object.assign(this, optionsArg); + } + } + + @plugins.smartdata.unI() + public id!: string; + + @plugins.smartdata.svDb() + public provisioningTokenHash!: string; + + @plugins.smartdata.svDb() + public provisioningTokenConsumedAt?: number; + + @plugins.smartdata.svDb() + public downloadTokenHash?: string; + + @plugins.smartdata.svDb() + public downloadTokenExpiresAt?: number; + + @plugins.smartdata.svDb() + public data!: IBaseOsImageBuildPublic['data']; + + public toPublicBuild(): IBaseOsImageBuildPublic { + return { + id: this.id, + data: this.data, + }; + } +} diff --git a/ts/manager.baseos/classes.baseosmanager.ts b/ts/manager.baseos/classes.baseosmanager.ts index 4600b25..3619370 100644 --- a/ts/manager.baseos/classes.baseosmanager.ts +++ b/ts/manager.baseos/classes.baseosmanager.ts @@ -1,4 +1,6 @@ import * as plugins from '../plugins.js'; +import * as crypto from 'node:crypto'; +import * as nodeStream from 'node:stream'; import type { Cloudly } from '../classes.cloudly.js'; import { logger } from '../logger.js'; import { @@ -7,6 +9,12 @@ import { type IBaseOsNodePublic, type IBaseOsRuntimeInfo, } from './classes.baseosnode.js'; +import { + BaseOsImageBuild, + type IBaseOsImageArtifact, + type IBaseOsImageBuildPublic, + type TBaseOsImageArchitecture, +} from './classes.baseosimagebuild.js'; interface IBaseOsRegisterRequest { joinToken?: string; @@ -42,6 +50,71 @@ interface IRequestGetBaseOsNodes { }; } +interface IBaseOsImageBuildRequest { + architecture: TBaseOsImageArchitecture; + cloudlyUrl?: string; + sourceImageUrl?: string; + ubuntuVersion?: string; + hostname?: string; + wifi?: { + ssid: string; + password?: string; + }; + sshPublicKey?: string; + artifactRetentionMs?: number; +} + +interface IRequestCreateBaseOsImageBuild { + method: 'createBaseOsImageBuild'; + request: { + identity: plugins.servezoneInterfaces.data.IIdentity; + build: IBaseOsImageBuildRequest; + }; + response: { + build: IBaseOsImageBuildPublic; + }; +} + +interface IRequestGetBaseOsImageBuilds { + method: 'getBaseOsImageBuilds'; + request: { + identity: plugins.servezoneInterfaces.data.IIdentity; + }; + response: { + builds: IBaseOsImageBuildPublic[]; + }; +} + +interface IRequestGetBaseOsImageBuildById { + method: 'getBaseOsImageBuildById'; + request: { + identity: plugins.servezoneInterfaces.data.IIdentity; + buildId: string; + }; + response: { + build: IBaseOsImageBuildPublic; + }; +} + +interface IRequestCreateBaseOsImageDownloadUrl { + method: 'createBaseOsImageDownloadUrl'; + request: { + identity: plugins.servezoneInterfaces.data.IIdentity; + buildId: string; + }; + response: { + url: string; + expiresAt: number; + }; +} + +interface ICoreBuildBaseOsImageResponse { + success: boolean; + artifact?: IBaseOsImageArtifact; + logs: string[]; + errorText?: string; +} + export class CloudlyBaseOsManager { public cloudlyRef: Cloudly; public typedRouter = new plugins.typedrequest.TypedRouter(); @@ -51,6 +124,7 @@ export class CloudlyBaseOsManager { } public CBaseOsNode = plugins.smartdata.setDefaultManagerForDoc(this, BaseOsNode); + public CBaseOsImageBuild = plugins.smartdata.setDefaultManagerForDoc(this, BaseOsImageBuild); constructor(cloudlyRefArg: Cloudly) { this.cloudlyRef = cloudlyRefArg; @@ -70,6 +144,64 @@ export class CloudlyBaseOsManager { }, ), ); + + this.typedRouter.addTypedHandler( + new plugins.typedrequest.TypedHandler( + 'createBaseOsImageBuild', + async (requestDataArg) => { + await plugins.smartguard.passGuardsOrReject( + { identity: requestDataArg.identity }, + [this.cloudlyRef.authManager.adminIdentityGuard], + ); + return { + build: await this.createImageBuild(requestDataArg.build), + }; + }, + ), + ); + + this.typedRouter.addTypedHandler( + new plugins.typedrequest.TypedHandler( + 'getBaseOsImageBuilds', + async (requestDataArg) => { + await plugins.smartguard.passGuardsOrReject( + { identity: requestDataArg.identity }, + [this.cloudlyRef.authManager.adminIdentityGuard], + ); + return { + builds: await this.getPublicImageBuilds(), + }; + }, + ), + ); + + this.typedRouter.addTypedHandler( + new plugins.typedrequest.TypedHandler( + 'getBaseOsImageBuildById', + async (requestDataArg) => { + await plugins.smartguard.passGuardsOrReject( + { identity: requestDataArg.identity }, + [this.cloudlyRef.authManager.adminIdentityGuard], + ); + return { + build: (await this.getImageBuildById(requestDataArg.buildId)).toPublicBuild(), + }; + }, + ), + ); + + this.typedRouter.addTypedHandler( + new plugins.typedrequest.TypedHandler( + 'createBaseOsImageDownloadUrl', + async (requestDataArg) => { + await plugins.smartguard.passGuardsOrReject( + { identity: requestDataArg.identity }, + [this.cloudlyRef.authManager.adminIdentityGuard], + ); + return await this.createImageDownloadUrl(requestDataArg.buildId); + }, + ), + ); } public async start() { @@ -112,6 +244,47 @@ export class CloudlyBaseOsManager { } } + public async handleImageDownloadHttpRequest( + reqArg: plugins.typedserver.Request, + resArg: plugins.typedserver.Response, + ) { + try { + const requestUrl = new URL((reqArg as any).originalUrl || reqArg.url || '/', 'http://localhost'); + const buildId = requestUrl.pathname.split('/').at(-2); + const token = requestUrl.searchParams.get('token'); + if (!buildId || !token) { + this.sendJson(resArg, 400, { errorText: 'build id or download token missing' }); + return; + } + const build = await this.getImageBuildById(buildId); + if (build.downloadTokenHash !== this.hashSecret(token) || (build.downloadTokenExpiresAt || 0) < Date.now()) { + this.sendJson(resArg, 403, { errorText: 'download token is invalid or expired' }); + return; + } + if (build.data.status !== 'ready' || !build.data.artifact) { + this.sendJson(resArg, 409, { errorText: 'image build is not ready' }); + return; + } + + const artifact = build.data.artifact; + const smartbucket = new plugins.smartbucket.SmartBucket({ + ...this.cloudlyRef.config.data.s3Descriptor, + port: Number((this.cloudlyRef.config.data.s3Descriptor as any).port || 443), + } as any); + const bucket = await smartbucket.getBucketByName(artifact.bucketName); + const artifactStream = await bucket.fastGetStream({ path: artifact.key }, 'nodestream'); + resArg.status(200); + resArg.setHeader('Content-Type', artifact.contentType || 'application/octet-stream'); + resArg.setHeader('Content-Length', String(artifact.size)); + resArg.setHeader('Content-Disposition', `attachment; filename="${artifact.filename}"`); + (artifactStream as nodeStream.Readable).pipe(resArg as any); + } catch (error) { + this.sendJson(resArg, 500, { + errorText: `BaseOS image download failed: ${(error as Error).message}`, + }); + } + } + public async registerNode( requestDataArg: IBaseOsRegisterRequest, ): Promise { @@ -133,6 +306,17 @@ export class CloudlyBaseOsManager { } } + const acceptedBuild = await this.consumeProvisioningToken(requestDataArg.joinToken); + if (acceptedBuild) { + const nodeToken = await this.cloudlyRef.authManager.createNewSecureToken(); + const node = await this.upsertNode(requestDataArg.status, nodeToken); + return { + accepted: true, + nodeId: node.id, + nodeToken, + }; + } + const configuredJoinToken = await this.cloudlyRef.settingsManager.getSetting('baseosJoinToken'); if (!configuredJoinToken) { return { @@ -193,6 +377,77 @@ export class CloudlyBaseOsManager { return nodes.map((nodeArg) => nodeArg.toPublicNode()); } + public async createImageBuild(buildRequestArg: IBaseOsImageBuildRequest) { + const s3Descriptor = this.cloudlyRef.config.data.s3Descriptor; + if (!s3Descriptor?.bucketName) { + throw new plugins.typedrequest.TypedResponseError('Cloudly S3 storage is required for BaseOS image builds'); + } + + const workerUrl = await this.cloudlyRef.settingsManager.getSetting('corebuildWorkerUrl'); + const workerToken = await this.cloudlyRef.settingsManager.getSetting('corebuildWorkerToken'); + if (!workerUrl) { + throw new plugins.typedrequest.TypedResponseError('corebuildWorkerUrl is not configured in Cloudly settings'); + } + + const now = Date.now(); + const buildId = await this.CBaseOsImageBuild.getNewId(); + const provisioningToken = await this.cloudlyRef.authManager.createNewSecureToken(); + const artifactRetentionMs = buildRequestArg.artifactRetentionMs || 1000 * 60 * 60 * 24 * 7; + const build = new this.CBaseOsImageBuild({ + id: buildId, + provisioningTokenHash: this.hashSecret(provisioningToken), + data: { + status: 'queued', + architecture: buildRequestArg.architecture, + cloudlyUrl: buildRequestArg.cloudlyUrl || this.getPublicCloudlyUrl(), + sourceImageUrl: buildRequestArg.sourceImageUrl, + ubuntuVersion: buildRequestArg.ubuntuVersion || '24.04', + hostname: buildRequestArg.hostname, + wifiSsid: buildRequestArg.wifi?.ssid, + sshPublicKey: buildRequestArg.sshPublicKey, + logs: [], + createdAt: now, + updatedAt: now, + expiresAt: now + artifactRetentionMs, + }, + }); + await build.save(); + + this.executeImageBuild(build, provisioningToken, buildRequestArg, workerUrl, workerToken).catch(async (error) => { + build.data.status = 'failed'; + build.data.errorText = (error as Error).message; + build.data.updatedAt = Date.now(); + build.data.completedAt = Date.now(); + await build.save(); + }); + + return build.toPublicBuild(); + } + + public async getPublicImageBuilds() { + const builds = await this.CBaseOsImageBuild.getInstances({}); + return builds + .sort((a, b) => b.data.createdAt - a.data.createdAt) + .map((buildArg) => buildArg.toPublicBuild()); + } + + public async createImageDownloadUrl(buildIdArg: string) { + const build = await this.getImageBuildById(buildIdArg); + if (build.data.status !== 'ready' || !build.data.artifact) { + throw new plugins.typedrequest.TypedResponseError('BaseOS image build is not ready'); + } + const token = await this.cloudlyRef.authManager.createNewSecureToken(); + const expiresAt = Date.now() + 1000 * 60 * 15; + build.downloadTokenHash = this.hashSecret(token); + build.downloadTokenExpiresAt = expiresAt; + build.data.updatedAt = Date.now(); + await build.save(); + return { + url: `/baseos/v1/images/${build.id}/download?token=${encodeURIComponent(token)}`, + expiresAt, + }; + } + private async upsertNode(statusArg: IBaseOsRuntimeInfo, nodeTokenArg: string) { const now = Date.now(); let node = await this.CBaseOsNode.getInstance({ @@ -224,6 +479,113 @@ export class CloudlyBaseOsManager { return node; } + private async executeImageBuild( + buildArg: BaseOsImageBuild, + provisioningTokenArg: string, + buildRequestArg: IBaseOsImageBuildRequest, + workerUrlArg: string, + workerTokenArg?: string, + ) { + buildArg.data.status = 'building'; + buildArg.data.startedAt = Date.now(); + buildArg.data.updatedAt = Date.now(); + await buildArg.save(); + + const artifactFilename = buildArg.data.architecture === 'amd64' + ? 'baseos.iso' + : buildArg.data.architecture === 'arm64' + ? 'baseos-arm64.iso' + : 'baseos-rpi.img'; + const artifactKey = `corebuild/baseos/${buildArg.id}/${artifactFilename}`; + const response = await fetch( + new URL('/corebuild/v1/jobs/baseos-image', workerUrlArg.endsWith('/') ? workerUrlArg : `${workerUrlArg}/`), + { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(workerTokenArg ? { authorization: `Bearer ${workerTokenArg}` } : {}), + }, + body: JSON.stringify({ + apiToken: workerTokenArg, + job: { + id: buildArg.id, + architecture: buildArg.data.architecture, + cloudlyUrl: buildArg.data.cloudlyUrl, + provisioningToken: provisioningTokenArg, + sourceImageUrl: buildArg.data.sourceImageUrl, + ubuntuVersion: buildArg.data.ubuntuVersion, + hostname: buildArg.data.hostname, + wifi: buildArg.data.wifiSsid + ? { + ssid: buildArg.data.wifiSsid, + password: buildRequestArg.wifi?.password, + } + : undefined, + sshPublicKey: buildArg.data.sshPublicKey, + s3Descriptor: this.cloudlyRef.config.data.s3Descriptor, + artifactKey, + }, + }), + }, + ); + const responseBody = await response.json() as ICoreBuildBaseOsImageResponse; + buildArg.data.logs = responseBody.logs || []; + if (!response.ok || !responseBody.success || !responseBody.artifact) { + buildArg.data.status = 'failed'; + buildArg.data.errorText = responseBody.errorText || `CoreBuild failed with HTTP ${response.status}`; + buildArg.data.updatedAt = Date.now(); + buildArg.data.completedAt = Date.now(); + await buildArg.save(); + return; + } + + buildArg.data.status = 'ready'; + buildArg.data.artifact = responseBody.artifact; + buildArg.data.updatedAt = Date.now(); + buildArg.data.completedAt = Date.now(); + await buildArg.save(); + } + + private async getImageBuildById(buildIdArg: string) { + const build = await this.CBaseOsImageBuild.getInstance({ id: buildIdArg }); + if (!build) { + throw new plugins.typedrequest.TypedResponseError(`BaseOS image build ${buildIdArg} not found`); + } + return build; + } + + private async consumeProvisioningToken(joinTokenArg?: string) { + if (!joinTokenArg) { + return null; + } + const tokenHash = this.hashSecret(joinTokenArg); + const builds = await this.CBaseOsImageBuild.getInstances({ + provisioningTokenHash: tokenHash, + }); + const build = builds.find((buildArg) => { + return !buildArg.provisioningTokenConsumedAt && (!buildArg.data.expiresAt || buildArg.data.expiresAt > Date.now()); + }); + if (!build) { + return null; + } + build.provisioningTokenConsumedAt = Date.now(); + build.data.updatedAt = Date.now(); + await build.save(); + return build; + } + + private hashSecret(secretArg: string) { + return crypto.createHash('sha256').update(secretArg).digest('hex'); + } + + private getPublicCloudlyUrl() { + const sslMode = this.cloudlyRef.config.data.sslMode; + const protocol = sslMode === 'none' ? 'http' : 'https'; + const port = this.cloudlyRef.config.data.publicPort; + const includePort = sslMode === 'none' && port && !['80', '443'].includes(port); + return `${protocol}://${this.cloudlyRef.config.data.publicUrl}${includePort ? `:${port}` : ''}`; + } + private async updateNodeRuntimeInfo( nodeArg: BaseOsNode, statusArg: IBaseOsRuntimeInfo, diff --git a/ts_web/elements/cloudly-dashboard.ts b/ts_web/elements/cloudly-dashboard.ts index 9a235ac..90b7384 100644 --- a/ts_web/elements/cloudly-dashboard.ts +++ b/ts_web/elements/cloudly-dashboard.ts @@ -12,6 +12,7 @@ import { state } from '@design.estate/dees-element'; import { CloudlyViewBackups } from './views/backups/index.js'; +import { CloudlyViewBaseOs } from './views/baseos/index.js'; import { CloudlyViewClusters } from './views/clusters/index.js'; import { CloudlyViewDbs } from './views/dbs/index.js'; import { CloudlyViewDeployments } from './views/deployments/index.js'; @@ -136,6 +137,11 @@ export class CloudlyDashboard extends DeesElement { iconName: 'lucide:Save', element: CloudlyViewBackups, }, + { + name: 'BaseOS', + iconName: 'lucide:HardDriveDownload', + element: CloudlyViewBaseOs, + }, { name: 'Fleet', iconName: 'lucide:Truck', diff --git a/ts_web/elements/index.ts b/ts_web/elements/index.ts index 71aeb9e..3bccccb 100644 --- a/ts_web/elements/index.ts +++ b/ts_web/elements/index.ts @@ -2,3 +2,4 @@ export * from './shared/index.js'; export * from './cloudly-dashboard.js'; export * from './views/secretgroups/index.js'; export * from './views/secretbundles/index.js'; +export * from './views/baseos/index.js'; diff --git a/ts_web/elements/views/baseos/index.ts b/ts_web/elements/views/baseos/index.ts new file mode 100644 index 0000000..5a4d2bc --- /dev/null +++ b/ts_web/elements/views/baseos/index.ts @@ -0,0 +1,206 @@ +import * as plugins from '../../../plugins.js'; +import * as appstate from '../../../appstate.js'; +import * as shared from '../../shared/index.js'; + +import { + DeesElement, + css, + cssManager, + customElement, + html, + state, +} from '@design.estate/dees-element'; + +type TBaseOsImageBuild = any; + +@customElement('cloudly-view-baseos') +export class CloudlyViewBaseOs extends DeesElement { + @state() private builds: TBaseOsImageBuild[] = []; + @state() private isLoading = false; + + private refreshTimer?: number; + + public static styles = [ + cssManager.defaultStyles, + shared.viewHostCss, + css` + .layout { + display: grid; + grid-template-columns: 420px 1fr; + gap: 16px; + padding: 24px 0; + } + + .builds { + display: flex; + flex-direction: column; + gap: 12px; + } + + .build { + border: 1px solid #2a2f3a; + border-radius: 12px; + padding: 16px; + background: #10151f; + } + + .build-head { + display: flex; + justify-content: space-between; + align-items: center; + gap: 12px; + } + + .meta { + color: #9aa4b2; + font-size: 13px; + margin-top: 8px; + } + + .logs { + margin-top: 12px; + max-height: 120px; + overflow: auto; + font-family: monospace; + font-size: 12px; + color: #bac4d1; + white-space: pre-wrap; + } + + @media (max-width: 900px) { + .layout { + grid-template-columns: 1fr; + } + } + `, + ]; + + public async connectedCallback() { + await super.connectedCallback(); + await this.loadBuilds(); + this.refreshTimer = window.setInterval(() => this.loadBuilds(), 5000); + } + + public async disconnectedCallback() { + await super.disconnectedCallback(); + if (this.refreshTimer) { + window.clearInterval(this.refreshTimer); + } + } + + public render() { + return html` + BaseOS Images +
+ + this.createBuild((eventArg.detail as any).data)}> + + + + + + + + + + +
+ ${this.builds.length === 0 + ? html`` + : this.builds.map((buildArg) => this.renderBuild(buildArg))} +
+
+ `; + } + + private renderBuild(buildArg: TBaseOsImageBuild) { + const data = buildArg.data; + return html` +
+
+
+ ${data.hostname || buildArg.id} +
${data.architecture} · ${data.cloudlyUrl}
+
+ +
+
+ ${data.artifact ? `${data.artifact.filename} · ${Math.round(data.artifact.size / 1024 / 1024)} MB · ${data.artifact.sha256.slice(0, 12)}...` : data.errorText || 'Waiting for artifact'} +
+ ${data.status === 'ready' + ? html` this.downloadBuild(buildArg.id)}>` + : ''} + ${data.logs?.length ? html`
${data.logs.slice(-8).join('\n')}
` : ''} +
+ `; + } + + private async loadBuilds() { + try { + const response = await this.fireBaseOsRequest('getBaseOsImageBuilds', {}); + this.builds = response.builds || []; + } catch (error) { + console.error('Failed to load BaseOS image builds:', error); + } + } + + private async createBuild(formDataArg: any) { + this.isLoading = true; + try { + const response = await this.fireBaseOsRequest('createBaseOsImageBuild', { + build: { + architecture: formDataArg.architecture || 'amd64', + cloudlyUrl: formDataArg.cloudlyUrl || window.location.origin, + hostname: formDataArg.hostname || undefined, + sourceImageUrl: formDataArg.sourceImageUrl || undefined, + wifi: formDataArg.wifiSsid + ? { + ssid: formDataArg.wifiSsid, + password: formDataArg.wifiPassword || undefined, + } + : undefined, + sshPublicKey: formDataArg.sshPublicKey || undefined, + }, + }); + this.builds = [response.build, ...this.builds]; + plugins.deesCatalog.DeesToast.createAndShow({ message: 'BaseOS image build queued', type: 'success' }); + } catch (error: any) { + plugins.deesCatalog.DeesToast.createAndShow({ message: `Failed to create image build: ${error.message}`, type: 'error' }); + } finally { + this.isLoading = false; + } + } + + private async downloadBuild(buildIdArg: string) { + const response = await this.fireBaseOsRequest('createBaseOsImageDownloadUrl', { + buildId: buildIdArg, + }); + window.location.href = response.url; + } + + private async fireBaseOsRequest(methodArg: string, payloadArg: Record) { + appstate.apiClient.identity = appstate.loginStatePart.getState()?.identity || null as any; + if (!appstate.apiClient.typedsocketClient) { + await appstate.apiClient.start(); + } + const request = appstate.apiClient.typedsocketClient.createTypedRequest(methodArg); + return await request.fire({ + identity: appstate.apiClient.identity, + ...payloadArg, + }); + } +} + +declare global { + interface HTMLElementTagNameMap { + 'cloudly-view-baseos': CloudlyViewBaseOs; + } +} diff --git a/ts_web/elements/views/settings/index.ts b/ts_web/elements/views/settings/index.ts index bfd9bef..b73703c 100644 --- a/ts_web/elements/views/settings/index.ts +++ b/ts_web/elements/views/settings/index.ts @@ -144,6 +144,13 @@ export class CloudlyViewSettings extends DeesElement { + +
+ + +
+ +
${this.renderProviderStatus('aws')}