serve.zone/onebox
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

feat(appstore): add service volumes and published ports

Browse Source
This commit is contained in:
Jürgen Kunz
2026-05-24 07:28:18 +00:00
parent e6ebac76b4
commit 5228eeaa23
26 changed files with 1790 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ts/classes/appstore-types.ts
+37 -1
View File
@@ -19,6 +19,27 @@ export interface ICatalogApp {
tags?: string[];
}
export interface IAppCatalogVolume {
name?: string;
source?: string;
mountPath: string;
driver?: string;
readOnly?: boolean;
backup?: boolean;
options?: Record<string, string>;
}
export type TAppCatalogVolumeSpec = string | IAppCatalogVolume;
export interface IAppCatalogPublishedPort {
targetPort: number;
targetPortEnd?: number;
publishedPort?: number;
publishedPortEnd?: number;
protocol?: 'tcp' | 'udp';
hostIp?: string;
}
export interface IAppMeta {
id: string;
name: string;
@@ -35,7 +56,8 @@ export interface IAppVersionConfig {
image: string;
port: number;
envVars?: Array<{ key: string; value: string; description: string; required?: boolean }>;
volumes?: string[];
volumes?: TAppCatalogVolumeSpec[];
publishedPorts?: IAppCatalogPublishedPort[];
platformRequirements?: {
mongodb?: boolean;
s3?: boolean;
@@ -46,6 +68,17 @@ export interface IAppVersionConfig {
minOneboxVersion?: string;
}
export interface IAppInstallOptions {
appId: string;
version?: string;
serviceName: string;
domain?: string;
port?: number;
publishedPorts?: IAppCatalogPublishedPort[];
envVars?: Record<string, string>;
autoDNS?: boolean;
}
export interface IMigrationContext {
service: {
name: string;
@@ -61,6 +94,9 @@ export interface IMigrationResult {
success: boolean;
envVars?: Record<string, string>;
image?: string;
port?: number;
volumes?: IAppCatalogVolume[];
publishedPorts?: IAppCatalogPublishedPort[];
warnings: string[];
}
ts/classes/appstore.ts
+225 -2
View File
@@ -8,6 +8,8 @@ import type {
ICatalog,
ICatalogApp,
IAppMeta,
IAppCatalogVolume,
IAppInstallOptions,
IAppVersionConfig,
IMigrationContext,
IMigrationResult,
@@ -16,7 +18,8 @@ import type {
import { logger } from '../logging.ts';
import { getErrorMessage } from '../utils/error.ts';
import type { Onebox } from './onebox.ts';
import type { IService } from '../types.ts';
import type { IService, IServiceVolume } from '../types.ts';
import { projectInfo } from '../info.ts';
export class AppStoreManager {
private oneboxRef: Onebox;
@@ -90,12 +93,50 @@ export class AppStoreManager {
*/
async getAppVersionConfig(appId: string, version: string): Promise<IAppVersionConfig> {
try {
return await this.fetchJson(`apps/${appId}/versions/${version}/config.json`) as IAppVersionConfig;
const config = await this.fetchJson(`apps/${appId}/versions/${version}/config.json`) as IAppVersionConfig;
this.validateAppVersionConfig(config, `${appId}@${version}`);
return config;
} catch (error) {
throw new Error(`Failed to fetch config for ${appId}@${version}: ${getErrorMessage(error)}`);
}
}
async installApp(optionsArg: IAppInstallOptions): Promise<IService> {
this.validateInstallOptions(optionsArg);
const appMeta = await this.getAppMeta(optionsArg.appId);
const version = optionsArg.version || appMeta.latestVersion;
const config = await this.getAppVersionConfig(optionsArg.appId, version);
this.assertRuntimeCompatibility(config);
const servicePort = optionsArg.port || config.port;
this.assertValidPort(servicePort, 'install service port');
const volumes = this.normalizeVolumes(config.volumes);
const publishedPorts = optionsArg.publishedPorts || config.publishedPorts || [];
this.validatePublishedPorts(publishedPorts, `${optionsArg.appId}@${version}`);
const envVars = this.getAppStoreEnvVars(config, optionsArg.envVars || {});
if (this.requiresTemplateValue(envVars, 'SERVICE_DOMAIN') && !optionsArg.domain) {
throw new Error('A domain is required because the app template uses ${SERVICE_DOMAIN}');
}
return await this.oneboxRef.services.deployService({
name: optionsArg.serviceName,
image: config.image,
port: servicePort,
domain: optionsArg.domain,
autoDNS: optionsArg.autoDNS,
envVars,
volumes,
publishedPorts,
enableMongoDB: Boolean(config.platformRequirements?.mongodb),
enableS3: Boolean(config.platformRequirements?.s3),
enableClickHouse: Boolean(config.platformRequirements?.clickhouse),
enableRedis: Boolean(config.platformRequirements?.redis),
enableMariaDB: Boolean(config.platformRequirements?.mariadb),
appTemplateId: optionsArg.appId,
appTemplateVersion: version,
});
}
/**
* Compare deployed services against catalog to find those with available upgrades
*/
@@ -165,6 +206,9 @@ export class AppStoreManager {
return {
success: true,
image: config.image,
port: config.port,
volumes: this.normalizeVolumes(config.volumes),
publishedPorts: config.publishedPorts,
envVars: undefined, // Keep existing env vars
warnings: [],
};
@@ -265,6 +309,18 @@ export class AppStoreManager {
updates.image = migrationResult.image;
}
if (migrationResult.port) {
updates.port = migrationResult.port;
}
if (migrationResult.volumes) {
updates.volumes = migrationResult.volumes;
}
if (migrationResult.publishedPorts) {
updates.publishedPorts = migrationResult.publishedPorts;
}
if (migrationResult.envVars) {
// Merge: migration result provides base, user overrides preserved
const mergedEnvVars = { ...migrationResult.envVars };
@@ -332,4 +388,171 @@ export class AppStoreManager {
}
return response.text();
}
public normalizeVolumes(volumesArg: IAppVersionConfig['volumes'] = []): IServiceVolume[] {
return volumesArg.map((volumeArg, indexArg): IAppCatalogVolume => {
if (typeof volumeArg === 'string') {
return { mountPath: volumeArg };
}
return volumeArg;
}).map((volumeArg, indexArg) => {
this.validateVolume(volumeArg, `volume ${indexArg + 1}`);
return volumeArg;
});
}
public validateAppVersionConfig(configArg: IAppVersionConfig, labelArg = 'app config'): void {
if (!configArg || typeof configArg !== 'object') {
throw new Error(`Invalid ${labelArg}: config must be an object`);
}
if (!configArg.image || typeof configArg.image !== 'string') {
throw new Error(`Invalid ${labelArg}: image is required`);
}
if (configArg.image.endsWith(':latest')) {
logger.warn(`App template ${labelArg} uses a mutable ':latest' image tag`);
}
this.assertValidPort(configArg.port, `${labelArg} port`);
for (const envVar of configArg.envVars || []) {
if (!envVar.key || !/^[A-Z_][A-Z0-9_]*$/.test(envVar.key)) {
throw new Error(`Invalid ${labelArg}: env var key '${envVar.key}' is not valid`);
}
if (envVar.value !== undefined && typeof envVar.value !== 'string') {
throw new Error(`Invalid ${labelArg}: env var '${envVar.key}' value must be a string`);
}
}
this.normalizeVolumes(configArg.volumes);
this.validatePublishedPorts(configArg.publishedPorts || [], labelArg);
}
private validateInstallOptions(optionsArg: IAppInstallOptions): void {
if (!optionsArg.appId || !/^[a-z0-9][a-z0-9-]*$/.test(optionsArg.appId)) {
throw new Error(`Invalid app id: ${optionsArg.appId}`);
}
if (!optionsArg.serviceName || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,119}$/.test(optionsArg.serviceName)) {
throw new Error(`Invalid service name: ${optionsArg.serviceName}`);
}
if (optionsArg.port !== undefined) {
this.assertValidPort(optionsArg.port, 'install service port');
}
if (optionsArg.publishedPorts) {
this.validatePublishedPorts(optionsArg.publishedPorts, `install options for ${optionsArg.appId}`);
}
}
private validateVolume(volumeArg: IAppCatalogVolume, labelArg: string): void {
if (!volumeArg.mountPath || !volumeArg.mountPath.startsWith('/')) {
throw new Error(`Invalid ${labelArg}: mountPath must be an absolute path`);
}
if (volumeArg.mountPath.includes(':')) {
throw new Error(`Invalid ${labelArg}: mountPath must not contain ':'`);
}
if ((volumeArg.source || volumeArg.name)?.includes(':')) {
throw new Error(`Invalid ${labelArg}: source/name must not contain ':'`);
}
}
private validatePublishedPorts(
publishedPortsArg: IAppVersionConfig['publishedPorts'] = [],
labelArg: string,
): void {
const seenPublishedPorts = new Set<string>();
for (const portArg of publishedPortsArg) {
const protocol = portArg.protocol || 'tcp';
const targetStart = portArg.targetPort;
const targetEnd = portArg.targetPortEnd || targetStart;
const publishedStart = portArg.publishedPort || targetStart;
const publishedEnd = portArg.publishedPortEnd || (publishedStart + (targetEnd - targetStart));
const hostIp = portArg.hostIp || '0.0.0.0';
if (!['tcp', 'udp'].includes(protocol)) {
throw new Error(`Invalid ${labelArg}: published port protocol '${protocol}' is not supported`);
}
this.assertValidPort(targetStart, `${labelArg} targetPort`);
this.assertValidPort(targetEnd, `${labelArg} targetPortEnd`);
this.assertValidPort(publishedStart, `${labelArg} publishedPort`);
this.assertValidPort(publishedEnd, `${labelArg} publishedPortEnd`);
if (targetEnd < targetStart || publishedEnd < publishedStart) {
throw new Error(`Invalid ${labelArg}: published port ranges must be ascending`);
}
if ((targetEnd - targetStart) !== (publishedEnd - publishedStart)) {
throw new Error(`Invalid ${labelArg}: target and published port ranges must have the same size`);
}
if ((targetEnd - targetStart) > 1000) {
throw new Error(`Invalid ${labelArg}: published port ranges may not exceed 1001 ports`);
}
for (let offset = 0; offset <= targetEnd - targetStart; offset++) {
const publishedPort = publishedStart + offset;
const publishedKey = `${hostIp}/${protocol}/${publishedPort}`;
const wildcardKey = `0.0.0.0/${protocol}/${publishedPort}`;
const conflictsWithWildcard = hostIp === '0.0.0.0'
? Array.from(seenPublishedPorts).some((keyArg) => keyArg.endsWith(`/${protocol}/${publishedPort}`))
: seenPublishedPorts.has(wildcardKey);
if (seenPublishedPorts.has(publishedKey) || conflictsWithWildcard) {
throw new Error(`Invalid ${labelArg}: duplicate published port ${hostIp}:${publishedPort}/${protocol}`);
}
seenPublishedPorts.add(publishedKey);
}
}
}
private assertValidPort(portArg: number, labelArg: string): void {
if (!Number.isInteger(portArg) || portArg < 1 || portArg > 65535) {
throw new Error(`Invalid ${labelArg}: ${portArg}. Expected an integer port between 1 and 65535.`);
}
}
private getAppStoreEnvVars(
configArg: IAppVersionConfig,
overridesArg: Record<string, string>,
): Record<string, string> {
const envVars: Record<string, string> = {};
const missingRequiredEnvVars: string[] = [];
for (const envVar of configArg.envVars || []) {
const value = overridesArg[envVar.key] ?? envVar.value ?? '';
if (envVar.required && !value) {
missingRequiredEnvVars.push(envVar.key);
}
envVars[envVar.key] = value;
}
for (const [key, value] of Object.entries(overridesArg)) {
envVars[key] = value;
}
if (missingRequiredEnvVars.length > 0) {
throw new Error(
`Missing required app env var(s): ${missingRequiredEnvVars.join(', ')}`,
);
}
return envVars;
}
private requiresTemplateValue(envVarsArg: Record<string, string>, templateNameArg: string): boolean {
return Object.values(envVarsArg).some((value) => value.includes(`\${${templateNameArg}}`));
}
private assertRuntimeCompatibility(configArg: IAppVersionConfig): void {
if (!configArg.minOneboxVersion) return;
if (this.compareVersions(projectInfo.version, configArg.minOneboxVersion) < 0) {
throw new Error(
`App requires Onebox >= ${configArg.minOneboxVersion}; current version is ${projectInfo.version}`,
);
}
}
private compareVersions(versionAArg: string, versionBArg: string): number {
const normalize = (versionArg: string) => versionArg.replace(/^v/, '').split('.').map((partArg) => Number(partArg) || 0);
const a = normalize(versionAArg);
const b = normalize(versionBArg);
for (let i = 0; i < Math.max(a.length, b.length); i++) {
const diff = (a[i] || 0) - (b[i] || 0);
if (diff !== 0) return diff > 0 ? 1 : -1;
}
return 0;
}
}
ts/classes/backup-manager.ts
+86 -4
View File
@@ -185,7 +185,12 @@ export class BackupManager {
await this.exportDockerImage(service.image, `${tempDir}/data/image/image.tar`);
}
// 4. Build ingest items from temp directory files
// 4. Export declared service volume data when the volume opts into backup.
if (service.volumes?.some((volumeArg) => volumeArg.backup !== false)) {
await this.exportServiceVolumes(service, tempDir);
}
// 5. Build ingest items from temp directory files
const items: Array<{ stream: NodeJS.ReadableStream; name: string; type?: string }> = [];
// Service config
@@ -218,6 +223,19 @@ export class BackupManager {
}
}
const volumeDataDir = `${tempDir}/data/volumes`;
try {
for await (const filePath of this.walkFiles(volumeDataDir)) {
items.push({
stream: plugins.nodeFs.createReadStream(filePath),
name: plugins.path.relative(tempDir, filePath).replaceAll('\\', '/'),
type: 'volume',
});
}
} catch {
// No service volume data was exported.
}
// Docker image
if (includeImage && service.image) {
const imagePath = `${tempDir}/data/image/image.tar`;
@@ -233,7 +251,7 @@ export class BackupManager {
}
}
// 5. Build snapshot tags
// 6. Build snapshot tags
const tags: Record<string, string> = {
serviceName: service.name,
serviceId: String(service.id),
@@ -245,10 +263,10 @@ export class BackupManager {
tags.scheduleId = String(options.scheduleId);
}
// 6. Ingest multi-item snapshot into containerarchive
// 7. Ingest multi-item snapshot into containerarchive
const snapshot = await this.archive.ingestMulti(items, { tags });
// 7. Store backup record in database
// 8. Store backup record in database
const backup: IBackup = {
serviceId: service.id!,
serviceName: service.name,
@@ -675,6 +693,8 @@ export class BackupManager {
registry: serviceConfig.registry,
port: serviceConfig.port,
domain: serviceConfig.domain,
volumes: serviceConfig.volumes,
publishedPorts: serviceConfig.publishedPorts,
useOneboxRegistry: serviceConfig.useOneboxRegistry,
registryRepository: serviceConfig.registryRepository,
registryImageTag: serviceConfig.registryImageTag,
@@ -705,6 +725,8 @@ export class BackupManager {
port: serviceConfig.port,
domain: options.mode === 'clone' ? undefined : serviceConfig.domain,
envVars: serviceConfig.envVars,
volumes: serviceConfig.volumes,
publishedPorts: serviceConfig.publishedPorts,
useOneboxRegistry: serviceConfig.useOneboxRegistry,
registryImageTag: serviceConfig.registryImageTag,
autoUpdateOnPush: serviceConfig.autoUpdateOnPush,
@@ -729,6 +751,8 @@ export class BackupManager {
}
}
await this.restoreServiceVolumes(service, serviceConfig.volumes || [], tempDir, warnings);
// Cleanup
await Deno.remove(tempDir, { recursive: true });
@@ -791,6 +815,8 @@ export class BackupManager {
image: service.image,
registry: service.registry,
envVars: service.envVars,
volumes: service.volumes,
publishedPorts: service.publishedPorts,
port: service.port,
domain: service.domain,
useOneboxRegistry: service.useOneboxRegistry,
@@ -802,6 +828,62 @@ export class BackupManager {
};
}
private getVolumeBackupName(volumeArg: { mountPath: string }, indexArg: number): string {
const safeMountPath = volumeArg.mountPath
.replace(/^\/+/, '')
.replace(/\/+$/g, '')
.replace(/[^a-zA-Z0-9_.-]+/g, '-') || 'root';
return `${String(indexArg).padStart(3, '0')}-${safeMountPath}`;
}
private async exportServiceVolumes(serviceArg: IService, tempDirArg: string): Promise<void> {
if (!serviceArg.containerID) {
throw new Error(`Cannot export service volumes for ${serviceArg.name}: service has no container ID`);
}
const volumes = (serviceArg.volumes || []).filter((volumeArg) => volumeArg.backup !== false);
for (let i = 0; i < volumes.length; i++) {
const volume = volumes[i];
const backupName = this.getVolumeBackupName(volume, i);
const outputPath = `${tempDirArg}/data/volumes/${backupName}`;
await Deno.mkdir(outputPath, { recursive: true });
await this.copyFromContainer(serviceArg.containerID, `${volume.mountPath}/.`, outputPath);
logger.info(`Exported volume ${volume.mountPath} for service ${serviceArg.name}`);
}
}
private async restoreServiceVolumes(
serviceArg: IService,
volumesArg: NonNullable<IBackupServiceConfig['volumes']>,
tempDirArg: string,
warningsArg: string[],
): Promise<void> {
if (!serviceArg.containerID) {
if (volumesArg.some((volumeArg) => volumeArg.backup !== false)) {
warningsArg.push(`Could not restore service volumes for ${serviceArg.name}: service has no container ID`);
}
return;
}
const volumes = volumesArg.filter((volumeArg) => volumeArg.backup !== false);
for (let i = 0; i < volumes.length; i++) {
const volume = volumes[i];
const backupName = this.getVolumeBackupName(volume, i);
const inputPath = `${tempDirArg}/data/volumes/${backupName}`;
try {
await Deno.stat(inputPath);
} catch {
continue;
}
try {
await this.copyToContainer(`${inputPath}/.`, serviceArg.containerID, volume.mountPath);
logger.info(`Restored volume ${volume.mountPath} for service ${serviceArg.name}`);
} catch (error) {
warningsArg.push(`Volume restore failed for ${volume.mountPath}: ${getErrorMessage(error)}`);
}
}
}
/**
* Export MongoDB database
*/
ts/classes/docker.ts
+274 -15
View File
@@ -5,14 +5,258 @@
*/
import * as plugins from '../plugins.ts';
import type { IService, IContainerStats } from '../types.ts';
import type { IService, IContainerStats, IServicePublishedPort } from '../types.ts';
import { logger } from '../logging.ts';
import { getErrorMessage } from '../utils/error.ts';
type TExpandedPublishedPort = Required<Pick<
IServicePublishedPort,
'targetPort' | 'publishedPort' | 'protocol' | 'hostIp'
>>;
export class OneboxDockerManager {
private dockerClient: InstanceType<typeof plugins.docker.Docker> | null = null;
private networkName = 'onebox-network';
private getDockerSafeName(valueArg: string, maxLengthArg = 120): string {
const safeName = valueArg
.replace(/[^a-zA-Z0-9_.-]+/g, '-')
.replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, '')
.slice(0, maxLengthArg)
.replace(/[^a-zA-Z0-9]+$/g, '');
return safeName || 'data';
}
private getServiceVolumeSource(serviceArg: IService, mountPathArg: string, requestedSourceArg?: string): string {
if (requestedSourceArg) {
return this.getDockerSafeName(requestedSourceArg);
}
const mountName = this.getDockerSafeName(mountPathArg.replace(/^\/+/, '').replace(/\/+$/g, ''), 40);
return this.getDockerSafeName(`onebox-${serviceArg.name}-${mountName}`);
}
private getStandaloneVolumeBinds(serviceArg: IService): string[] {
return (serviceArg.volumes || []).map((volumeArg) => {
const source = this.getServiceVolumeSource(serviceArg, volumeArg.mountPath, volumeArg.source || volumeArg.name);
return `${source}:${volumeArg.mountPath}${volumeArg.readOnly ? ':ro' : ''}`;
});
}
private getSwarmVolumeMounts(serviceArg: IService): Array<Record<string, unknown>> {
return (serviceArg.volumes || []).map((volumeArg) => ({
Type: 'volume',
Source: this.getServiceVolumeSource(serviceArg, volumeArg.mountPath, volumeArg.source || volumeArg.name),
Target: volumeArg.mountPath,
ReadOnly: Boolean(volumeArg.readOnly),
VolumeOptions: {
DriverConfig: {
Name: volumeArg.driver || 'local',
Options: volumeArg.options || {},
},
Labels: {
'managed-by': 'onebox',
'onebox-service': serviceArg.name,
'onebox-mount-path': volumeArg.mountPath,
'onebox-backup': String(volumeArg.backup !== false),
},
},
}));
}
public validateServiceSpec(serviceArg: IService): void {
this.assertValidPort(serviceArg.port, `service port for ${serviceArg.name}`);
for (const volumeArg of serviceArg.volumes || []) {
if (!volumeArg.mountPath || !volumeArg.mountPath.startsWith('/')) {
throw new Error(`Volume mountPath for service ${serviceArg.name} must be an absolute path`);
}
if (volumeArg.mountPath.includes(':')) {
throw new Error(`Volume mountPath for service ${serviceArg.name} must not contain ':'`);
}
if ((volumeArg.source || volumeArg.name)?.includes(':')) {
throw new Error(`Volume source/name for service ${serviceArg.name} must not contain ':'`);
}
}
this.expandPublishedPorts(serviceArg);
}
private assertValidPort(portArg: number, labelArg: string): void {
if (!Number.isInteger(portArg) || portArg < 1 || portArg > 65535) {
throw new Error(`Invalid ${labelArg}: ${portArg}. Expected an integer port between 1 and 65535.`);
}
}
private expandPublishedPorts(serviceArg: IService): TExpandedPublishedPort[] {
const expandedPorts: TExpandedPublishedPort[] = [];
const seenPublishedPorts = new Set<string>();
for (const portArg of serviceArg.publishedPorts || []) {
const protocol = portArg.protocol || 'tcp';
const targetStart = portArg.targetPort;
const targetEnd = portArg.targetPortEnd || targetStart;
const publishedStart = portArg.publishedPort || targetStart;
const publishedEnd = portArg.publishedPortEnd || (publishedStart + (targetEnd - targetStart));
const hostIp = portArg.hostIp || '0.0.0.0';
if (!['tcp', 'udp'].includes(protocol)) {
throw new Error(`Invalid published port protocol for service ${serviceArg.name}: ${protocol}`);
}
this.assertValidPort(targetStart, `published targetPort for service ${serviceArg.name}`);
this.assertValidPort(targetEnd, `published targetPortEnd for service ${serviceArg.name}`);
this.assertValidPort(publishedStart, `published publishedPort for service ${serviceArg.name}`);
this.assertValidPort(publishedEnd, `published publishedPortEnd for service ${serviceArg.name}`);
if (targetEnd < targetStart) {
throw new Error(`Invalid target port range for service ${serviceArg.name}: ${targetStart}-${targetEnd}`);
}
if (publishedEnd < publishedStart) {
throw new Error(`Invalid published port range for service ${serviceArg.name}: ${publishedStart}-${publishedEnd}`);
}
if ((targetEnd - targetStart) !== (publishedEnd - publishedStart)) {
throw new Error(
`Published port range size must match target port range size for service ${serviceArg.name}`,
);
}
if (!this.isValidHostIp(hostIp)) {
throw new Error(`Invalid hostIp for service ${serviceArg.name}: ${hostIp}`);
}
for (let offset = 0; offset <= targetEnd - targetStart; offset++) {
const publishedPort = publishedStart + offset;
const publishedKey = `${hostIp}/${protocol}/${publishedPort}`;
const wildcardKey = `0.0.0.0/${protocol}/${publishedPort}`;
const conflictsWithWildcard = hostIp === '0.0.0.0'
? Array.from(seenPublishedPorts).some((keyArg) => keyArg.endsWith(`/${protocol}/${publishedPort}`))
: seenPublishedPorts.has(wildcardKey);
if (seenPublishedPorts.has(publishedKey) || conflictsWithWildcard) {
throw new Error(`Duplicate published port for service ${serviceArg.name}: ${hostIp}:${publishedPort}/${protocol}`);
}
seenPublishedPorts.add(publishedKey);
expandedPorts.push({
targetPort: targetStart + offset,
publishedPort,
protocol,
hostIp,
});
}
}
return expandedPorts;
}
private isValidHostIp(hostIpArg: string): boolean {
if (['0.0.0.0', '127.0.0.1', '::', '::1', 'localhost'].includes(hostIpArg)) return true;
if (/^(\d{1,3}\.){3}\d{1,3}$/.test(hostIpArg)) {
return hostIpArg.split('.').every((partArg) => Number(partArg) >= 0 && Number(partArg) <= 255);
}
return /^[0-9a-fA-F:]+$/.test(hostIpArg);
}
private async assertPublishedPortsAvailable(serviceArg: IService): Promise<void> {
const publishedPorts = this.expandPublishedPorts(serviceArg);
if (publishedPorts.length === 0) return;
await this.assertPublishedPortsNotUsedByDocker(serviceArg, publishedPorts);
await this.assertPublishedPortsNotUsedByHost(serviceArg, publishedPorts);
}
private async assertPublishedPortsNotUsedByDocker(
serviceArg: IService,
publishedPortsArg: TExpandedPublishedPort[],
): Promise<void> {
const requestedPorts = new Set(
publishedPortsArg.map((portArg) => `${portArg.protocol}/${portArg.publishedPort}`),
);
try {
const containersResponse = await this.dockerClient!.request('GET', '/containers/json?all=true', {});
if (containersResponse.statusCode === 200 && Array.isArray(containersResponse.body)) {
for (const containerArg of containersResponse.body) {
const labels = containerArg.Labels || {};
if (labels['onebox-service'] === serviceArg.name) continue;
for (const portArg of containerArg.Ports || []) {
if (!portArg.PublicPort || !portArg.Type) continue;
if (requestedPorts.has(`${portArg.Type}/${portArg.PublicPort}`)) {
throw new Error(
`Published port ${portArg.PublicPort}/${portArg.Type} is already used by container ${containerArg.Names?.[0] || containerArg.Id}`,
);
}
}
}
}
const servicesResponse = await this.dockerClient!.request('GET', '/services', {});
if (servicesResponse.statusCode === 200 && Array.isArray(servicesResponse.body)) {
for (const service of servicesResponse.body) {
if (service.Spec?.Name === `onebox-${serviceArg.name}`) continue;
for (const portArg of service.Endpoint?.Ports || []) {
if (!portArg.PublishedPort || !portArg.Protocol) continue;
if (requestedPorts.has(`${portArg.Protocol}/${portArg.PublishedPort}`)) {
throw new Error(
`Published port ${portArg.PublishedPort}/${portArg.Protocol} is already used by Docker service ${service.Spec?.Name || service.ID}`,
);
}
}
}
}
} catch (error) {
if (error instanceof Error && error.message.startsWith('Published port ')) throw error;
logger.warn(`Could not complete Docker published-port preflight: ${getErrorMessage(error)}`);
}
}
private async assertPublishedPortsNotUsedByHost(
serviceArg: IService,
publishedPortsArg: TExpandedPublishedPort[],
): Promise<void> {
for (const portArg of publishedPortsArg) {
try {
if (portArg.protocol === 'udp') {
await this.assertUdpPortAvailable(portArg.hostIp, portArg.publishedPort);
} else {
const listener = Deno.listen({ hostname: portArg.hostIp, port: portArg.publishedPort });
listener.close();
}
} catch (error) {
throw new Error(
`Published port ${portArg.hostIp}:${portArg.publishedPort}/${portArg.protocol} for service ${serviceArg.name} is not available: ${getErrorMessage(error)}`,
);
}
}
}
private async assertUdpPortAvailable(hostIpArg: string, portArg: number): Promise<void> {
const dgram = await import('node:dgram');
const socket = dgram.createSocket(hostIpArg.includes(':') ? 'udp6' : 'udp4');
await new Promise<void>((resolve, reject) => {
socket.once('error', reject);
socket.bind(portArg, hostIpArg, () => {
socket.close();
resolve();
});
});
}
private getStandalonePortConfig(serviceArg: IService): {
exposedPorts: Record<string, Record<string, never>>;
portBindings: Record<string, Array<{ HostIp: string; HostPort: string }>>;
} {
const exposedPorts: Record<string, Record<string, never>> = {
[`${serviceArg.port}/tcp`]: {},
};
const portBindings: Record<string, Array<{ HostIp: string; HostPort: string }>> = {
[`${serviceArg.port}/tcp`]: [],
};
for (const publishedPort of this.expandPublishedPorts(serviceArg)) {
const key = `${publishedPort.targetPort}/${publishedPort.protocol}`;
exposedPorts[key] = {};
portBindings[key] = [{ HostIp: publishedPort.hostIp, HostPort: String(publishedPort.publishedPort) }];
}
return { exposedPorts, portBindings };
}
/**
* Initialize Docker client and create onebox network
*/
@@ -122,6 +366,9 @@ export class OneboxDockerManager {
*/
async createContainer(service: IService): Promise<string> {
try {
this.validateServiceSpec(service);
await this.assertPublishedPortsAvailable(service);
// Check if Docker is in Swarm mode
let isSwarmMode = false;
try {
@@ -158,6 +405,8 @@ export class OneboxDockerManager {
env.push(`${key}=${value}`);
}
const portConfig = this.getStandalonePortConfig(service);
// Create container using Docker REST API directly
const response = await this.dockerClient!.request('POST', `/containers/create?name=onebox-${service.name}`, {
Image: fullImage,
@@ -166,18 +415,14 @@ export class OneboxDockerManager {
'managed-by': 'onebox',
'onebox-service': service.name,
},
ExposedPorts: {
[`${service.port}/tcp`]: {},
},
ExposedPorts: portConfig.exposedPorts,
HostConfig: {
NetworkMode: this.networkName,
RestartPolicy: {
Name: 'unless-stopped',
},
PortBindings: {
// Don't bind to host ports - nginx will proxy
[`${service.port}/tcp`]: [],
},
PortBindings: portConfig.portBindings,
Binds: this.getStandaloneVolumeBinds(service),
},
});
@@ -207,6 +452,25 @@ export class OneboxDockerManager {
env.push(`${key}=${value}`);
}
const expandedPublishedPorts = this.expandPublishedPorts(service);
const endpointPorts: Array<Record<string, unknown>> = [];
if (!expandedPublishedPorts.some((publishedPort) => publishedPort.protocol === 'tcp' && publishedPort.targetPort === service.port)) {
endpointPorts.push({
Protocol: 'tcp',
TargetPort: service.port,
PublishMode: 'host',
});
}
for (const publishedPort of expandedPublishedPorts) {
endpointPorts.push({
Protocol: publishedPort.protocol,
TargetPort: publishedPort.targetPort,
PublishedPort: publishedPort.publishedPort,
PublishMode: 'host',
});
}
// Create Swarm service using Docker REST API
const response = await this.dockerClient!.request('POST', '/services/create', {
Name: `onebox-${service.name}`,
@@ -218,6 +482,7 @@ export class OneboxDockerManager {
ContainerSpec: {
Image: fullImage,
Env: env,
Mounts: this.getSwarmVolumeMounts(service),
Labels: {
'managed-by': 'onebox',
'onebox-service': service.name,
@@ -239,13 +504,7 @@ export class OneboxDockerManager {
},
},
EndpointSpec: {
Ports: [
{
Protocol: 'tcp',
TargetPort: service.port,
PublishMode: 'host',
},
],
Ports: endpointPorts,
},
});
ts/classes/external-gateway.ts
+39
View File
@@ -133,6 +133,45 @@ export class ExternalGatewayManager {
}
await this.syncDomains();
await this.syncServiceRoutes();
}
public async syncServiceRoutes(): Promise<void> {
const services = this.database.getAllServices()
.filter((service) => service.domain && service.status === 'running');
const activeHostnames = new Set(services.map((service) => service.domain!));
for (const service of services) {
try {
await this.syncServiceRoute(service);
} catch (error) {
logger.warn(`Failed to sync external gateway route for ${service.domain}: ${getErrorMessage(error)}`);
}
}
await this.deleteStaleServiceRoutes(activeHostnames);
}
private async deleteStaleServiceRoutes(activeHostnamesArg: Set<string>): Promise<void> {
const records = await this.getGatewayDnsRecords();
const staleRecordsByHostname = new Map<string, IGatewayDnsRecord>();
for (const record of records) {
if (!record.hostname || activeHostnamesArg.has(record.hostname)) continue;
if (!record.routeId && !record.appId && !record.serviceName) continue;
staleRecordsByHostname.set(record.hostname, record);
}
for (const record of staleRecordsByHostname.values()) {
try {
await this.deleteServiceRoute({
name: record.serviceName || record.appId,
domain: record.hostname,
});
} catch (error) {
logger.warn(`Failed to delete stale external gateway route for ${record.hostname}: ${getErrorMessage(error)}`);
}
}
}
public async isConfigured(): Promise<boolean> {
ts/classes/managed-dcrouter.ts
+22
View File
@@ -227,6 +227,7 @@ export class ManagedDcRouterManager {
const image = this.getImage();
const token = await this.getAdminToken();
const dataDir = await this.getAbsoluteDataDir();
await this.writeManagedConfig(dataDir);
await this.oneboxRef.docker.pullImage(image);
@@ -234,6 +235,7 @@ export class ManagedDcRouterManager {
Image: image,
Env: [
`DCROUTER_BASE_DIR=${internalBaseDir}`,
`DCROUTER_CONFIG_PATH=${internalBaseDir}/managed-config.json`,
`DCROUTER_ADMIN_API_TOKEN=${token}`,
'DCROUTER_ADMIN_API_TOKEN_NAME=Onebox Managed Admin Token',
],
@@ -268,6 +270,26 @@ export class ManagedDcRouterManager {
logger.success(`Managed dcrouter container started: ${response.body.Id}`);
}
private async writeManagedConfig(dataDirArg: string): Promise<void> {
const configPath = plugins.path.join(dataDirArg, 'managed-config.json');
try {
const existingConfig = await Deno.readTextFile(configPath);
JSON.parse(existingConfig);
return;
} catch (error) {
if (!(error instanceof Deno.errors.NotFound)) {
throw new Error(`Managed dcrouter config exists but is not valid JSON: ${getErrorMessage(error)}`);
}
}
const config = {
smartProxyConfig: {
routes: [],
},
};
await Deno.writeTextFile(configPath, JSON.stringify(config, null, 2));
}
private async getExistingContainer(): Promise<any | null> {
const filters = encodeURIComponent(JSON.stringify({ name: [containerName] }));
const response = await this.dockerClient!.request('GET', `/containers/json?all=true&filters=${filters}`, {});
ts/classes/services.ts
+6
View File
@@ -95,6 +95,8 @@ export class OneboxServicesManager {
image: options.useOneboxRegistry ? imageToPull : options.image,
registry: options.registry,
envVars: options.envVars || {},
volumes: options.volumes || [],
publishedPorts: options.publishedPorts || [],
port: options.port,
domain: options.domain,
status: 'stopped',
@@ -578,6 +580,8 @@ export class OneboxServicesManager {
port?: number;
domain?: string;
envVars?: Record<string, string>;
volumes?: IService['volumes'];
publishedPorts?: IService['publishedPorts'];
}
): Promise<IService> {
try {
@@ -616,6 +620,8 @@ export class OneboxServicesManager {
if (updates.port !== undefined) updateData.port = updates.port;
if (updates.domain !== undefined) updateData.domain = updates.domain;
if (updates.envVars !== undefined) updateData.envVars = updates.envVars;
if (updates.volumes !== undefined) updateData.volumes = updates.volumes;
if (updates.publishedPorts !== undefined) updateData.publishedPorts = updates.publishedPorts;
this.database.updateService(service.id!, updateData);
ts/classes/smartproxy.ts
+34 -1
View File
@@ -175,8 +175,10 @@ export class SmartProxyManager {
throw new Error(`Failed to create SmartProxy service: HTTP ${response.statusCode} - ${JSON.stringify(response.body)}`);
}
logger.info(`SmartProxy service created: ${response.body.ID}`);
const serviceId = response.body.ID;
logger.info(`SmartProxy service created: ${serviceId}`);
await this.waitForServiceTaskRunning(serviceId);
await this.waitForReady();
this.serviceRunning = true;
await this.reloadConfig({ skipRunningCheck: true });
@@ -232,6 +234,37 @@ export class SmartProxyManager {
throw new Error('SmartProxy service failed to start within timeout');
}
private async waitForServiceTaskRunning(
serviceIdArg: string,
maxAttempts = 30,
intervalMs = 1000,
): Promise<void> {
let lastState = 'unknown';
for (let i = 0; i < maxAttempts; i++) {
const tasksResponse = await this.dockerClient!.request(
'GET',
`/tasks?filters=${encodeURIComponent(JSON.stringify({ service: [serviceIdArg] }))}`,
{},
);
if (tasksResponse.statusCode === 200 && Array.isArray(tasksResponse.body)) {
const tasks = tasksResponse.body;
const runningTask = tasks.find((task: any) => task.Status?.State === 'running');
if (runningTask) {
return;
}
const latestTask = tasks[0];
lastState = latestTask?.Status?.State || lastState;
}
await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
throw new Error(`SmartProxy service task did not reach running state (last state: ${lastState})`);
}
async stop(): Promise<void> {
try {
await this.ensureDockerClient();
ts/cli.ts
+10 -18
View File
@@ -214,33 +214,25 @@ async function handleAppStoreCommand(onebox: Onebox, subcommand: string, args: s
const appMeta = await onebox.appStore.getAppMeta(appId);
const version = getArg(args, '--version') || appMeta.latestVersion;
const config = await onebox.appStore.getAppVersionConfig(appId, version);
const serviceName = getArg(args, '--name') || appId;
const domain = getArg(args, '--domain');
const port = parseInt(getArg(args, '--port') || String(config.port), 10);
const envVars = getAppStoreEnvVars(config, parseEnvArgs(args));
const portArg = getArg(args, '--port');
const port = portArg ? parseInt(portArg, 10) : undefined;
const autoDNS = getBooleanArg(args, '--auto-dns', true);
requireValue(serviceName, '--name');
assertValidPort(port, '--port');
if (requiresTemplateValue(envVars, 'SERVICE_DOMAIN')) {
requireValue(domain, '--domain');
if (port !== undefined) {
assertValidPort(port, '--port');
}
const service = await onebox.services.deployService({
name: serviceName,
image: config.image,
port,
const service = await onebox.appStore.installApp({
appId,
version,
serviceName,
domain,
port,
autoDNS,
envVars,
enableMongoDB: Boolean(config.platformRequirements?.mongodb),
enableS3: Boolean(config.platformRequirements?.s3),
enableClickHouse: Boolean(config.platformRequirements?.clickhouse),
enableRedis: Boolean(config.platformRequirements?.redis),
enableMariaDB: Boolean(config.platformRequirements?.mariadb),
appTemplateId: appId,
appTemplateVersion: version,
envVars: parseEnvArgs(args),
});
logger.success(`Installed ${appMeta.name} ${version} as ${service.name}`);
ts/database/migrations/migration-016-service-volumes.ts
+11
View File
@@ -0,0 +1,11 @@
import { BaseMigration } from './base-migration.ts';
import type { TQueryFunction } from '../types.ts';
export class Migration016ServiceVolumes extends BaseMigration {
readonly version = 16;
readonly description = 'Add persistent volume declarations to services';
up(query: TQueryFunction): void {
query(`ALTER TABLE services ADD COLUMN volumes TEXT DEFAULT '[]'`);
}
}
ts/database/migrations/migration-017-service-published-ports.ts
+11
View File
@@ -0,0 +1,11 @@
import { BaseMigration } from './base-migration.ts';
import type { TQueryFunction } from '../types.ts';
export class Migration017ServicePublishedPorts extends BaseMigration {
readonly version = 17;
readonly description = 'Add raw published port declarations to services';
up(query: TQueryFunction): void {
query(`ALTER TABLE services ADD COLUMN published_ports TEXT DEFAULT '[]'`);
}
}
ts/database/migrations/migration-runner.ts
+4
View File
@@ -22,6 +22,8 @@ import { Migration012GfsRetention } from './migration-012-gfs-retention.ts';
import { Migration013AppTemplateVersion } from './migration-013-app-template-version.ts';
import { Migration014ContainerArchive } from './migration-014-containerarchive.ts';
import { Migration015SmartProxyPlatformService } from './migration-015-smartproxy-platform-service.ts';
import { Migration016ServiceVolumes } from './migration-016-service-volumes.ts';
import { Migration017ServicePublishedPorts } from './migration-017-service-published-ports.ts';
import type { BaseMigration } from './base-migration.ts';
export class MigrationRunner {
@@ -48,6 +50,8 @@ export class MigrationRunner {
new Migration013AppTemplateVersion(),
new Migration014ContainerArchive(),
new Migration015SmartProxyPlatformService(),
new Migration016ServiceVolumes(),
new Migration017ServicePublishedPorts(),
].sort((a, b) => a.version - b.version);
}
ts/database/repositories/service.repository.ts
+42 -8
View File
@@ -14,17 +14,19 @@ export class ServiceRepository extends BaseRepository {
const now = Date.now();
this.query(
`INSERT INTO services (
name, image, registry, env_vars, port, domain, container_id, status,
name, image, registry, env_vars, volumes, published_ports, port, domain, container_id, status,
created_at, updated_at,
use_onebox_registry, registry_repository, registry_image_tag,
auto_update_on_push, image_digest, platform_requirements,
app_template_id, app_template_version
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
service.name,
service.image,
service.registry || null,
JSON.stringify(service.envVars),
JSON.stringify(service.volumes || []),
JSON.stringify(service.publishedPorts || []),
service.port,
service.domain || null,
service.containerID || null,
@@ -82,6 +84,14 @@ export class ServiceRepository extends BaseRepository {
fields.push('env_vars = ?');
values.push(JSON.stringify(updates.envVars));
}
if (updates.volumes !== undefined) {
fields.push('volumes = ?');
values.push(JSON.stringify(updates.volumes));
}
if (updates.publishedPorts !== undefined) {
fields.push('published_ports = ?');
values.push(JSON.stringify(updates.publishedPorts));
}
if (updates.port !== undefined) {
fields.push('port = ?');
values.push(updates.port);
@@ -169,18 +179,42 @@ export class ServiceRepository extends BaseRepository {
}
}
let volumes = [];
const volumesRaw = row.volumes ?? row[20];
if (volumesRaw && volumesRaw !== 'undefined' && volumesRaw !== 'null') {
try {
volumes = JSON.parse(String(volumesRaw));
} catch (e) {
logger.warn(`Failed to parse volumes for service: ${getErrorMessage(e)}`);
volumes = [];
}
}
let publishedPorts = [];
const publishedPortsRaw = row.published_ports;
if (publishedPortsRaw && publishedPortsRaw !== 'undefined' && publishedPortsRaw !== 'null') {
try {
publishedPorts = JSON.parse(String(publishedPortsRaw));
} catch (e) {
logger.warn(`Failed to parse published_ports for service: ${getErrorMessage(e)}`);
publishedPorts = [];
}
}
return {
id: Number(row.id || row[0]),
name: String(row.name || row[1]),
image: String(row.image || row[2]),
registry: (row.registry || row[3]) ? String(row.registry || row[3]) : undefined,
envVars,
port: Number(row.port || row[5]),
domain: (row.domain || row[6]) ? String(row.domain || row[6]) : undefined,
containerID: (row.container_id || row[7]) ? String(row.container_id || row[7]) : undefined,
status: String(row.status || row[8]) as IService['status'],
createdAt: Number(row.created_at || row[9]),
updatedAt: Number(row.updated_at || row[10]),
volumes,
publishedPorts,
port: Number(row.port ?? row[6] ?? row[5]),
domain: (row.domain ?? row[7] ?? row[6]) ? String(row.domain ?? row[7] ?? row[6]) : undefined,
containerID: (row.container_id ?? row[8] ?? row[7]) ? String(row.container_id ?? row[8] ?? row[7]) : undefined,
status: String(row.status ?? row[9] ?? row[8]) as IService['status'],
createdAt: Number(row.created_at ?? row[10] ?? row[9]),
updatedAt: Number(row.updated_at ?? row[11] ?? row[10]),
useOneboxRegistry: row.use_onebox_registry ? Boolean(row.use_onebox_registry) : undefined,
registryRepository: row.registry_repository ? String(row.registry_repository) : undefined,
registryImageTag: row.registry_image_tag ? String(row.registry_image_tag) : undefined,
ts/opsserver/handlers/appstore.handler.ts
+11
View File
@@ -41,6 +41,17 @@ export class AppStoreHandler {
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_InstallAppTemplate>(
'installAppTemplate',
async (dataArg) => {
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
const service = await this.opsServerRef.oneboxRef.appStore.installApp(dataArg.install);
return { service };
},
),
);
// Get services with available upgrades
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetUpgradeableServices>(
ts/types.ts
+27
View File
@@ -9,6 +9,8 @@ export interface IService {
image: string;
registry?: string;
envVars: Record<string, string>;
volumes?: IServiceVolume[];
publishedPorts?: IServicePublishedPort[];
port: number;
domain?: string;
containerID?: string;
@@ -30,6 +32,27 @@ export interface IService {
appTemplateVersion?: string;
}
export interface IServiceVolume {
name?: string;
source?: string;
mountPath: string;
driver?: string;
readOnly?: boolean;
backup?: boolean;
options?: Record<string, string>;
}
export type TServicePortProtocol = 'tcp' | 'udp';
export interface IServicePublishedPort {
targetPort: number;
targetPortEnd?: number;
publishedPort?: number;
publishedPortEnd?: number;
protocol?: TServicePortProtocol;
hostIp?: string;
}
// Registry types
export interface IRegistry {
id?: number;
@@ -299,6 +322,8 @@ export interface IServiceDeployOptions {
image: string;
registry?: string;
envVars?: Record<string, string>;
volumes?: IServiceVolume[];
publishedPorts?: IServicePublishedPort[];
port: number;
domain?: string;
autoSSL?: boolean;
@@ -397,6 +422,8 @@ export interface IBackupServiceConfig {
image: string;
registry?: string;
envVars: Record<string, string>;
volumes?: IServiceVolume[];
publishedPorts?: IServicePublishedPort[];
port: number;
domain?: string;
useOneboxRegistry?: boolean;