feat: add built-in OCI registry

ts/classes.cloudly.ts

@@ -22,6 +22,7 @@ import { CloudlyNodeManager } from './manager.node/classes.nodemanager.js';
 import { CloudlyBaremetalManager } from './manager.baremetal/classes.baremetalmanager.js';
 import { ExternalApiManager } from './manager.status/statusmanager.js';
 import { ExternalRegistryManager } from './manager.externalregistry/index.js';
+import { CloudlyRegistryManager } from './manager.registry/index.js';
 import { ImageManager } from './manager.image/classes.imagemanager.js';
 import { ServiceManager } from './manager.service/classes.servicemanager.js';
 import { DeploymentManager } from './manager.deployment/classes.deploymentmanager.js';
@@ -65,6 +66,7 @@ export class Cloudly {
   public coreflowManager: CloudlyCoreflowManager;
   public externalApiManager: ExternalApiManager;
   public externalRegistryManager: ExternalRegistryManager;
+  public registryManager: CloudlyRegistryManager;
   public imageManager: ImageManager;
   public serviceManager: ServiceManager;
   public deploymentManager: DeploymentManager;
@@ -99,6 +101,7 @@ export class Cloudly {
     this.coreflowManager = new CloudlyCoreflowManager(this);
     this.externalApiManager = new ExternalApiManager(this);
     this.externalRegistryManager = new ExternalRegistryManager(this);
+    this.registryManager = new CloudlyRegistryManager(this);
     this.imageManager = new ImageManager(this);
     this.serviceManager = new ServiceManager(this);
     this.deploymentManager = new DeploymentManager(this);
@@ -133,6 +136,7 @@ export class Cloudly {
     await this.platformManager.start();
     await this.deploymentManager.start();
     await this.taskManager.init();
+    await this.registryManager.start();
     
     await this.cloudflareConnector.init();
     await this.letsencryptConnector.init();
@@ -157,6 +161,7 @@ export class Cloudly {
     await this.platformManager.stop();
     await this.deploymentManager.stop();
     await this.taskManager.stop();
+    await this.registryManager.stop();
     await this.externalRegistryManager.stop();
   }
 }

ts/classes.server.ts

@@ -93,6 +93,18 @@ export class CloudlyServer {
       preferredCompressionMethod: 'gzip',
     });
     this.typedServer.typedrouter.addTypedRouter(this.typedrouter);
+    this.typedServer.server.addRoute(
+      '/v2',
+      new plugins.typedserver.servertools.Handler('ALL', async (req, res) => {
+        await this.cloudlyRef.registryManager.handleHttpRequest(req, res);
+      }),
+    );
+    this.typedServer.server.addRoute(
+      '/v2/{*splat}',
+      new plugins.typedserver.servertools.Handler('ALL', async (req, res) => {
+        await this.cloudlyRef.registryManager.handleHttpRequest(req, res);
+      }),
+    );
     this.typedServer.server.addRoute(
       '/curlfresh/:scriptname',
       this.cloudlyRef.nodeManager.curlfreshInstance.handler,

ts/manager.registry/classes.registrymanager.ts

@@ -0,0 +1,258 @@
+import type { Cloudly } from '../classes.cloudly.js';
+import { logger } from '../logger.js';
+import * as plugins from '../plugins.js';
+
+type TAuthenticatedRegistryUser = {
+  userId: string;
+  username: string;
+  canWrite: boolean;
+};
+
+export class CloudlyRegistryManager {
+  private cloudlyRef: Cloudly;
+  private smartRegistry!: plugins.smartregistry.SmartRegistry;
+  private started = false;
+
+  constructor(cloudlyRefArg: Cloudly) {
+    this.cloudlyRef = cloudlyRefArg;
+  }
+
+  public async start() {
+    const publicRegistryUrl = this.getPublicRegistryUrl();
+    const registryJwtSecret = JSON.stringify(this.cloudlyRef.authManager.smartjwtInstance.getKeyPairAsJson());
+    const s3Descriptor = this.cloudlyRef.config.data.s3Descriptor;
+    if (!s3Descriptor?.bucketName) {
+      throw new Error('Cloudly registry requires an S3 bucketName');
+    }
+
+    this.smartRegistry = new plugins.smartregistry.SmartRegistry({
+      storage: s3Descriptor as plugins.smartregistry.IStorageConfig,
+      auth: {
+        jwtSecret: registryJwtSecret,
+        tokenStore: 'memory',
+        npmTokens: { enabled: false },
+        ociTokens: {
+          enabled: true,
+          realm: `${publicRegistryUrl}/v2/token`,
+          service: this.cloudlyRef.config.data.publicUrl || 'cloudly',
+        },
+        pypiTokens: { enabled: false },
+        rubygemsTokens: { enabled: false },
+      },
+      oci: {
+        enabled: true,
+        basePath: '/v2',
+        registryUrl: publicRegistryUrl,
+      },
+    });
+
+    await this.smartRegistry.init();
+    this.started = true;
+    logger.log('info', `Cloudly OCI registry available at ${publicRegistryUrl}/v2`);
+  }
+
+  public async stop() {
+    if (this.smartRegistry) {
+      this.smartRegistry.destroy();
+    }
+    this.started = false;
+  }
+
+  public async handleHttpRequest(
+    req: plugins.typedserver.Request,
+    res: plugins.typedserver.Response,
+  ) {
+    try {
+      const requestUrl = new URL((req as any).originalUrl || req.url || '/', 'http://localhost');
+
+      if (requestUrl.pathname === '/v2/token') {
+        await this.handleTokenRequest(req, res, requestUrl);
+        return;
+      }
+
+      if (!this.started) {
+        res.status(503);
+        res.end('registry is not ready');
+        return;
+      }
+
+      const rawBody = await this.getRawBody(req);
+      const response = await this.smartRegistry.handleRequest({
+        method: req.method || 'GET',
+        path: requestUrl.pathname,
+        query: Object.fromEntries(requestUrl.searchParams),
+        headers: this.headersToRecord(req.headers),
+        rawBody: rawBody.length > 0 ? rawBody : undefined,
+      });
+
+      await this.sendRegistryResponse(res, response);
+    } catch (error) {
+      logger.log('error', `registry request failed: ${(error as Error).message}`);
+      res.status(500);
+      res.end('registry request failed');
+    }
+  }
+
+  private async handleTokenRequest(
+    req: plugins.typedserver.Request,
+    res: plugins.typedserver.Response,
+    requestUrl: URL,
+  ) {
+    const user = await this.authenticateRequest(req);
+    if (!user) {
+      res.status(401);
+      res.setHeader('WWW-Authenticate', 'Basic realm="Cloudly Registry"');
+      res.end('authentication required');
+      return;
+    }
+
+    const requestedScopes = this.getRequestedOciScopes(requestUrl.searchParams);
+    const requestedWriteAccess = requestedScopes.some((scopeArg) => {
+      const action = scopeArg.split(':').at(-1);
+      return action === 'push' || action === 'delete';
+    });
+    if (requestedWriteAccess && !user.canWrite) {
+      res.status(403);
+      res.end('registry write access denied');
+      return;
+    }
+
+    const token = await this.smartRegistry.getAuthManager().createOciToken(
+      user.userId,
+      requestedScopes,
+      3600,
+    );
+    res.status(200);
+    res.setHeader('Content-Type', 'application/json');
+    res.end(
+      JSON.stringify({
+        token,
+        access_token: token,
+        expires_in: 3600,
+        issued_at: new Date().toISOString(),
+      }),
+    );
+  }
+
+  private async authenticateRequest(
+    req: plugins.typedserver.Request,
+  ): Promise<TAuthenticatedRegistryUser | null> {
+    const credentials = this.getBasicCredentials(req);
+    if (!credentials) {
+      return null;
+    }
+
+    const users = await this.cloudlyRef.authManager.CUser.getInstances({});
+    for (const user of users) {
+      if (user.data?.username !== credentials.username) {
+        continue;
+      }
+
+      const passwordMatches = user.data.password === credentials.password;
+      const matchingToken = user.data.tokens?.find((tokenArg) => {
+        return tokenArg.token === credentials.password && tokenArg.expiresAt > Date.now();
+      });
+      if (!passwordMatches && !matchingToken) {
+        continue;
+      }
+
+      const assignedRoles = matchingToken?.assignedRoles || [];
+      return {
+        userId: user.id,
+        username: user.data.username,
+        canWrite: user.data.role === 'admin' || assignedRoles.includes('admin'),
+      };
+    }
+
+    return null;
+  }
+
+  private getBasicCredentials(req: plugins.typedserver.Request) {
+    const authHeader = req.headers.authorization;
+    if (!authHeader?.startsWith('Basic ')) {
+      return null;
+    }
+
+    const decoded = Buffer.from(authHeader.slice('Basic '.length), 'base64').toString('utf8');
+    const separatorIndex = decoded.indexOf(':');
+    if (separatorIndex <= 0) {
+      return null;
+    }
+
+    return {
+      username: decoded.slice(0, separatorIndex),
+      password: decoded.slice(separatorIndex + 1),
+    };
+  }
+
+  private getRequestedOciScopes(searchParamsArg: URLSearchParams) {
+    const scopes: string[] = [];
+    for (const scope of searchParamsArg.getAll('scope')) {
+      const [scopeType, scopeName, actionsString] = scope.split(':');
+      if (scopeType !== 'repository' || !scopeName || !actionsString) {
+        continue;
+      }
+      for (const action of actionsString.split(',')) {
+        if (action) {
+          scopes.push(`oci:${scopeType}:${scopeName}:${action}`);
+        }
+      }
+    }
+    return scopes;
+  }
+
+  private getPublicRegistryUrl() {
+    if (!this.cloudlyRef.config.data.publicUrl) {
+      throw new Error('Cloudly registry requires publicUrl');
+    }
+    const publicPort = this.cloudlyRef.config.data.publicPort;
+    const includePort =
+      this.cloudlyRef.config.data.sslMode === 'none' && publicPort && !['80', '443'].includes(publicPort);
+    return `${this.cloudlyRef.config.data.sslMode === 'none' ? 'http' : 'https'}://${this.cloudlyRef.config.data.publicUrl}${includePort ? `:${publicPort}` : ''}`;
+  }
+
+  private headersToRecord(headersArg: plugins.typedserver.Request['headers']) {
+    return Object.fromEntries(
+      Object.entries(headersArg).map(([key, value]) => [
+        key.toLowerCase(),
+        Array.isArray(value) ? value.join(', ') : value || '',
+      ]),
+    );
+  }
+
+  private async getRawBody(req: plugins.typedserver.Request) {
+    const chunks: Buffer[] = [];
+    for await (const chunk of req as any) {
+      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    }
+    return Buffer.concat(chunks);
+  }
+
+  private async sendRegistryResponse(
+    res: plugins.typedserver.Response,
+    responseArg: plugins.smartregistry.IResponse,
+  ) {
+    res.status(responseArg.status);
+    for (const [key, value] of Object.entries(responseArg.headers)) {
+      res.setHeader(key, value);
+    }
+
+    if (!responseArg.body) {
+      res.end();
+      return;
+    }
+
+    if (responseArg.body instanceof ReadableStream) {
+      plugins.stream.Readable.fromWeb(responseArg.body as any).pipe(res);
+      return;
+    }
+
+    if (Buffer.isBuffer(responseArg.body) || typeof responseArg.body === 'string') {
+      res.end(responseArg.body);
+      return;
+    }
+
+    res.setHeader('Content-Type', responseArg.headers['Content-Type'] || 'application/json');
+    res.end(JSON.stringify(responseArg.body));
+  }
+}

ts/manager.registry/index.ts

@@ -0,0 +1 @@
+export * from './classes.registrymanager.js';

ts/plugins.ts

@@ -1,7 +1,8 @@
 // node native
 import * as path from 'path';
+import * as stream from 'stream';
 
-export { path };
+export { path, stream };
 
 // @apiglobal scope
 import * as typedrequest from '@api.global/typedrequest';
@@ -42,6 +43,7 @@ import * as smartlog from '@push.rocks/smartlog';
 import * as smartpath from '@push.rocks/smartpath';
 import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrequest from '@push.rocks/smartrequest';
+import * as smartregistry from '@push.rocks/smartregistry';
 import * as smartssh from '@push.rocks/smartssh';
 import * as smartstream from '@push.rocks/smartstream';
 import * as smartstring from '@push.rocks/smartstring';
@@ -68,6 +70,7 @@ export {
   smartlog,
   smartpath,
   smartpromise,
+  smartregistry,
   smartrequest,
   smartssh,
   smartstream,