prepare service management

package.json

@@ -26,15 +26,14 @@
     "@git.zone/tstest": "^1.0.90",
     "@git.zone/tswatch": "^2.0.23",
     "@push.rocks/tapbundle": "^5.0.23",
-    "@types/node": "^20.13.0"
+    "@types/node": "^20.14.2"
   },
   "dependencies": {
     "@api.global/typedrequest": "3.0.30",
     "@api.global/typedserver": "^3.0.50",
     "@api.global/typedsocket": "^3.0.1",
     "@apiclient.xyz/cloudflare": "^6.0.1",
-    "@apiclient.xyz/digitalocean": "^1.0.5",
+    "@apiclient.xyz/docker": "^1.2.2",
-    "@apiclient.xyz/docker": "^1.0.112",
     "@apiclient.xyz/hetznercloud": "^1.0.18",
     "@apiclient.xyz/slack": "^3.0.9",
     "@design.estate/dees-catalog": "^1.0.289",
@@ -42,33 +41,34 @@
     "@design.estate/dees-element": "^2.0.34",
     "@git.zone/tsrun": "^1.2.37",
     "@push.rocks/early": "^4.0.3",
-    "@push.rocks/npmextra": "^5.0.13",
+    "@push.rocks/npmextra": "^5.0.17",
     "@push.rocks/projectinfo": "^5.0.1",
     "@push.rocks/qenv": "^6.0.5",
     "@push.rocks/smartacme": "^4.0.8",
-    "@push.rocks/smartbucket": "^3.0.9",
+    "@push.rocks/smartbucket": "^3.0.15",
     "@push.rocks/smartcli": "^4.0.11",
     "@push.rocks/smartdata": "^5.2.4",
     "@push.rocks/smartdelay": "^3.0.5",
     "@push.rocks/smartexit": "^1.0.23",
-    "@push.rocks/smartfile": "^11.0.16",
+    "@push.rocks/smartfile": "^11.0.20",
     "@push.rocks/smartguard": "^3.0.2",
     "@push.rocks/smartjson": "^5.0.19",
     "@push.rocks/smartjwt": "^2.0.4",
-    "@push.rocks/smartlog": "^3.0.6",
+    "@push.rocks/smartlog": "^3.0.7",
     "@push.rocks/smartlog-destination-clickhouse": "^1.0.11",
     "@push.rocks/smartpath": "^5.0.18",
     "@push.rocks/smartpromise": "^4.0.3",
     "@push.rocks/smartrequest": "^2.0.22",
     "@push.rocks/smartrx": "^3.0.7",
     "@push.rocks/smartssh": "^2.0.1",
-    "@push.rocks/smartstream": "^3.0.39",
+    "@push.rocks/smartstate": "^2.0.17",
+    "@push.rocks/smartstream": "^3.0.44",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/smartunique": "^3.0.9",
     "@push.rocks/taskbuffer": "^3.0.2",
     "@push.rocks/webjwt": "^1.0.9",
-    "@serve.zone/interfaces": "^1.0.62",
+    "@serve.zone/interfaces": "^1.0.70",
-    "@tsclass/tsclass": "^4.0.54"
+    "@tsclass/tsclass": "^4.0.55"
   },
   "files": [
     "ts/**/*",

ts/00demo/demo.data.secrets.ts

@@ -63,6 +63,8 @@ for (let i = 0; i < demoSecretGroups.length; i++) {
     id: `configBundleId${i + 1}`,
     data: {
       name: `Demo Config Bundle ${i + 1}`,
+      includedImages: [],
+      type: 'external',
       description: 'Demo Purpose',
       includedSecretGroupIds: [secretGroup.id],
       includedTags: secretGroup.data.tags,

ts/00demo/demo.data.users.ts

@@ -0,0 +1,19 @@
+import * as plugins from '../plugins.js';
+import * as paths from '../paths.js';
+import type { Cloudly } from '../classes.cloudly.js';
+
+export const getUsers = async (cloudlyRef: Cloudly) => {
+  const users: plugins.servezoneInterfaces.data.IUser[] = [];
+  const envAdminUser = await cloudlyRef.config.appData.waitForAndGetKey('servezoneAdminaccount');
+  if (envAdminUser) {
+    users.push({
+      id: 'envadmin',
+      data: {
+        username: envAdminUser.split(':')[0],
+        password: envAdminUser.split(':')[1],
+        role: 'admin',
+      },
+    });
+  }
+  return users;
+};

ts/00demo/index.ts

@@ -43,7 +43,7 @@ export const installDemoData = async (cloudlyRef: Cloudly) => {
   }
 
   const demoDataUsers = await import('./demo.data.users.js');
-  for (const user of demoDataUsers.users) {
+  for (const user of await demoDataUsers.getUsers(cloudlyRef)) {
     const userInstance = new cloudlyRef.authManager.CUser();
     Object.assign(userInstance, user);
     await userInstance.save();

ts/classes.cloudly.ts

@@ -15,10 +15,10 @@ import { MongodbConnector } from './connector.mongodb/connector.js';
 
 // processes
 import { CloudlyCoreflowManager } from './manager.coreflow/coreflowmanager.js';
-import { ClusterManager } from './manager.cluster/clustermanager.js';
+import { ClusterManager } from './manager.cluster/classes.clustermanager.js';
 import { CloudlyTaskmanager } from './manager.task/taskmanager.js';
 import { CloudlySecretManager } from './manager.secret/classes.secretmanager.js'
-import { CloudlyServerManager } from './manager.server/servermanager.js';
+import { CloudlyServerManager } from './manager.server/classes.servermanager.js';
 import { ExternalApiManager } from './manager.status/statusmanager.js';
 import { ImageManager } from './manager.image/classes.imagemanager.js';
 import { logger } from './logger.js';

ts/classes.config.ts

@@ -40,6 +40,7 @@ export class CloudlyConfig {
           useSsl: true,
         },
         sslMode: 'SERVEZONE_SSLMODE' as plugins.servezoneInterfaces.data.ICloudlyConfig['sslMode'],
+        servezoneAdminaccount: 'SERVEZONE_ADMINACCOUNT',
       },
       requiredKeys: [
         'cfToken',

ts/classes.server.ts

@@ -10,7 +10,8 @@ export class CloudlyServer {
   /**
    * a reference to the cloudly instance
    */
-  private cloudlyRef: Cloudly;
+  public cloudlyRef: Cloudly;
+  public additionalHandlers: plugins.typedserver.servertools.Handler[] = [];
 
   /**
    * the smartexpress server handling the actual requests
@@ -37,18 +38,24 @@ export class CloudlyServer {
    * init the reception instance
    */
   public async start() {
-    logger.log('info', `cloudly domain is ${this.cloudlyRef.config.data.publicUrl}`)
+    logger.log('info', `cloudly domain is ${this.cloudlyRef.config.data.publicUrl}`);
     let sslCert: plugins.smartacme.Cert;
 
     if (this.cloudlyRef.config.data.sslMode === 'letsencrypt') {
-      logger.log('info', `Using letsencrypt for ssl mode. Trying to obtain a certificate...`)
+      logger.log('info', `Using letsencrypt for ssl mode. Trying to obtain a certificate...`);
-      logger.log('info', `This might take 10 minutes...`)
+      logger.log('info', `This might take 10 minutes...`);
       sslCert = await this.cloudlyRef.letsencryptConnector.getCertificateForDomain(
         this.cloudlyRef.config.data.publicUrl
       );
-      logger.log('success', `Successfully obtained certificate for cloudly domain ${this.cloudlyRef.config.data.publicUrl}`)
+      logger.log(
+        'success',
+        `Successfully obtained certificate for cloudly domain ${this.cloudlyRef.config.data.publicUrl}`
+      );
     } else if (this.cloudlyRef.config.data.sslMode === 'external') {
-      logger.log('info', `Using external certificate for ssl mode, meaning cloudly is not in charge of ssl termination.`)
+      logger.log(
+        'info',
+        `Using external certificate for ssl mode, meaning cloudly is not in charge of ssl termination.`
+      );
     }
 
     interface IRequestGuardData {
@@ -72,11 +79,13 @@ export class CloudlyServer {
     this.typedServer = new plugins.typedserver.TypedServer({
       cors: true,
       forceSsl: false,
-      port: this.cloudlyRef.config.data.publicPort,
+      port: this.cloudlyRef.config.data.publicPort,
-      ...(sslCert ? {
+      ...(sslCert
-        privateKey: sslCert.privateKey,
+        ? {
-        publicKey: sslCert.publicKey,
+            privateKey: sslCert.privateKey,
-      } : {}),
+            publicKey: sslCert.publicKey,
+          }
+        : {}),
       injectReload: true,
       serveDir: paths.distServeDir,
       watch: true,
@@ -84,6 +93,10 @@ export class CloudlyServer {
       preferredCompressionMethod: 'gzip',
     });
     this.typedServer.typedrouter.addTypedRouter(this.typedrouter);
+    this.typedServer.server.addRoute(
+      '/curlfresh/:scriptname',
+      this.cloudlyRef.serverManager.curlfreshInstance.handler
+    );
     await this.typedServer.start();
   }
 

ts/demo/demo.data.users.ts

@@ -1,12 +0,0 @@
-import * as plugins from '../plugins.js';
-
-export const users: plugins.servezoneInterfaces.data.IUser[] = [
-  {
-    id: 'user1',
-    data: {
-      username: 'admin',
-      password: 'password',
-      role: 'admin',
-    }
-  }
-]

ts/index.ts

@@ -21,7 +21,7 @@ const runCli = async () => {
   );
 
   await cloudlyInstance.start();
-  const demoMod = await import('./demo/index.js');
+  const demoMod = await import('./00demo/index.js');
   demoMod.installDemoData(cloudlyInstance);
 };
 

ts/manager.cluster/classes.clustermanager.ts

@@ -3,7 +3,7 @@ import * as paths from '../paths.js';
 import { Cloudly } from '../classes.cloudly.js';
 import { logger } from '../logger.js';
 
-import { Cluster } from './cluster.js';
+import { Cluster } from './classes.cluster.js';
 
 export class ClusterManager {
   public ready = plugins.smartpromise.defer();
@@ -28,7 +28,6 @@ export class ClusterManager {
             name: dataArg.clusterName,
             jumpCode: plugins.smartunique.uniSimple('cluster'),
             jumpCodeUsedAt: null,
-            secretKey: plugins.smartunique.shortId(16),
             acmeInfo: null,
             cloudlyUrl: `https://${this.cloudlyRef.config.data.publicUrl}:${this.cloudlyRef.config.data.publicPort}/`,
             servers: [],
@@ -54,6 +53,17 @@ export class ClusterManager {
         };
       })
     );
+
+    // delete cluster
+    this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.cluster.IRequest_DeleteCluster>(
+      new plugins.typedrequest.TypedHandler('deleteCluster', async (reqDataArg, toolsArg) => {
+        await toolsArg.passGuards([this.cloudlyRef.authManager.adminJwtGuard], reqDataArg);
+        await this.deleteCluster(reqDataArg.clusterId);
+        return {
+          success: true,
+        };
+      })
+    );
   }
 
   public async init() {
@@ -86,9 +96,9 @@ export class ClusterManager {
     await this.ready.promise;
 
     return await Cluster.getInstance({
+      id: clusterIdentifier.clusterId,
       data: {
         name: clusterIdentifier.clusterName,
-        secretKey: clusterIdentifier.secretKey,
       },
     });
   }
@@ -128,4 +138,10 @@ export class ClusterManager {
     await clusterInstance.save();
     return clusterInstance;
   }
+
+  public async deleteCluster(clusterId: string) {
+    await this.ready.promise;
+    const clusterInstance = await Cluster.getInstance({ id: clusterId });
+    await clusterInstance.delete();
+  }
 }

ts/manager.image/classes.imagemanager.ts

@@ -8,6 +8,7 @@ export class ImageManager {
   public typedrouter = new plugins.typedrequest.TypedRouter();
   public smartbucketInstance: plugins.smartbucket.SmartBucket;
   public imageDir: plugins.smartbucket.Directory;
+  public dockerImageStore: plugins.docker.DockerImageStore;
   
   get db() {
     return this.cloudlyRef.mongodbConnector.smartdataDb;
@@ -72,9 +73,18 @@ export class ImageManager {
     this.typedrouter.addTypedHandler(
       new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.image.IRequest_PushImageVersion>(
         'pushImageVersion',
-        async (reqArg) => {
+        async (reqArg, toolsArg) => {
-          const pushStream = reqArg.imageStream;
+          const image = await this.CImage.getInstance({
-          return {};
+            id: reqArg.imageId,
+          });
+          if (!image) {
+            throw new plugins.typedrequest.TypedResponseError('Image not found');
+          }
+          const imageVersion = reqArg.versionString;
+          const imagePushStream = reqArg.imageStream;
+          return {
+            allowed: true,
+          };
         }
       )
     );
@@ -87,7 +97,9 @@ export class ImageManager {
               id: reqArg.imageId,
           });
           const imageVersion = image.data.versions.find((version) => version.versionString === reqArg.versionString);
-          const readable = this.imageDir.fastGetStream(await image.getStoragePath(reqArg.versionString));
+          const readable = this.imageDir.fastGetStream({
+            path: await image.getStoragePath(reqArg.versionString),
+          }, 'webstream');
           const imageVirtualStream = new plugins.typedrequest.VirtualStream();
           return {
             imageStream: imageVirtualStream,
@@ -111,10 +123,4 @@ export class ImageManager {
       path: 'images',
     });
   }
-
-  public async createImage(nameArg: string) {
-    const newImage = await this.CImage.create({
-      name: nameArg,
-    });
-  }
 }

ts/manager.server/classes.curlfresh.ts

@@ -0,0 +1,84 @@
+import { logger } from '../logger.js';
+import * as plugins from '../plugins.js';
+import type { CloudlyServerManager } from './classes.servermanager.js';
+
+export class CurlFresh {
+  public optionsArg = {
+    npmRegistry: 'https://registry.npmjs.org',
+  }
+  public scripts = {
+    'setup.sh': `#!/bin/bash
+
+# lets update the system and install curl
+# might be installed already, but entrypoint could have been wget
+apt-get update
+apt-get install -y --force-yes curl
+
+# Basic updating of the software lists
+echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+apt-get update
+apt-get upgrade -y --force-yes
+apt-get install -y --force-yes fail2ban curl git
+curl -sL https://deb.nodesource.com/setup_18.x | bash
+
+# Install docker
+curl -sSL https://get.docker.com/ | sh
+
+# Install default nodejs to run nodejs tools
+apt-get install -y nodejs zsh
+zsh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended
+npm config set unsafe-perm true
+
+# lets make sure we use the correct npm registry
+bash -c "npm config set registry ${this.optionsArg.npmRegistry}"
+
+# lets install spark
+bash -c "npm install -g @serve.zone/spark"
+
+# lets install the spark daemon
+bash -c "spark installdaemon"
+
+# TODO: start spark with jump code
+`,
+  };
+
+  public serverManagerRef: CloudlyServerManager;
+  public curlFreshRoute: plugins.typedserver.servertools.Route;
+  public handler = new plugins.typedserver.servertools.Handler('ALL', async (req, res) => {
+    logger.log('info', 'curlfresh handler called. a server might be coming online soon :)');
+    const scriptname = req.params.scriptname;
+    switch(scriptname) {
+      case 'setup.sh':
+        logger.log('info', 'sending setup.sh');
+        res.type('application/x-sh');
+        res.send(this.scripts['setup.sh']);
+        break;
+      default:
+        res.send('no script found');
+        break;
+    }
+  });
+
+  constructor(serverManagerRefArg: CloudlyServerManager) {
+    this.serverManagerRef = serverManagerRefArg;
+  }
+  public async getServerUserData(): Promise<string> {
+    const sslMode = await this.serverManagerRef.cloudlyRef.config.appData.waitForAndGetKey('sslMode');
+    let protocol: 'http' | 'https';
+    if (sslMode === 'none') {
+      protocol = 'http';
+    } else {
+      protocol = 'https';
+    }
+  
+    const domain = await this.serverManagerRef.cloudlyRef.config.appData.waitForAndGetKey('publicUrl');
+    const port = await this.serverManagerRef.cloudlyRef.config.appData.waitForAndGetKey('publicPort');
+  
+    const serverUserData = `#cloud-config
+  runcmd:
+    - curl -o- ${protocol}://${domain}:${port}/curlfresh/setup.sh | sh
+  `
+    console.log(serverUserData);
+    return serverUserData;
+  };
+}

ts/manager.server/classes.servermanager.ts

@@ -1,11 +1,13 @@
 import * as plugins from '../plugins.js';
 import { Cloudly } from '../classes.cloudly.js';
-import { Cluster } from '../manager.cluster/cluster.js';
+import { Cluster } from '../manager.cluster/classes.cluster.js';
-import { Server } from './server.js';
+import { Server } from './classes.server.js';
+import { CurlFresh } from './classes.curlfresh.js';
 
 export class CloudlyServerManager {
   public cloudlyRef: Cloudly;
   public typedRouter = new plugins.typedrequest.TypedRouter();
+  public curlfreshInstance = new CurlFresh(this);
 
   public hetznerAccount: plugins.hetznercloud.HetznerAccount;
 
@@ -63,7 +65,8 @@ export class CloudlyServerManager {
           labels: {
             clusterId: cluster.id,
             priority: '1',
-          }
+          },
+          userData: await this.curlfreshInstance.getServerUserData()
         });
         const newServer = await Server.createFromHetznerServer(server);
         console.log(`cluster created new server for cluster ${cluster.id}`);

ts/manager.service/classes.service.ts

@@ -0,0 +1,12 @@
+import * as plugins from '../plugins.js';
+
+export class Service extends plugins.smartdata.SmartDataDbDoc<Service, plugins.servezoneInterfaces.data.IService, ServiceManager> {
+  
+  @plugins.smartdata.svDb()
+  public id: string;
+
+  @plugins.smartdata.svDb()
+  public data: plugins.servezoneInterfaces.data.IService['data'];
+
+
+}

ts/manager.service/classes.servicemanager.ts

@@ -0,0 +1,13 @@
+import type { Cloudly } from '../classes.cloudly.js';
+import * as plugins from '../plugins.js';
+
+export class ServiceManager {
+  public typedrouter = new plugins.typedrequest.TypedRouter();
+  public cloudlyRef: Cloudly;
+  
+  get db() {
+    return this.cloudlyRef.mongodbConnector.smartdataDb;
+  }
+
+  public CService = plugins.smartdata.setDefaultManagerForDoc(this, Service);
+}

ts/plugins.ts

@@ -9,13 +9,13 @@ import * as typedsocket from '@api.global/typedsocket';
 
 export { typedrequest, typedsocket };
 
-// @mojoio scope
+// @apiclient.xyz scope
 import * as cloudflare from '@apiclient.xyz/cloudflare';
-import * as digitalocean from '@apiclient.xyz/digitalocean';
+import * as docker from '@apiclient.xyz/docker';
 import * as hetznercloud from '@apiclient.xyz/hetznercloud';
 import * as slack from '@apiclient.xyz/slack';
 
-export { cloudflare, digitalocean, hetznercloud, slack };
+export { cloudflare, docker, hetznercloud, slack };
 
 // @tsclass scope
 import * as tsclass from '@tsclass/tsclass';

ts_web/appstate.ts

@@ -5,7 +5,7 @@ const appstate = new plugins.deesDomtools.plugins.smartstate.Smartstate();
 export interface ILoginState {
   jwt: string;
 }
-export const loginStatePart = await appstate.getStatePart<ILoginState>(
+export const loginStatePart: plugins.smartstate.StatePart<unknown, ILoginState> = await appstate.getStatePart<ILoginState>(
   'login',
   { jwt: null },
   'persistent'

ts_web/plugins.ts

@@ -13,7 +13,9 @@ export { deesDomtools, deesElement, deesCatalog };
 
 // @push.rocks scope
 import * as webjwt from '@push.rocks/webjwt';
+import * as smartstate from '@push.rocks/smartstate';
 
 export {
   webjwt,
+  smartstate,
 }