Compare commits

...

68 Commits

Author SHA1 Message Date
jkunz d5445609a0 v5.9.0
Docker (tags) / release (push) Failing after 1s
2026-05-24 13:15:16 +00:00
jkunz fd7c7b4313 fix(cloudly): invalidate expired dashboard sessions 2026-05-24 13:14:49 +00:00
jkunz 057af996aa chore(cloudly): consume released Spark interfaces 2026-05-24 12:54:09 +00:00
jkunz 6565c44c29 feat(cloudly): accept Spark node heartbeats 2026-05-24 12:47:15 +00:00
jkunz ebb4f36c67 v5.8.2
Docker (tags) / release (push) Failing after 13m53s
2026-05-24 01:30:46 +00:00
jkunz e7d3140f7a chore(cloudly): update api dependency and stabilize tests 2026-05-24 01:30:32 +00:00
jkunz 304767a75c v5.8.1
Docker (tags) / release (push) Failing after 0s
2026-05-23 10:59:24 +00:00
jkunz 0fa95d6c99 fix(cloudly): allow Docker install of fresh interfaces 2026-05-23 10:59:06 +00:00
jkunz 5d6d43b564 v5.8.0
Docker (tags) / release (push) Failing after 0s
2026-05-23 10:47:12 +00:00
jkunz bef236cd86 feat(cloudly): add service runtime and onboarding 2026-05-23 10:46:52 +00:00
jkunz 59043b7281 v5.7.1
Docker (tags) / release (push) Failing after 0s
2026-05-21 22:35:19 +00:00
jkunz befd0efdc0 fix(web): clean dashboard console errors 2026-05-21 22:30:38 +00:00
jkunz b1a0ce684a v5.7.0
Docker (tags) / release (push) Failing after 0s
2026-05-21 16:17:21 +00:00
jkunz d0b15ab51b feat(web): add dashboard SPA routing 2026-05-21 16:16:00 +00:00
jkunz 50bcbe0f45 v5.6.0
Docker (tags) / release (push) Failing after 0s
2026-05-21 11:35:49 +00:00
jkunz bb86f8882c docs(changelog): add dashboard navigation entry 2026-05-21 11:34:58 +00:00
jkunz 1874d791b2 feat(web): group dashboard navigation 2026-05-21 11:18:06 +00:00
jkunz f40ef6b7c0 chore: update cloudly dependency stack
Align Cloudly with the current typedserver, smartconfig, smartstate, and Docker tooling releases so builds and Docker output stay compatible with the upgraded stack.
2026-05-08 13:56:20 +00:00
jkunz 80226c8a1c fix: correct cloudly package exports 2026-05-07 20:49:33 +00:00
jkunz 5ba2bb2168 chore: use baseos interfaces release 2026-05-07 20:47:31 +00:00
jkunz 60c51fbf5d feat: add baseos source presets 2026-05-07 20:33:14 +00:00
jkunz 0fcf35c019 docs: refresh readme and legal info 2026-05-07 20:22:12 +00:00
jkunz d9dcc5b048 feat: add corebuild worker selection 2026-05-07 19:49:56 +00:00
jkunz c55eb5b832 feat: harden baseos image builds 2026-05-07 19:04:12 +00:00
jkunz 1792ea89e1 feat: add backup replication targets 2026-05-07 17:44:31 +00:00
jkunz b0f0963143 feat: add baseos image builds 2026-05-07 17:44:31 +00:00
jkunz be7735a9c3 feat: add baseos node enrollment 2026-05-07 15:53:16 +00:00
jkunz 3624c78f9d feat: orchestrate service backups 2026-05-02 21:59:42 +00:00
jkunz d1ce149487 feat: expose external gateway settings 2026-04-29 15:57:02 +00:00
jkunz 4eba247472 feat: discover external gateway domains 2026-04-29 15:34:48 +00:00
jkunz 452aaa3862 feat: pass external gateway config to coreflow 2026-04-29 15:29:17 +00:00
jkunz 7d380d04fc fix: use relative runtime imports 2026-04-29 08:25:58 +00:00
jkunz aab6e9044d fix: modernize docker publishing 2026-04-29 08:13:29 +00:00
jkunz 195236b693 fix: trim registry repository slugs 2026-04-29 01:39:40 +00:00
jkunz 1925f66efc fix: generate docker-compatible registry repositories 2026-04-28 19:46:44 +00:00
jkunz 865c8f2546 fix: allow coreflow deployment input reads 2026-04-28 16:57:54 +00:00
jkunz 1bed907f53 feat: push config updates to coreflow 2026-04-28 16:02:05 +00:00
jkunz ee6d4c3d04 feat: wire service registry targets 2026-04-28 15:50:59 +00:00
jkunz 94f1199858 feat: add built-in OCI registry 2026-04-28 15:23:51 +00:00
jkunz 333cbeb221 fix: make startup bootstrap production-safe 2026-04-28 15:07:08 +00:00
jkunz 13a7d5969d fix: send registry update payload 2026-04-28 14:35:19 +00:00
jkunz 675a95f857 chore: clean local tooling metadata 2026-04-28 14:35:19 +00:00
jkunz 3080075811 feat: add platform desired state manager 2026-04-28 12:18:12 +00:00
jkunz 84d3e8f52f fix(test): use dynamic Cloudly test port 2026-04-28 08:54:29 +00:00
jkunz 500cec008a refactor(cloudly): consume external api package 2026-04-25 15:03:12 +00:00
jkunz 37512cfaa6 refactor(cloudly): consume external interfaces package 2026-04-25 13:57:59 +00:00
jkunz 94e0c38191 feat(domain): improve domain update logic and ensure default activation state 2025-09-14 17:44:59 +00:00
jkunz 6cc3700d29 feat(domains): enhance domain management with activation states and sync options 2025-09-14 17:38:16 +00:00
jkunz bb313fd9dc feat: Add settings view for cloud provider configurations
- Implemented CloudlyViewSettings component for managing cloud provider settings including Hetzner, Cloudflare, AWS, DigitalOcean, Azure, and Google Cloud.
- Added functionality to load, save, and test connections for each provider.
- Enhanced UI with loading states and success/error notifications.

feat: Create tasks view with execution history

- Developed CloudlyViewTasks component to display and manage tasks and their executions.
- Integrated auto-refresh functionality for task executions.
- Added filtering and searching capabilities for tasks.

feat: Implement execution details and task panel components

- Created CloudlyExecutionDetails component to show detailed information about task executions including logs and metrics.
- Developed CloudlyTaskPanel component to display individual tasks with execution status and actions to run or cancel tasks.

feat: Utility functions for formatting and categorization

- Added utility functions for formatting dates, durations, and cron expressions.
- Implemented functions to retrieve category icons and hues for task categorization.
2025-09-14 17:28:21 +00:00
jkunz 5ef8621db7 feat(task-execution): implement task cancellation handling and improve UI feedback for canceling tasks 2025-09-12 23:53:10 +00:00
jkunz 6cd348ca28 feat(cloudly-view-tasks): add search and category filters, implement auto-refresh for task executions 2025-09-12 23:38:18 +00:00
jkunz 3183f9e909 chore(deps): update dependencies for @push.rocks/smartjson, @push.rocks/smartstate, @push.rocks/smartstring, and @git.zone/tstest 2025-09-12 22:20:30 +00:00
jkunz ff7004412b feat(appstate): Remove helper function for stripping class instances from data fetching 2025-09-12 10:58:16 +00:00
jkunz f07bcc4660 feat(appstate): Refactor data fetching to use helper for stripping class instances 2025-09-12 07:56:06 +00:00
jkunz d773e13aab feat(api-client): Add update and delete methods for external registries and secret bundles/groups 2025-09-10 20:33:32 +00:00
jkunz dc0d128892 feat(api-client): Add advanced cluster creation method and refactor login actions to use API client 2025-09-10 20:23:12 +00:00
jkunz 124c4ca46f feat: Enhance API client integration across web and CLI
- Added typedRequestInterfaces import to plugins.ts for better type handling.
- Updated CLI client to utilize environment variables for Cloudly API credentials and improved authentication flow.
- Refactored appstate.ts to use a shared API client instance, reducing redundancy in API calls for various actions.
- Simplified external registry actions in appstate.ts by leveraging the shared API client.
- Updated CloudlyDashboard and CloudlyViewSettings components to utilize the shared API client for fetching settings and managing connections.
- Removed redundant TypedRequest instances in favor of direct API client calls for improved performance and maintainability.
- Exposed the API client in plugins.ts for easier access in UI components.
2025-09-10 19:06:16 +00:00
jkunz 5d281d9b6c feat(tasks): Enhance task management with identity handling and initial data loading 2025-09-10 17:04:18 +00:00
jkunz 5b37bb5b11 feat: Implement Cloudly Task Manager with predefined tasks and execution tracking
- Added CloudlyTaskManager class for managing tasks, including registration, execution, scheduling, and cancellation.
- Created predefined tasks: DNS Sync, Certificate Renewal, Cleanup, Health Check, Resource Report, Database Maintenance, Security Scan, and Docker Cleanup.
- Introduced ITaskExecution interface for tracking task execution details and outcomes.
- Developed API request interfaces for task management operations (getTasks, getTaskExecutions, triggerTask, cancelTask).
- Implemented CloudlyViewTasks web component for displaying tasks and their execution history, including filtering and detailed views.
2025-09-10 16:37:03 +00:00
jkunz fd1da01a3f feat(dns): Enhance DNS management with auto-generated entries and service activation 2025-09-10 15:38:42 +00:00
jkunz 6a447369f8 feat(external-registry): Enhance authentication handling and update UI for external registries 2025-09-10 08:50:32 +00:00
jkunz 01d877f7ed feat(external-registry): Implement CRUD operations and connection verification for external registries 2025-09-10 08:24:55 +00:00
jkunz 73505d1ed8 feat(dns): Add domain validation and dropdown for DNS entry creation and updates 2025-09-09 15:13:44 +00:00
jkunz 766191899c feat(dns): Implement DNS management functionality
- Added DnsManager and DnsEntry classes to handle DNS entries.
- Introduced new interfaces for DNS entry requests and data structures.
- Updated Cloudly class to include DnsManager instance.
- Enhanced app state to manage DNS entries and actions for creating, updating, and deleting DNS records.
- Created UI components for DNS management, including forms for adding and editing DNS entries.
- Updated overview and services views to reflect DNS entries.
- Added validation and formatting methods for DNS entries.
2025-09-09 15:08:28 +00:00
jkunz 38e8b4086d feat(requests): Add deploymentRequests to index for improved request handling 2025-09-08 13:03:33 +00:00
jkunz ce047d1bb0 feat(deployment): Implement Deployment and DeploymentManager classes with CRUD operations and service integration 2025-09-08 12:46:23 +00:00
jkunz 4e38d2ff43 5.3.0
Docker (tags) / security (push) Successful in 52s
Docker (tags) / test (push) Failing after 2m4s
Docker (tags) / release (push) Has been skipped
Docker (tags) / metadata (push) Has been skipped
2025-09-08 06:46:14 +00:00
jkunz e19639c9be feat(web): Add deployments API typings and web UI improvements: services & deployments management with CRUD and actions 2025-09-08 06:46:14 +00:00
192 changed files with 17383 additions and 14755 deletions
+4
View File
@@ -1 +1,5 @@
node_modules/
.nogit/
.git/
.pnpm-store/
.vagrant/
+12 -48
View File
@@ -1,4 +1,4 @@
name: Docker (tags)
name: Docker (non-tag pushes)
on:
push:
@@ -6,44 +6,12 @@ on:
- '**'
env:
IMAGE: code.foss.global/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_LOGIN_DOCKER_GITEA: ${{ github.server_url }}|${{ gitea.repository_owner }}|${{ secrets.GITEA_TOKEN }}
IMAGE: code.foss.global/host.today/ht-docker-node:szci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_LOGIN_DOCKER_DOCKERREGISTRY: ${{ secrets.NPMCI_LOGIN_DOCKER_DOCKERREGISTRY }}
jobs:
security:
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Install pnpm and npmci
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
@@ -54,18 +22,14 @@ jobs:
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
npmci npm prepare
pnpm install -g @git.zone/tsdocker@latest
pnpm install
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test
run: pnpm test
- name: Test build
run: |
npmci npm prepare
npmci node install stable
npmci npm install
npmci command npm run build
- name: Build image
run: tsdocker build
- name: Test image
run: tsdocker test
+16 -81
View File
@@ -6,76 +6,15 @@ on:
- '*'
env:
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
# NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
# NPMCI_LOGIN_DOCKER_GITEA: ${{ github.server_url }}|${{ gitea.repository_owner }}|${{ secrets.GITEA_TOKEN }}
IMAGE: code.foss.global/host.today/ht-docker-node:szci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_LOGIN_DOCKER_DOCKERREGISTRY: ${{ secrets.NPMCI_LOGIN_DOCKER_DOCKERREGISTRY }}
NPMCI_SECRET01: ${{ secrets.NPMCI_SECRET01 }}
jobs:
security:
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci command npm run build
release:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: code.foss.global/host.today/ht-docker-dbase:npmci
image: code.foss.global/host.today/ht-docker-dbase:szci
steps:
- uses: actions/checkout@v3
@@ -83,24 +22,20 @@ jobs:
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
pnpm install -g @git.zone/tsdocker@latest
pnpm install
- name: Release
run: |
npmci docker login
npmci docker build
npmci docker test
npmci docker push code.foss.global
- name: Login to registries
run: tsdocker login
metadata:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
- name: List images
run: tsdocker list
steps:
- uses: actions/checkout@v3
- name: Build images
run: tsdocker build
- name: Trigger
run: npmci trigger
- name: Test images
run: tsdocker test
- name: Push to code.foss.global
run: tsdocker push code.foss.global
-1
View File
@@ -1 +0,0 @@
/cache
-68
View File
@@ -1,68 +0,0 @@
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
# * For C, use cpp
# * For JavaScript, use typescript
# Special requirements:
# * csharp: Requires the presence of a .sln file in the project folder.
language: typescript
# whether to use the project's gitignore file to ignore files
# Added on 2025-04-07
ignore_all_files_in_gitignore: true
# list of additional paths to ignore
# same syntax as gitignore, so you can use * and **
# Was previously called `ignored_dirs`, please update your config if you are using that.
# Added (renamed) on 2025-04-07
ignored_paths: []
# whether the project is in read-only mode
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
# Added on 2025-04-18
read_only: false
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
# Below is the complete list of tools for convenience.
# To make sure you have the latest list of tools, and to view their descriptions,
# execute `uv run scripts/print_tool_overview.py`.
#
# * `activate_project`: Activates a project by name.
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
# * `create_text_file`: Creates/overwrites a file in the project directory.
# * `delete_lines`: Deletes a range of lines within a file.
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
# * `execute_shell_command`: Executes a shell command.
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
# * `initial_instructions`: Gets the initial instructions for the current project.
# Should only be used in settings where the system prompt cannot be set,
# e.g. in clients you have no control over, like Claude Desktop.
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
# * `insert_at_line`: Inserts content at a given line in a file.
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
# * `list_memories`: Lists memories in Serena's project-specific memory store.
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
# * `read_file`: Reads a file within the project directory.
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
# * `remove_project`: Removes a project from the Serena configuration.
# * `replace_lines`: Replaces a range of lines within a file with new content.
# * `replace_symbol_body`: Replaces the full definition of a symbol.
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
# * `search_for_pattern`: Performs a search for a pattern in the project.
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
# * `switch_modes`: Activates modes by providing a list of their names
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
excluded_tools: []
# initial prompt for the project. It will always be given to the LLM upon activating the project
# (contrary to the memories, which are loaded on demand).
initial_prompt: ""
project_name: "cloudly"
+83 -4
View File
@@ -1,5 +1,70 @@
{
"npmci": {
"@git.zone/tsdocker": {
"registries": [
"code.foss.global"
],
"registryRepoMap": {
"code.foss.global": "serve.zone/cloudly"
},
"platforms": [
"linux/amd64",
"linux/arm64"
]
},
"@git.zone/tsbundle": {
"bundles": [
{
"from": "./ts_web/index.ts",
"to": "./dist_serve/bundle.js",
"outputMode": "bundle",
"bundler": "esbuild",
"production": true,
"includeFiles": [
"./html/**/*"
]
}
]
},
"@git.zone/tswatch": {
"preset": "website",
"server": {
"enabled": true,
"port": 3000,
"serveDir": "./dist_serve/",
"liveReload": true
},
"watchers": [
{
"name": "backend",
"watch": [
"./ts/**/*",
"./ts_cliclient/**/*"
],
"command": "pnpm run startTs",
"restart": true,
"debounce": 300,
"runOnStart": true
}
],
"bundles": [
{
"name": "website",
"from": "./ts_web/index.ts",
"to": "./dist_serve/bundle.js",
"watchPatterns": [
"./ts_web/**/*",
"./html/**/*"
],
"triggerReload": true,
"bundler": "esbuild",
"production": false,
"includeFiles": [
"./html/**/*"
]
}
]
},
"@ship.zone/szci": {
"npmGlobalTools": [],
"npmAccessLevel": "public",
"npmRegistryUrl": "verdaccio.lossless.digital",
@@ -8,7 +73,8 @@
},
"dockerBuildargEnvMap": {}
},
"gitzone": {
"@git.zone/cli": {
"schemaVersion": 2,
"projectType": "service",
"module": {
"githost": "code.foss.global",
@@ -45,9 +111,22 @@
"backend",
"security"
]
},
"release": {
"targets": {
"git": {
"enabled": true,
"remote": "origin"
},
"docker": {
"enabled": true,
"engine": "tsdocker",
"patterns": []
}
}
}
},
"tsdoc": {
"@git.zone/tsdoc": {
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
}
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
{
"json.schemas": [
{
"fileMatch": ["/npmextra.json"],
"fileMatch": ["/.smartconfig.json"],
"schema": {
"type": "object",
"properties": {
+23 -34
View File
@@ -1,46 +1,35 @@
# gitzone dockerfile_service
## STAGE 1 // BUILD
FROM code.foss.global/host.today/ht-docker-node:npmci as node1
COPY ./ /app
FROM code.foss.global/host.today/ht-docker-node:lts AS build
WORKDIR /app
ARG NPMCI_TOKEN_NPM2
ENV NPMCI_TOKEN_NPM2 $NPMCI_TOKEN_NPM2
RUN npmci npm prepare
RUN pnpm config set store-dir .pnpm-store
RUN rm -rf node_modules && pnpm install
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
RUN pnpm install --frozen-lockfile
COPY . ./
RUN pnpm run build
RUN pnpm prune --prod
## STAGE 2 // PRODUCTION
FROM code.foss.global/host.today/ht-docker-node:lts AS production
# gitzone dockerfile_service
## STAGE 2 // install production
FROM code.foss.global/host.today/ht-docker-node:npmci as node2
WORKDIR /app
COPY --from=node1 /app /app
RUN rm -rf .pnpm-store
ARG NPMCI_TOKEN_NPM2
ENV NPMCI_TOKEN_NPM2 $NPMCI_TOKEN_NPM2
RUN npmci npm prepare
RUN pnpm config set store-dir .pnpm-store
RUN rm -rf node_modules/ && pnpm install --prod
ENV NODE_ENV=production
ENV SERVEZONE_PORT=80
## STAGE 3 // rebuild dependencies for alpine
FROM code.foss.global/host.today/ht-docker-node:alpine_npmci as node3
WORKDIR /app
COPY --from=node2 /app /app
ARG NPMCI_TOKEN_NPM2
ENV NPMCI_TOKEN_NPM2 $NPMCI_TOKEN_NPM2
RUN npmci npm prepare
RUN pnpm config set store-dir .pnpm-store
RUN pnpm rebuild -r
COPY --from=build /app/package.json ./package.json
COPY --from=build /app/node_modules ./node_modules
COPY --from=build /app/cli.js ./cli.js
COPY --from=build /app/dist_ts ./dist_ts
COPY --from=build /app/dist_serve ./dist_serve
## STAGE 4 // the final production image with all dependencies in place
FROM code.foss.global/host.today/ht-docker-node:alpine as node4
WORKDIR /app
COPY --from=node3 /app /app
LABEL org.opencontainers.image.title="cloudly" \
org.opencontainers.image.description="serve.zone control plane" \
org.opencontainers.image.source="https://code.foss.global/serve.zone/cloudly"
### Healthchecks
RUN pnpm install -g @servezone/healthy
HEALTHCHECK --interval=30s --timeout=30s --start-period=30s --retries=3 CMD [ "healthy" ]
HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 CMD node -e "const http=require('node:http');const port=process.env.SERVEZONE_PORT||80;const req=http.get({host:'127.0.0.1',port,path:'/'},res=>process.exit(res.statusCode<500?0:1));req.on('error',()=>process.exit(1));req.setTimeout(5000,()=>{req.destroy();process.exit(1);});"
EXPOSE 80
CMD ["npm", "start"]
CMD ["node", "cli.js"]
+82
View File
@@ -1,5 +1,87 @@
# Changelog
## Pending
## 2026-05-24 - 5.9.0
### Features
- accept Spark node heartbeats
- Adds a Spark heartbeat HTTP endpoint for cluster nodes
- Stores Spark metrics and runtime info on cluster node records
- Extends jump onboarding with per-node Spark telemetry credentials
### Fixes
- invalidate expired dashboard sessions and return admins to login
### Maintenance
- refresh release tooling dependencies
- update `@serve.zone/interfaces` to the Spark telemetry contract release
## 2026-05-24 - 5.8.2
- update Cloudly to consume the released Jump Code API client
- bumps `@serve.zone/api` to `^5.3.8`
- keeps Docker release installs working with fresh serve.zone package releases
- refreshes mature build, bundle, test, docs, and watch tooling
- stabilizes API client integration test setup and cleanup
## 2026-05-23 - 5.8.1
### Fixes
- allow Docker builds to install freshly released serve.zone interfaces
- Copies pnpm-workspace.yaml into the Docker dependency install layer
- Excludes @serve.zone/interfaces from pnpm 11 minimum release age checks during release builds
## 2026-05-23 - 5.8.0
### Features
- add service detail runtime actions and app catalog onboarding
- Adds service detail pages with live deployments, restart, kill, and deployment IDE access
- Adds app catalog install/update detection contracts and Cloudly handlers
- Adds node jump codes for connecting systems to clusters
- Updates Cloudly to pnpm 11 and @serve.zone/interfaces 5.9.0
## 2026-05-21 - 5.7.1
### Fixes
- clean up Cloudly dashboard console and asset errors
- replace invalid Lucide icon references in table actions and context menu items
- add PWA manifest and local SVG favicon assets to avoid 404s
- align local Cloudly custom element tags with canonical kebab-case names
- remove noisy frontend debug logging from login and revenue checks
## 2026-05-21 - 5.7.0
### Features
- add SPA dashboard path navigation (web)
- support direct links to dashboard views and subviews via URL paths
- sync appdash selection with browser history and enable server SPA fallback
## 2026-05-21 - 5.6.0
### Features
- group dashboard navigation (web)
- reorganize Cloudly sidebar into Platform, Runtime, Registry & Build, Secrets, Domains & Messaging, Storage, and Logs sections
- keep Images next to Services in the Runtime group because services reference imageId and imageVersion
## 2025-09-08 - 5.3.0 - feat(web)
Add deployments API typings and web UI improvements: services & deployments management with CRUD and actions
- Add deployment request interfaces (ts_interfaces/requests/deployment.ts) to define typed API for create/read/update/delete/scale/restart operations.
- Extend web app state (ts_web/appstate.ts) to include typed services and deployments, and add actions for create/update/delete of services and deployments.
- Enhance web views (ts_web/elements/*): CloudlyViewServices and CloudlyViewDeployments now include richer display, styling, and UI actions (create, edit, deploy, restart, stop, delete).
- Fix subscription variable naming in several web components (subecription -> subscription) and improve table display functions to handle missing data safely.
- Add .claude/settings.local.json (tooling/permissions) used for local development/test tooling.
## 2025-09-07 - 5.2.0 - feat(settings)
Add runtime settings management, node & baremetal managers, and settings UI
+5
View File
@@ -0,0 +1,5 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" role="img" aria-label="Cloudly">
<rect width="64" height="64" rx="14" fill="#050505"/>
<path d="M19 40h27a9 9 0 0 0 1.5-17.9A14 14 0 0 0 20.2 17 11.5 11.5 0 0 0 19 40Z" fill="none" stroke="#ffffff" stroke-width="4" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M23 32h18" stroke="#7dd3fc" stroke-width="4" stroke-linecap="round"/>
</svg>

After

Width:  |  Height:  |  Size: 416 B

+1 -2
View File
@@ -14,7 +14,7 @@
<!--Lets make sure we recognize this as an PWA-->
<link rel="manifest" href="/manifest.json" />
<link rel="icon" type="image/png" href="/assetbroker/manifest/favicon.png" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<!--Lets load standard fonts-->
<link rel="preconnect" href="https://assetbroker.lossless.one/" crossorigin>
@@ -114,7 +114,6 @@
} else {
window.revenueEnabled = false;
}
console.log(`revenue enabled: ${window.revenueEnabled}`);
};
runRevenueCheck();
+18
View File
@@ -0,0 +1,18 @@
{
"name": "Cloudly",
"short_name": "Cloudly",
"description": "Cloudly infrastructure management dashboard",
"start_url": "/",
"scope": "/",
"display": "standalone",
"background_color": "#000000",
"theme_color": "#000000",
"icons": [
{
"src": "/favicon.svg",
"sizes": "any",
"type": "image/svg+xml",
"purpose": "any maskable"
}
]
}
+61 -53
View File
@@ -1,90 +1,98 @@
{
"name": "@serve.zone/cloudly",
"version": "5.2.0",
"private": false,
"version": "5.9.0",
"private": true,
"description": "A comprehensive tool for managing containerized applications across multiple cloud providers using Docker Swarmkit, featuring web, CLI, and API interfaces.",
"type": "module",
"exports": {
".": "./dist/index.js",
"./apiclient": "./dist_apiclient/index.js",
"./cliclient": "./dist_cliclient/index.js",
"./web": "./dist_web/index.js"
".": "./dist_ts/index.js",
"./cliclient": "./dist_ts_cliclient/index.js",
"./web": "./dist_ts_web/index.js"
},
"author": "Task Venture Capital GmbH",
"license": "MIT",
"scripts": {
"test": "(tstest test/ --verbose --logfile --timeout 120)",
"build": "tsbuild tsfolders --web --allowimplicitany && tsbundle website --production",
"build": "tsbuild tsfolders && tsbundle",
"build:docker": "tsdocker build --verbose",
"start": "node cli.js",
"startTs": "node cli.ts.js",
"watch": "tswatch website",
"watch": "tswatch",
"release:docker": "tsdocker push --verbose",
"publish": "tspublish",
"docs": "tsdoc aidoc"
},
"devDependencies": {
"@git.zone/tsbuild": "^2.6.8",
"@git.zone/tsbundle": "^2.5.1",
"@git.zone/tsdoc": "^1.5.2",
"@git.zone/tspublish": "^1.10.3",
"@git.zone/tstest": "^2.3.6",
"@git.zone/tswatch": "^2.2.1",
"@types/node": "^22.0.0"
"@git.zone/tsbuild": "^4.4.1",
"@git.zone/tsbundle": "^2.10.4",
"@git.zone/tsdoc": "^2.0.5",
"@git.zone/tsdocker": "^2.3.0",
"@git.zone/tspublish": "^1.11.7",
"@git.zone/tstest": "^3.6.6",
"@git.zone/tswatch": "^3.3.5",
"@push.rocks/smartnetwork": "^4.7.1",
"@types/node": "^25.8.0"
},
"dependencies": {
"@api.global/typedrequest": "3.1.10",
"@api.global/typedrequest": "3.3.1",
"@api.global/typedrequest-interfaces": "^3.0.19",
"@api.global/typedserver": "^3.0.79",
"@api.global/typedsocket": "^3.0.1",
"@apiclient.xyz/cloudflare": "^6.4.1",
"@apiclient.xyz/docker": "^1.3.5",
"@apiclient.xyz/hetznercloud": "^1.2.0",
"@api.global/typedserver": "^8.4.6",
"@api.global/typedsocket": "^4.1.3",
"@apiclient.xyz/cloudflare": "^7.1.0",
"@apiclient.xyz/docker": "^5.1.4",
"@apiclient.xyz/hetznercloud": "^1.2.1",
"@apiclient.xyz/slack": "^3.0.9",
"@design.estate/dees-catalog": "^1.11.2",
"@design.estate/dees-domtools": "^2.3.3",
"@design.estate/dees-element": "^2.1.2",
"@git.zone/tsrun": "^1.3.3",
"@push.rocks/early": "^4.0.3",
"@push.rocks/npmextra": "^5.3.3",
"@push.rocks/projectinfo": "^5.0.1",
"@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartacme": "^8.0.0",
"@push.rocks/smartbucket": "^3.3.10",
"@push.rocks/smartcli": "^4.0.11",
"@push.rocks/smartclickhouse": "^2.0.17",
"@push.rocks/smartdata": "^5.16.4",
"@push.rocks/smartdelay": "^3.0.5",
"@push.rocks/smartexit": "^1.0.23",
"@design.estate/dees-catalog": "^3.81.0",
"@design.estate/dees-domtools": "^2.5.6",
"@design.estate/dees-element": "^2.2.4",
"@git.zone/tsrun": "^2.0.4",
"@push.rocks/early": "^4.0.4",
"@push.rocks/projectinfo": "^5.1.0",
"@push.rocks/qenv": "^6.1.4",
"@push.rocks/smartacme": "^9.5.0",
"@push.rocks/smartbucket": "^4.6.1",
"@push.rocks/smartcli": "^4.0.21",
"@push.rocks/smartclickhouse": "^2.2.1",
"@push.rocks/smartconfig": "^6.1.1",
"@push.rocks/smartdata": "^7.1.7",
"@push.rocks/smartdelay": "^3.1.0",
"@push.rocks/smartexit": "^2.0.3",
"@push.rocks/smartexpect": "^2.5.0",
"@push.rocks/smartfile": "^11.2.7",
"@push.rocks/smartfile": "^13.1.3",
"@push.rocks/smartguard": "^3.1.0",
"@push.rocks/smartjson": "^5.0.19",
"@push.rocks/smartjwt": "^2.2.1",
"@push.rocks/smartlog": "^3.1.9",
"@push.rocks/smartjson": "^6.0.1",
"@push.rocks/smartjwt": "^2.2.2",
"@push.rocks/smartlog": "^3.2.2",
"@push.rocks/smartlog-destination-clickhouse": "^1.0.13",
"@push.rocks/smartlog-interfaces": "^3.0.2",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartpromise": "^4.2.3",
"@push.rocks/smartrequest": "^4.3.1",
"@push.rocks/smartpromise": "^4.2.4",
"@push.rocks/smartregistry": "^2.9.2",
"@push.rocks/smartrequest": "^5.0.3",
"@push.rocks/smartrx": "^3.0.10",
"@push.rocks/smartssh": "^2.0.1",
"@push.rocks/smartstate": "^2.0.26",
"@push.rocks/smartstream": "^3.2.5",
"@push.rocks/smartstring": "^4.0.15",
"@push.rocks/smartsamba": "^0.2.0",
"@push.rocks/smartssh": "^2.1.0",
"@push.rocks/smartstate": "^2.3.1",
"@push.rocks/smartstream": "^3.4.2",
"@push.rocks/smartstring": "^4.1.1",
"@push.rocks/smartunique": "^3.0.9",
"@push.rocks/taskbuffer": "^3.4.0",
"@push.rocks/webjwt": "^1.0.9",
"@tsclass/tsclass": "^9.2.0"
"@push.rocks/taskbuffer": "^8.0.2",
"@push.rocks/webjwt": "^1.0.10",
"@serve.zone/api": "^5.3.8",
"@serve.zone/interfaces": "^5.10.0",
"@tsclass/tsclass": "^9.5.1"
},
"files": [
"ts/**/*",
"ts_cliclient/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_*/**/*",
"dist_serve/**/*",
"dist_ts/**/*",
"dist_ts_cliclient/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
".smartconfig.json",
"readme.md"
],
"browserslist": [
@@ -126,5 +134,5 @@
"backend",
"security"
],
"packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6"
"packageManager": "pnpm@11.2.2"
}
+4742 -7170
View File
File diff suppressed because it is too large Load Diff
+12 -4
View File
@@ -1,4 +1,12 @@
onlyBuiltDependencies:
- esbuild
- mongodb-memory-server
- puppeteer
minimumReleaseAgeExclude:
- '@serve.zone/api'
- '@serve.zone/interfaces'
allowBuilds:
'@design.estate/dees-catalog': false
cpu-features: true
esbuild: true
mongodb-memory-server: false
puppeteer: false
sharp: false
ssh2: true
+243 -178
View File
@@ -1,258 +1,323 @@
# @serve.zone/cloudly 🚀
# @serve.zone/cloudly
**Multi-cloud orchestration made simple.** Manage containerized applications across cloud providers with Docker Swarmkit, featuring web dashboards, CLI tools, and powerful APIs.
Cloudly is the serve.zone control plane: a TypeScript service and browser dashboard that stores desired infrastructure state, authenticates humans and machines, coordinates clusters, serves an OCI registry, manages workload metadata, and pushes runtime configuration to connected node components.
## 🎯 What is Cloudly?
## Issue Reporting and Security
Cloudly is your command center for multi-cloud infrastructure. It abstracts away the complexity of managing resources across different cloud providers while giving you the power and flexibility you need for modern DevOps workflows.
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
### ✨ Key Features
## Why It Exists
- **🌐 Multi-Cloud Management** - Seamlessly orchestrate resources across Cloudflare, Hetzner, DigitalOcean and more
- **🐳 Docker Swarmkit Integration** - Native container orchestration with production-grade reliability
- **🔐 Secret Management** - Secure handling of credentials, API keys, and sensitive configuration
- **🎨 Web Dashboard** - Beautiful, responsive UI built with modern web components
- **⚡ CLI & API** - Full programmatic control through TypeScript/JavaScript APIs and command-line tools
- **🔒 SSL/TLS Automation** - Automatic certificate provisioning via Let's Encrypt
- **📊 Comprehensive Logging** - Built-in log aggregation and monitoring capabilities
- **🔄 Task Scheduling** - Automated workflows and background job management
Cloudly is the place where serve.zone operators describe what should run. It does not directly run every workload itself. Instead, it keeps the authoritative desired state in MongoDB and exposes TypedRequest/TypedSocket APIs so runtime components can reconcile that state where the containers actually live.
## 🚀 Quick Start
The current runtime pattern is reverse-connect:
### Installation
```bash
# Install the main package
pnpm add @serve.zone/cloudly
# Or install the CLI globally
pnpm add -g @serve.zone/cli
# Or just the API client
pnpm add @serve.zone/api
```text
browser / CLI / SDK
-> Cloudly HTTP + TypedSocket API
-> MongoDB-backed managers
-> S3-backed image and artifact storage
<- Coreflow cluster agents connect outward
-> Docker Swarm reconciliation
-> Coretraffic routing updates
-> Corestore platform resources and backups
```
### Basic Setup
## What Cloudly Manages
```typescript
Cloudly currently coordinates these areas:
- **Authentication and identity**: human admin login, JWT identities, machine tokens, and cluster identities.
- **Clusters**: desired cluster records and machine users used by Coreflow to authenticate back to Cloudly.
- **Services**: workload definitions, image references, domains, ports, scale factors, secret bundles, volumes, and deployment metadata.
- **Deployments**: deployment records, node placement metadata, health/resource fields, restart/scale API stubs, and DNS activation/deactivation hooks.
- **Images and registries**: image metadata, S3-backed image storage, external registry records, and an embedded OCI registry mounted at `/v2`.
- **Secrets**: secret groups and bundles that Coreflow flattens into Docker secrets for workloads.
- **Domains and DNS**: domain records, DNS entries, and optional domain sync from a dcrouter external gateway.
- **Platform bindings**: capabilities such as `database`, `objectstorage`, `logging`, `backup`, and RPC-style platform services that Coreflow/Corestore can reconcile.
- **Backups**: backup records, service backup/restore requests, scheduled backup tasks, and archive replication handshakes with Coreflow/Corestore.
- **BaseOS**: managed BaseOS node registration, heartbeat handling, desired-state response, image build tracking, and image download URLs.
- **CoreBuild workers**: selection of external build workers for BaseOS ISO and balena raw-image artifact generation.
- **Tasks**: TaskBuffer-backed operational tasks with execution history, metrics, logs, manual triggers, cancellation, and cron schedules.
- **Node and bare-metal inventory**: Hetzner-backed node creation paths and bare-metal metadata records where configured.
- **Dashboard**: a web component UI rendered from `ts_web` with views for overview, settings, secrets, clusters, external registries, images, services, deployments, tasks, domains, DNS, mail/log/storage/database shells, backups, and BaseOS.
## Runtime Components
| Component | Role |
| --- | --- |
| `Cloudly` | Main service coordinator. Creates connectors and managers, then starts the API server. |
| `CloudlyServer` | TypedServer/TypedSocket HTTP server, dashboard static server, OCI registry HTTP bridge, and BaseOS HTTP endpoints. |
| `MongodbConnector` | SmartData persistence layer for Cloudly records. |
| `CloudflareConnector` | Optional Cloudflare account used by ACME DNS-01 when `cloudflareToken` is configured in settings. |
| `LetsencryptConnector` | SmartACME certificate issuance and certificate lookup. |
| `CloudlyCoreflowManager` | Authenticates Coreflow, returns cluster config payloads, and pushes config updates to connected Coreflow clients. |
| `CloudlyJumpManager` | Creates short-lived Jump Codes for onboarding existing systems into clusters. |
| `CloudlyRegistryManager` | Embedded OCI registry backed by configured S3 storage, including deploy-on-push metadata updates. |
| `CloudlyBaseOsManager` | BaseOS registration, heartbeat, image build orchestration, worker selection, and artifact downloads. |
| `CloudlyBackupManager` | Service backup/restore orchestration and remote archive object replication. |
| `CloudlyTaskManager` | Registers predefined and runtime tasks, tracks task executions, schedules cron jobs, and exposes task APIs. |
| `CloudlySettingsManager` | Stores runtime settings in MongoDB, masks sensitive values for API responses, and refreshes gateway/Coreflow state after relevant changes. |
## Configuration
Cloudly uses `@push.rocks/smartconfig` `AppData` with environment mappings. The runtime entry point loads `.nogit`/environment values through `@push.rocks/qenv`, and embedded callers can override values by constructing `new Cloudly(config)` programmatically.
Required runtime configuration:
| Variable | Purpose |
| --- | --- |
| `SERVEZONE_ENVIRONMENT` | ACME/runtime environment, currently `production` or `integration`. |
| `SERVEZONE_URL` | Public Cloudly hostname without protocol. |
| `SERVEZONE_PORT` | Public API/dashboard port as a string. |
| `SERVEZONE_SSLMODE` | `none`, `external`, or `letsencrypt`. |
| `SERVEZONE_ADMINACCOUNT` | First-run admin bootstrap in `username:password` format. |
| `MONGODB_URL` | MongoDB connection URL used by SmartData. |
| `MONGODB_NAME` | MongoDB database name. |
| `MONGODB_USER` | MongoDB username. |
| `MONGODB_PASS` | MongoDB password. |
| `S3_ENDPOINT` | S3-compatible endpoint for registry, images, and artifacts. |
| `S3_ACCESSKEY` | S3 access key. |
| `S3_SECRETKEY` | S3 secret key. |
| `S3_BUCKET` | S3 bucket name. |
| `S3_PORT` | S3 endpoint port. |
| `S3_USESSL` | Boolean SSL flag for the S3 endpoint. |
Common optional settings are stored through the Cloudly settings manager rather than direct environment variables:
| Setting | Purpose |
| --- | --- |
| `cloudflareToken` | Enables Cloudflare-backed ACME DNS-01 challenges. |
| `hetznerToken` | Enables Hetzner node and bare-metal provisioning paths. |
| `baseosJoinToken` | Allows BaseOS devices to enroll without a one-time image provisioning token. |
| `corebuildWorkersJson` | JSON array of CoreBuild worker URLs or `{ "url", "token", "id" }` objects. |
| `corebuildWorkerUrl` / `corebuildWorkerToken` | Legacy single-worker CoreBuild settings. |
| `dcrouterGatewayUrl` / `dcrouterGatewayApiToken` | Optional external gateway integration for domain and route sync. |
| `dcrouterWorkHosterId` | Optional stable external gateway work hoster ID; defaults to the cluster ID. |
| `dcrouterTargetHost` / `dcrouterTargetPort` | Optional target address that dcrouter should forward workload traffic to. |
Optional runtime environment variables:
| Variable | Purpose |
| --- | --- |
| `SERVEZONE_INSTALL_DEMO_DATA` | Runs the destructive demo data installer when set to `true`. |
| `CLOUDLY_BACKUP_CRON` | Enables the scheduled `backup-all-services` task with the supplied cron expression. |
| `CLOUDLY_BACKUP_KEEP_LAST` | Number of completed/failed backups to retain per service; defaults to `24`. |
| `CLOUDLY_BACKUP_TARGET_TYPE` | Remote archive replication target, currently `s3` or `smb`. |
| `CLOUDLY_BACKUP_TARGET_PREFIX` | Remote backup path prefix; defaults to `serve.zone-backups`. |
| `CLOUDLY_BASEOS_IMAGE_CLEANUP_INTERVAL_MS` | BaseOS image artifact cleanup interval; defaults to 12 hours. |
For backup replication with `CLOUDLY_BACKUP_TARGET_TYPE=s3`, set `CLOUDLY_BACKUP_S3_ENDPOINT`, `CLOUDLY_BACKUP_S3_ACCESS_KEY`, `CLOUDLY_BACKUP_S3_SECRET_KEY`, and `CLOUDLY_BACKUP_S3_BUCKET`. Optional S3 variables are `CLOUDLY_BACKUP_S3_REGION`, `CLOUDLY_BACKUP_S3_PORT`, and `CLOUDLY_BACKUP_S3_USE_SSL`.
For backup replication with `CLOUDLY_BACKUP_TARGET_TYPE=smb`, set `CLOUDLY_BACKUP_SMB_HOST` and `CLOUDLY_BACKUP_SMB_SHARE`. Optional SMB variables are `CLOUDLY_BACKUP_SMB_PORT`, `CLOUDLY_BACKUP_SMB_USERNAME`, `CLOUDLY_BACKUP_SMB_PASSWORD`, and `CLOUDLY_BACKUP_SMB_DOMAIN`.
## Starting Cloudly
Install and build with pnpm:
```sh
pnpm install
pnpm build
pnpm start
```
Run the TypeScript entry point during development:
```sh
pnpm run startTs
```
Start from code when embedding the control plane in another Node.js process:
```ts
import { Cloudly } from '@serve.zone/cloudly';
// Initialize Cloudly with your configuration
const cloudly = new Cloudly({
cfToken: process.env.CLOUDFLARE_TOKEN,
hetznerToken: process.env.HETZNER_TOKEN,
environment: 'production',
letsEncryptEmail: 'certs@example.com',
publicUrl: 'cloudly.example.com',
publicPort: 443,
publicPort: '443',
sslMode: 'external',
servezoneAdminaccount: 'admin:change-me',
mongoDescriptor: {
mongoDbUrl: process.env.MONGODB_URL,
mongoDbName: 'cloudly',
mongoDbUser: process.env.MONGODB_USER,
mongoDbPass: process.env.MONGODB_PASS,
}
},
s3Descriptor: {
endpoint: process.env.S3_ENDPOINT,
accessKey: process.env.S3_ACCESSKEY,
accessSecret: process.env.S3_SECRETKEY,
bucketName: process.env.S3_BUCKET,
port: process.env.S3_PORT,
useSsl: true,
},
});
// Start the platform
await cloudly.start();
console.log('🎉 Cloudly is running!');
```
## 🏗️ Architecture
Set `SERVEZONE_INSTALL_DEMO_DATA=true` only when you intentionally want the demo data installer to run. The code labels that path destructive.
Cloudly follows a modular architecture with clear separation of concerns:
## API Model
```
┌─────────────────────────────────────────────┐
│ Web Dashboard (UI) │
├─────────────────────────────────────────────┤
│ API Layer (TypedRouter) │
├─────────────────────────────────────────────┤
│ Core Managers │
│ ┌─────────┐ ┌─────────┐ ┌─────────┐ │
│ │ Cluster │ │ Image │ │ Secret │ ... │
│ └─────────┘ └─────────┘ └─────────┘ │
├─────────────────────────────────────────────┤
│ Cloud Connectors │
│ ┌──────────┐ ┌─────────┐ ┌──────────----┐ │
│ │Cloudflare│ │ Hetzner │ │ DigitalOcean │ │
│ └──────────┘ └─────────┘ └──────────----┘ │
└─────────────────────────────────────────────┘
```
Cloudly exposes a single composed TypedRouter. Managers add their own typed handlers to the main router, and `CloudlyServer` exposes that router through the HTTP/WebSocket server.
## 💻 Core Components
On first startup, Cloudly bootstraps the first human admin from `SERVEZONE_ADMINACCOUNT`. Human clients authenticate through `adminLoginWithUsernameAndPassword`; machine clients authenticate through `getIdentityByToken`. Cluster creation creates a machine user and token for Coreflow.
### 🔧 Managers
Typical consumers use `@serve.zone/api`:
- **AuthManager** - Identity and access management
- **ClusterManager** - Docker Swarm cluster orchestration
- **ImageManager** - Container image lifecycle management
- **SecretManager** - Secure credential storage and distribution
- **ServerManager** - Cloud server provisioning and management
- **TaskManager** - Background job scheduling and execution
```ts
import { CloudlyApiClient } from '@serve.zone/api';
### 🔌 Connectors
- **CloudflareConnector** - DNS, CDN, and edge services
- **LetsencryptConnector** - Automatic SSL certificate provisioning
- **MongodbConnector** - Database persistence layer
- **HetznerConnector** - German cloud infrastructure
- **DigitalOceanConnector** - Developer-friendly cloud resources
## 📚 Usage Examples
### Managing Clusters
```typescript
// Create a new cluster
const cluster = await cloudly.clusterManager.createCluster({
name: 'production-cluster',
region: 'eu-central',
nodeCount: 3
const client = new CloudlyApiClient({
registerAs: 'admin-tool',
cloudlyUrl: 'https://cloudly.example.com',
});
// Deploy a service
await cluster.deployService({
name: 'api-service',
image: 'myapp:latest',
replicas: 3,
ports: [{ published: 80, target: 3000 }]
});
await client.start();
const identity = await client.loginWithUsernameAndPassword('admin', 'change-me');
const clusters = await client.cluster.getClusters();
```
### Secret Management
Machine clients such as Coreflow authenticate with `getIdentityByToken`, request a stateful identity, and tag their WebSocket connection. That lets Cloudly push configuration to already-connected Coreflow instances instead of opening inbound connections to cluster nodes.
```typescript
// Create a secret group
const secretGroup = await cloudly.secretManager.createSecretGroup({
name: 'api-credentials',
secrets: [
{ key: 'API_KEY', value: process.env.API_KEY },
{ key: 'DB_PASSWORD', value: process.env.DB_PASSWORD }
]
});
## Cluster Flow
// Create a bundle for deployment
const bundle = await cloudly.secretManager.createSecretBundle({
name: 'production-secrets',
secretGroups: [secretGroup]
});
The implemented cluster flow is intentionally simple:
1. An admin creates a Cloudly cluster record.
2. Cloudly creates a machine user with a long-lived cluster token.
3. Coreflow starts on a Docker Swarm manager node with `CLOUDLY_URL` and `JUMPCODE`.
4. Coreflow authenticates to Cloudly and requests the cluster configuration payload.
5. Cloudly returns cluster data, workload services, platform bindings, provider configs, and optional external gateway configuration.
6. Coreflow reconciles Docker networks, base services, workload services, secrets, volumes, platform bindings, backups, and routing.
When service, platform, or gateway settings change, Cloudly pushes updated config to connected Coreflow clients where supported.
### Jump Codes for Existing Systems
Admins can generate a short-lived, single-use Jump Code for a cluster. The dashboard displays a command in this form:
```sh
curl -fsSL 'https://cloudly.example.com/jump/<code>' | sudo bash
```
### DNS Management
The public `/jump/<code>` URL renders a browser landing page for humans and a shell bootstrap script for `curl`/CLI clients. The script installs the required host tooling, claims the code through `POST /jump/v1/claim`, receives the cluster runtime token, and starts Spark in `coreflow-node` mode. The long-lived cluster token is never displayed in the dashboard command.
```typescript
// Create DNS records via Cloudflare
const record = await cloudly.cloudflareConnector.createDNSRecord(
'example.com',
'api.example.com',
'A',
'192.168.1.1'
);
Jump Codes expire by default after 30 minutes and are consumed on first successful claim.
## Registry and Deploy-On-Push
Cloudly serves an OCI registry under `/v2` through `CloudlyRegistryManager`. The registry uses configured S3 storage and issues OCI tokens from Cloudly authentication state.
For Cloudly-managed services, `getServiceRegistryTarget()` creates stable registry targets like:
```text
<cloudly-host>/workloads/<service-name>-<service-id-prefix>:<tag>
```
### Web Dashboard
Registry push hooks record tag/digest metadata on the linked image and service. Unless `deployOnPush` is explicitly `false`, a successful push updates the service image version and asks connected Coreflow clients to reconcile.
```typescript
import { html } from '@design.estate/dees-element';
Registry token requests use HTTP Basic credentials against Cloudly users. User passwords and unexpired user tokens are accepted; push/delete scopes require an admin user or a token with the `admin` assigned role.
// Create a custom dashboard view
const dashboard = html`
<cloudly-dashboard>
<cloudly-view-clusters></cloudly-view-clusters>
<cloudly-view-dns></cloudly-view-dns>
<cloudly-view-images></cloudly-view-images>
</cloudly-dashboard>
`;
## BaseOS and CoreBuild
document.body.appendChild(dashboard);
```
Cloudly can manage BaseOS nodes and image builds:
## 🛠️ CLI Usage
- BaseOS devices register through `POST /baseos/v1/nodes/register` and heartbeat through `POST /baseos/v1/nodes/heartbeat`.
- A configured `baseosJoinToken` accepts generic device enrollment.
- BaseOS image builds create one-time provisioning tokens that are embedded in generated images.
- Cloudly selects a CoreBuild worker based on `/corebuild/v1/capabilities` and sends the build to `/corebuild/v1/jobs/baseos-image`.
- Supported build kinds are `ubuntu-iso` and `balena-raw`; Raspberry Pi builds use `balena-raw`.
- Supported architecture values are `amd64`, `arm64`, and `rpi`.
- Completed artifacts are stored in the configured S3 bucket and served through short-lived `/baseos/v1/images/:buildId/download` URLs.
The CLI provides quick access to all Cloudly features:
CoreBuild worker configuration can use `corebuildWorkersJson` for multiple workers or the legacy `corebuildWorkerUrl` and `corebuildWorkerToken` settings for one worker.
```bash
# Login to your Cloudly instance
servezone login --url https://cloudly.example.com
## Backups and Corestore
# List clusters
servezone clusters list
Cloudly owns backup records and user-facing backup/restore requests. Coreflow executes the cluster-local work, and Corestore snapshots volumes, database resources, object storage resources, and archive objects.
# Deploy a service
servezone deploy --cluster prod --image myapp:latest
The backup path includes:
# Manage secrets
servezone secrets create --name api-key --value "secret123"
- `createServiceBackup` and `restoreServiceBackup` typed requests for admins.
- `executeServiceBackup` and `executeServiceRestore` requests from Cloudly to Coreflow.
- Corestore volume/resource snapshot and restore endpoints behind Coreflow.
- Optional archive replication through `prepareBackupReplication`, `uploadBackupArchiveObject`, `completeBackupReplication`, `getBackupArchiveManifest`, and `downloadBackupArchiveObject`.
- Optional scheduled `backup-all-services` task when `CLOUDLY_BACKUP_CRON` is set.
# View logs
servezone logs --service api-service --follow
```
Manual `createServiceBackup` requests expect Coreflow to complete remote archive replication. Cloudly validates archive object size and SHA-256 checksums, writes a manifest, records target metadata, and marks completed backups as `replicated`. Restores read the manifest and objects back through the configured target writer.
## 📦 Package Exports
## Task Automation
This monorepo publishes multiple packages:
Cloudly registers a TaskBuffer-backed task manager. The API and dashboard can list tasks, trigger tasks manually, inspect execution logs/metrics, and request cancellation for running tasks.
- **@serve.zone/cloudly** - Main orchestration platform
- **@serve.zone/api** - TypeScript/JavaScript API client
- **@serve.zone/cli** - Command-line interface
- **@serve.zone/interfaces** - Shared TypeScript interfaces
Predefined tasks currently include:
## 🔒 Security Features
| Task | Purpose |
| --- | --- |
| `cloudflare-domain-sync` | Imports and updates domains from configured Cloudflare zones. |
| `dns-sync` | Iterates DNS entries marked as external; provider sync is currently a placeholder. |
| `cert-renewal` | Checks activated domains for certificate renewal; renewal logic is currently a placeholder. |
| `cleanup` | Removes old task executions and contains placeholders for log/image cleanup. |
| `health-check` | Iterates deployments and records health metrics; runtime health checks are currently placeholders. |
| `resource-report` | Generates node resource metrics; values are currently placeholders until runtime metrics are wired in. |
| `db-maintenance` | Maintenance shell for database optimization tasks. |
| `security-scan` | Security scan shell for exposed ports, image freshness, and weak configuration checks. |
| `docker-cleanup` | Docker cleanup shell for containers, images, volumes, and networks. |
| `backup-all-services` | Registered by the backup manager and enabled only when `CLOUDLY_BACKUP_CRON` is set. |
- **End-to-end encryption** for secrets
- **Role-based access control** (RBAC)
- **Automatic SSL/TLS** certificate management
- **Secure token-based authentication**
- **Audit logging** for compliance
## External Gateway Integration
## 🚢 Production Ready
Cloudly can integrate with a dcrouter gateway when the gateway URL and API token are present in settings. The current integration syncs externally available domains into Cloudly and passes an external gateway route configuration to Coreflow. Coreflow can then ask dcrouter for certificates and synchronize public routes while still routing to cluster-local Coretraffic.
Cloudly is battle-tested in production environments managing:
- High-traffic web applications
- Microservice architectures
- CI/CD pipelines
- Data processing workloads
- Real-time communication systems
## Development
## 🤝 Development
Common commands:
```bash
# Clone the repository
git clone https://gitlab.com/servezone/private/cloudly.git
# Install dependencies
```sh
pnpm install
# Run tests
pnpm test
# Build the project
pnpm build
# Start development mode
pnpm watch
pnpm test
pnpm run build:docker
pnpm run release:docker
pnpm run docs
```
## 📖 Documentation
Important paths:
For detailed documentation, API references, and guides, visit our [documentation site](https://cloudly.serve.zone).
| Path | Purpose |
| --- | --- |
| `ts/index.ts` | CLI/runtime entry point exporting `runCli`, `Cloudly`, and `ICloudlyConfig`. |
| `ts/classes.cloudly.ts` | Main service coordinator and startup order. |
| `ts/classes.server.ts` | API/dashboard server, registry bridge, and BaseOS HTTP routes. |
| `ts/manager.*` | Domain managers for auth, clusters, services, images, registry, platform, backups, BaseOS, and more. |
| `ts/connector.*` | External system connectors for MongoDB, Cloudflare, and Let's Encrypt. |
| `ts_web/` | Browser dashboard web components. |
| `ts_cliclient/` | Published `@serve.zone/cli` submodule. |
## Accuracy Notes
The package metadata and settings schema include fields for several cloud providers. The code paths currently exercised in this repository are Cloudflare for ACME DNS-01 and domain sync, Hetzner for selected node/bare-metal provisioning paths, S3-compatible storage, SMB/S3 backup archive targets, MongoDB/SmartData, CoreBuild, Coreflow, Corestore, and optional dcrouter integration. Several provider connection tests and predefined tasks are configuration checks or implementation shells; verify provider-specific behavior in the relevant manager before relying on it operationally.
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [license](./license) file.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
### Trademarks
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
### Company Information
Task Venture Capital GmbH
Registered at District court Bremen HRB 35230 HB, Germany
Registered at District Court Bremen HRB 35230 HB, Germany
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
For any legal inquiries or further information, please contact us via email at hello@task.vc.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
+8 -10
View File
@@ -1,4 +1,4 @@
import { expect, tap } from '@push.rocks/tapbundle';
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { Qenv } from '@push.rocks/qenv';
const testQenv = new Qenv('./', './.nogit/');
@@ -12,17 +12,15 @@ let testCloudly: cloudly.Cloudly;
tap.test('first test', async () => {
const cloudlyConfig: cloudly.ICloudlyConfig = {
cfToken: await testQenv.getEnvVarOnDemand('CF_TOKEN'),
environment: 'integration',
letsEncryptEmail: await testQenv.getEnvVarOnDemand('LETSENCRYPT_EMAIL'),
publicUrl: await testQenv.getEnvVarOnDemand('SERVEZONE_URL'),
publicPort: await testQenv.getEnvVarOnDemand('SERVEZONE_PORT'),
letsEncryptEmail: await testQenv.getEnvVarOnDemandStrict('LETSENCRYPT_EMAIL'),
publicUrl: await testQenv.getEnvVarOnDemandStrict('SERVEZONE_URL'),
publicPort: await testQenv.getEnvVarOnDemandStrict('SERVEZONE_PORT'),
mongoDescriptor: {
mongoDbName: await testQenv.getEnvVarOnDemand('MONGODB_DATABASE'),
mongoDbPass: await testQenv.getEnvVarOnDemand('MONGODB_PASSWORD'),
mongoDbUrl: await testQenv.getEnvVarOnDemand('MONGODB_URL')
mongoDbName: await testQenv.getEnvVarOnDemandStrict('MONGODB_DATABASE'),
mongoDbPass: await testQenv.getEnvVarOnDemandStrict('MONGODB_PASSWORD'),
mongoDbUrl: await testQenv.getEnvVarOnDemandStrict('MONGODB_URL')
},
digitalOceanToken: await testQenv.getEnvVarOnDemand('DIGITALOCEAN_TOKEN')
};
testCloudly = new cloudly.Cloudly(cloudlyConfig);
expect(testCloudly).toBeInstanceOf(cloudly.Cloudly);
@@ -36,4 +34,4 @@ tap.test('should end the service', async () => {
await testCloudly.stop();
});
tap.start();
export default tap.start();
+10 -12
View File
@@ -1,10 +1,10 @@
import { expect, tap } from '@push.rocks/tapbundle';
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { Qenv } from '@push.rocks/qenv';
const testQenv = new Qenv('./', './.nogit/');
delete process.env.CLI_CALL;
import * as cloudly from '../ts/index';
import * as cloudly from '../ts/index.js';
process.env.TESTING_CLOUDLY = 'true';
@@ -12,20 +12,18 @@ let testCloudly: cloudly.Cloudly;
tap.test('first test', async () => {
const cloudlyConfig: cloudly.ICloudlyConfig = {
cfToken: testQenv.getEnvVarOnDemand('CF_TOKEN'),
environment: 'integration',
letsEncryptEmail: testQenv.getEnvVarOnDemand('LETSENCRYPT_EMAIL'),
publicUrl: testQenv.getEnvVarOnDemand('SERVEZONE_URL'),
publicPort: testQenv.getEnvVarOnDemand('SERVEZONE_PORT'),
letsEncryptEmail: await testQenv.getEnvVarOnDemandStrict('LETSENCRYPT_EMAIL'),
publicUrl: await testQenv.getEnvVarOnDemandStrict('SERVEZONE_URL'),
publicPort: await testQenv.getEnvVarOnDemandStrict('SERVEZONE_PORT'),
mongoDescriptor: {
mongoDbName: testQenv.getEnvVarOnDemand('MONGODB_DATABASE'),
mongoDbPass: testQenv.getEnvVarOnDemand('MONGODB_PASSWORD'),
mongoDbUrl: testQenv.getEnvVarOnDemand('MONGODB_URL')
mongoDbName: await testQenv.getEnvVarOnDemandStrict('MONGODB_DATABASE'),
mongoDbPass: await testQenv.getEnvVarOnDemandStrict('MONGODB_PASSWORD'),
mongoDbUrl: await testQenv.getEnvVarOnDemandStrict('MONGODB_URL')
},
digitalOceanToken: testQenv.getEnvVarOnDemand('DIGITALOCEAN_TOKEN')
};
testCloudly = new cloudly.Cloudly(cloudlyConfig);
expect(testCloudly).to.be.instanceof(cloudly.Cloudly);
expect(testCloudly).toBeInstanceOf(cloudly.Cloudly);
});
tap.test('should init servezone', async () => {
@@ -36,4 +34,4 @@ tap.test('should end the service', async () => {
await testCloudly.stop();
});
tap.start();
export default tap.start();
+38 -9
View File
@@ -1,32 +1,57 @@
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import { Qenv } from '@push.rocks/qenv';
import { SmartNetwork } from '@push.rocks/smartnetwork';
import { tap } from '@git.zone/tstest/tapbundle';
import { TapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
const tapNodeTools = new TapNodeTools(tap);
const testQenv = new Qenv('./', './.nogit/');
import * as cloudly from '../../ts/index.js';
const stopFunctions: Array<() => Promise<void>> = [];
const tapToolsNodeMod = await import('@git.zone/tstest/tapbundle_node');
const smartmongo = await tapToolsNodeMod.tapNodeTools.createSmartmongo();
const getPublicPort = async () => {
if (process.env.SERVEZONE_TEST_PORT) {
return process.env.SERVEZONE_TEST_PORT;
}
const smartnetwork = new SmartNetwork();
const publicPort = await smartnetwork.findFreePort(30000, 40000, { randomize: true });
if (!publicPort) {
throw new Error('Could not find a free Cloudly test port in range 30000-40000');
}
return String(publicPort);
};
const smartmongo = await tapNodeTools.createSmartmongo();
stopFunctions.push(async () => {
await smartmongo.stopAndDumpToDir('./.nogit/mongodump');
});
const smarts3 = await tapToolsNodeMod.tapNodeTools.createSmarts3();
await smarts3.createBucket('cloudly_test_bucket');
const smartstorage = await tapNodeTools.createSmartStorage();
await smartstorage.createBucket('cloudly_test_bucket');
stopFunctions.push(async () => {
await smarts3.stop();
await smartstorage.stop();
});
export const testCloudlyAdminAccount = {
username: 'testadmin',
password: 'testpassword',
};
export const testCloudlyConfig: cloudly.ICloudlyConfig = {
cfToken: await testQenv.getEnvVarOnDemand('CF_TOKEN'),
environment: 'integration',
letsEncryptEmail: 'test@serve.zone',
publicUrl: '127.0.0.1',
publicPort: '8080',
publicPort: await getPublicPort(),
mongoDescriptor: await smartmongo.getMongoDescriptor(),
s3Descriptor: await smarts3.getS3Descriptor({
s3Descriptor: await smartstorage.getStorageDescriptor({
bucketName: 'cloudly_test_bucket'
}),
sslMode: 'none',
servezoneAdminaccount: `${testCloudlyAdminAccount.username}:${testCloudlyAdminAccount.password}`,
...(() => {
if (process.env.NPMCI_SECRET01) {
return {
@@ -36,7 +61,11 @@ export const testCloudlyConfig: cloudly.ICloudlyConfig = {
})(),
};
await tapToolsNodeMod.tapNodeTools.testFileProvider.getDockerAlpineImageAsLocalTarball();
const alpineImageTarballPath = path.join(process.cwd(), '.nogit/testfiles/alpine.tar');
const existingAlpineImageTarball = await fs.stat(alpineImageTarballPath).catch(() => undefined);
if (!existingAlpineImageTarball?.isFile() || existingAlpineImageTarball.size === 0) {
await tapNodeTools.testFileProvider.getDockerAlpineImageAsLocalTarball();
}
export const createCloudly = async () => {
const cloudlyInstance = new cloudly.Cloudly(testCloudlyConfig);
+350 -16
View File
@@ -2,12 +2,20 @@ import { tap, expect } from '@git.zone/tstest/tapbundle';
import * as helpers from './helpers/index.js';
import * as cloudly from '../ts/index.js';
import * as cloudlyApiClient from '../ts_apiclient/index.js';
import { Image } from '../ts_apiclient/classes.image.js';
import * as cloudlyApiClient from '@serve.zone/api';
let testCloudly: cloudly.Cloudly;
let testClient: cloudlyApiClient.CloudlyApiClient;
const logErrorDetails = (errorArg: unknown) => {
if (errorArg instanceof Error) {
console.error(` - Error message: ${errorArg.message}`);
console.error(` - Error stack:`, errorArg.stack);
return;
}
console.error(` - Error:`, errorArg);
};
tap.preTask('should start cloudly', async () => {
testCloudly = await helpers.createCloudly();
await testCloudly.start();
@@ -32,7 +40,7 @@ tap.preTask('should create a new machine user for testing', async () => {
console.log(` - Username: ${machineUser.data.username}`);
console.log(` - Role: ${machineUser.data.role}`);
console.log(` - Token: 'test'`);
console.log(` - Token roles: ${machineUser.data.tokens[0].assignedRoles}`);
console.log(` - Token roles: ${machineUser.data.tokens?.[0]?.assignedRoles?.join(', ') ?? ''}`);
await machineUser.save();
console.log('✅ PreTask: First machine user saved successfully');
});
@@ -57,8 +65,8 @@ tap.test('DEBUG: Check existing users', async () => {
console.log(` - User: ${user.data.username} (ID: ${user.id})`);
console.log(` - Type: ${user.data.type}`);
console.log(` - Role: ${user.data.role}`);
console.log(` - Tokens: ${user.data.tokens.length}`);
for (const token of user.data.tokens) {
console.log(` - Tokens: ${user.data.tokens?.length ?? 0}`);
for (const token of user.data.tokens ?? []) {
console.log(` - Token: '${token.token}' | Roles: ${token.assignedRoles?.join(', ')}`);
}
}
@@ -79,13 +87,341 @@ tap.test('should get an identity', async () => {
expect(identity).toBeTruthy();
} catch (error) {
console.error('❌ Failed to get identity:');
console.error(` - Error message: ${error.message}`);
console.error(` - Error stack:`, error.stack);
logErrorDetails(error);
throw error;
}
});
let image: Image;
tap.test('should create and consume node jump codes', async () => {
const cluster = await testClient.cluster.createCluster('Jump Code Test Cluster');
const createJumpCommandTR = testClient.typedsocketClient.createTypedRequest<any>('createNodeJumpCommand');
const jumpCommand = await createJumpCommandTR.fire({
identity: testClient.identity,
clusterId: cluster.id,
});
expect(jumpCommand.jumpUrl.includes('/jump/')).toBeTrue();
expect(jumpCommand.command.includes(jumpCommand.jumpUrl)).toBeTrue();
const setupResponse = await fetch(jumpCommand.jumpUrl, {
headers: {
accept: '*/*',
'user-agent': 'curl/8.0',
},
});
const setupScript = await setupResponse.text();
expect(setupResponse.status).toEqual(200);
expect(setupScript.includes('spark installdaemon --mode=coreflow-node')).toBeTrue();
const claimResponse = await fetch(
`http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}/jump/v1/claim`,
{
method: 'POST',
headers: {
'content-type': 'application/json',
},
body: JSON.stringify({ jumpCode: jumpCommand.jumpCode, hostname: 'jump-code-test-node' }),
},
);
const claimBody = await claimResponse.json();
expect(claimResponse.status).toEqual(200);
expect(claimBody.accepted).toBeTrue();
expect(claimBody.nodeId).toBeTruthy();
expect(claimBody.coreflowJumpCode).toBeTruthy();
const secondClaimResponse = await fetch(
`http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}/jump/v1/claim`,
{
method: 'POST',
headers: {
'content-type': 'application/json',
},
body: JSON.stringify({ jumpCode: jumpCommand.jumpCode }),
},
);
const secondClaimBody = await secondClaimResponse.json();
expect(secondClaimResponse.status).toEqual(400);
expect(secondClaimBody.accepted).toBeFalse();
});
tap.test('should expose the OCI registry endpoint', async () => {
const response = await fetch(
`http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}/v2/`,
);
expect(response.status).toEqual(200);
expect(response.headers.get('docker-distribution-api-version')).toEqual('registry/2.0');
});
tap.test('should require authentication for OCI registry tokens', async () => {
const response = await fetch(
`http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}/v2/token?service=cloudly&scope=repository:test/app:pull`,
);
expect(response.status).toEqual(401);
});
tap.test('should issue OCI registry tokens for the initial admin', async () => {
const credentials = Buffer.from(
`${helpers.testCloudlyAdminAccount.username}:${helpers.testCloudlyAdminAccount.password}`,
).toString('base64');
const response = await fetch(
`http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}/v2/token?service=cloudly&scope=repository:test/app:pull,push`,
{
headers: {
Authorization: `Basic ${credentials}`,
},
},
);
const body = await response.json();
expect(response.status).toEqual(200);
expect(body.token).toBeTruthy();
expect(body.access_token).toEqual(body.token);
});
tap.test('should deny OCI registry push tokens for non-admin users', async () => {
const readonlyUsername = 'registry-readonly';
const readonlyToken = 'registry-readonly-token';
const readonlyUser = new testCloudly.authManager.CUser();
readonlyUser.id = await testCloudly.authManager.CUser.getNewId();
readonlyUser.data = {
type: 'machine',
username: readonlyUsername,
password: readonlyToken,
tokens: [{
token: readonlyToken,
expiresAt: Date.now() + 3600 * 1000,
assignedRoles: [],
}],
role: 'user',
};
await readonlyUser.save();
const credentials = Buffer.from(`${readonlyUsername}:${readonlyToken}`).toString('base64');
const pullResponse = await fetch(
`http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}/v2/token?service=cloudly&scope=repository:test/readonly:pull`,
{
headers: {
Authorization: `Basic ${credentials}`,
},
},
);
expect(pullResponse.status).toEqual(200);
const pushResponse = await fetch(
`http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}/v2/token?service=cloudly&scope=repository:test/readonly:pull,push`,
{
headers: {
Authorization: `Basic ${credentials}`,
},
},
);
expect(pushResponse.status).toEqual(403);
});
tap.test('should expose generated service registry targets', async () => {
const image = await testClient.image.createImage({
name: 'Registry Target Test Image',
description: 'Image used by the registry target test',
});
const service = await testClient.services.createService({
name: 'Registry Target Test Service',
description: 'Service used by the registry target test',
imageId: image.id,
imageVersion: 'latest',
environment: {},
secretBundleId: '',
serviceCategory: 'workload',
deploymentStrategy: 'custom',
scaleFactor: 1,
balancingStrategy: 'round-robin',
ports: {
web: 3000,
},
domains: [],
deploymentIds: [],
});
const registryTarget = await testClient.services.getRegistryTarget(service.id, 'latest');
expect(registryTarget.protocol).toEqual('oci');
expect(registryTarget.registryHost).toEqual(`${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}`);
expect(registryTarget.repository.startsWith('workloads/registry-target-test-service-')).toBeTrue();
expect(registryTarget.repository.split('/').every((partArg) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(partArg))).toBeTrue();
expect(registryTarget.imageUrl).toEqual(`${registryTarget.registryHost}/${registryTarget.repository}:latest`);
const refreshedService = await testClient.services.getServiceById(service.id);
expect(refreshedService.data.registryTarget?.imageUrl).toEqual(registryTarget.imageUrl);
});
tap.test('should trim truncated registry repository suffixes', async () => {
const registryTarget = testCloudly.registryManager.getServiceRegistryTarget({
id: 'service-5gv-123456',
data: {
name: 'Registry Target Test Service',
},
} as any);
expect(registryTarget.repository).toEqual('workloads/registry-target-test-service-service-5gv');
expect(registryTarget.repository.split('/').every((partArg) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(partArg))).toBeTrue();
});
tap.test('should push service config updates to connected coreflows', async (toolsArg) => {
const cluster = await testClient.cluster.createCluster('Registry Config Push Test Cluster');
const persistedCluster = await testCloudly.clusterManager.getConfigBy_ConfigID(cluster.id);
const clusterUser = await testCloudly.authManager.CUser.getInstance({
id: persistedCluster.data.userId,
});
const clusterToken = clusterUser.data.tokens?.[0]?.token;
expect(clusterToken).toBeTruthy();
const coreflowClient = new cloudlyApiClient.CloudlyApiClient({
registerAs: 'coreflow',
cloudlyUrl: `http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}`,
});
const configUpdates: any[] = [];
let subscription: { unsubscribe: () => void } | undefined;
try {
await coreflowClient.start();
await coreflowClient.getIdentityByToken(clusterToken!, {
statefullIdentity: true,
tagConnection: true,
});
subscription = coreflowClient.configUpdateSubject.subscribe((updateArg) => {
configUpdates.push(updateArg);
});
const image = await testClient.image.createImage({
name: 'Registry Config Push Test Image',
description: 'Image used by the config push test',
});
const service = await testClient.services.createService({
name: 'Registry Config Push Test Service',
description: 'Service used by the config push test',
imageId: image.id,
imageVersion: 'latest',
environment: {},
secretBundleId: '',
serviceCategory: 'workload',
deploymentStrategy: 'custom',
scaleFactor: 1,
balancingStrategy: 'round-robin',
ports: {
web: 3000,
},
domains: [],
deploymentIds: [],
});
await toolsArg.delayFor(100);
expect(configUpdates[0]?.configData.id).toEqual(cluster.id);
expect(configUpdates[0]?.services.find((serviceArg: any) => serviceArg.id === service.id)).toBeTruthy();
} finally {
subscription?.unsubscribe();
await coreflowClient.stop();
}
});
tap.test('should allow cluster coreflows to read deployment inputs', async () => {
const cluster = await testClient.cluster.createCluster('Registry Coreflow Read Test Cluster');
const persistedCluster = await testCloudly.clusterManager.getConfigBy_ConfigID(cluster.id);
const clusterUser = await testCloudly.authManager.CUser.getInstance({
id: persistedCluster.data.userId,
});
const clusterToken = clusterUser.data.tokens?.[0]?.token;
expect(clusterToken).toBeTruthy();
const image = await testClient.image.createImage({
name: 'Registry Coreflow Read Test Image',
description: 'Image used by the coreflow read test',
});
const secretBundle = await testClient.secretbundle.createSecretBundle({
name: 'Registry Coreflow Read Test Secret Bundle',
description: 'Secret bundle used by the coreflow read test',
type: 'service',
includedSecretGroupIds: [],
includedTags: [],
imageClaims: [],
authorizations: [
{
environment: 'production',
secretAccessKey: 'registry-coreflow-read-test',
},
],
});
const coreflowClient = new cloudlyApiClient.CloudlyApiClient({
registerAs: 'coreflow',
cloudlyUrl: `http://${helpers.testCloudlyConfig.publicUrl}:${helpers.testCloudlyConfig.publicPort}`,
});
try {
await coreflowClient.start();
await coreflowClient.getIdentityByToken(clusterToken!, {
statefullIdentity: true,
tagConnection: true,
});
const clusterImage = await coreflowClient.image.getImageById(image.id);
const clusterSecretBundle = await coreflowClient.secretbundle.getSecretBundleById(secretBundle.id);
expect(clusterImage.id).toEqual(image.id);
expect(clusterSecretBundle.id).toEqual(secretBundle.id);
} finally {
await coreflowClient.stop();
}
});
tap.test('should expose platform desired state', async () => {
const capabilitiesResponse = await testClient.platform.getPlatformCapabilities();
expect(capabilitiesResponse.capabilities.find((capability) => capability.id === 'database')).toBeTruthy();
const desiredState = await testClient.platform.getPlatformDesiredState();
expect(desiredState.capabilities).toBeTruthy();
expect(desiredState.providerConfigs).toBeTruthy();
expect(desiredState.bindings).toBeTruthy();
});
let platformProviderConfigId: string;
let platformBindingId: string;
tap.test('should upsert platform provider config and binding', async () => {
const providerConfigResponse = await testClient.platform.upsertPlatformProviderConfig({
id: '',
capability: 'database',
providerType: 'docker',
name: 'Local Docker Database',
enabled: true,
});
platformProviderConfigId = providerConfigResponse.providerConfig.id;
expect(platformProviderConfigId).toBeTruthy();
const bindingResponse = await testClient.platform.upsertPlatformBinding({
id: '',
serviceId: 'test-service',
capability: 'database',
desiredState: 'enabled',
status: 'requested',
providerConfigId: platformProviderConfigId,
});
platformBindingId = bindingResponse.binding.id;
expect(platformBindingId).toBeTruthy();
const statusResponse = await testClient.platform.updatePlatformBindingStatus({
bindingId: platformBindingId,
status: 'ready',
endpoints: [
{
name: 'primary',
capability: 'database',
protocol: 'mongodb',
internalUrl: 'mongodb://platform-database:27017/test-service',
},
],
});
expect(statusResponse.binding.status).toEqual('ready');
const bindingsResponse = await testClient.platform.getPlatformBindings({
serviceId: 'test-service',
});
expect(bindingsResponse.bindings.find((binding) => binding.id === platformBindingId)).toBeTruthy();
});
let image: any;
tap.test('should create and upload an image', async () => {
console.log('🔵 Test: Creating and uploading image...');
console.log(` - Image name: 'test'`);
@@ -99,11 +435,10 @@ tap.test('should create and upload an image', async () => {
console.log('✅ Image created successfully:');
console.log(` - Image ID: ${image?.id}`);
console.log(` - Image data:`, image);
expect(image).toBeInstanceOf(Image);
expect(image).toBeTruthy();
} catch (error) {
console.error('❌ Failed to create image:');
console.error(` - Error message: ${error.message}`);
console.error(` - Error stack:`, error.stack);
logErrorDetails(error);
throw error;
}
})
@@ -122,17 +457,16 @@ tap.test('should upload an image version', async () => {
console.log('✅ Image version uploaded successfully');
} catch (error) {
console.error('❌ Failed to upload image version:');
console.error(` - Error message: ${error.message}`);
console.error(` - Error stack:`, error.stack);
logErrorDetails(error);
throw error;
}
});
tap.test('should stop the apiclient', async (toolsArg) => {
await toolsArg.delayFor(10000);
await helpers.stopCloudly();
await testClient.stop();
await testCloudly.stop();
await helpers.stopCloudly();
toolsArg.delayFor(1000).then(() => process.exit());
})
export default tap.start();
export default tap.start();
+1 -1
View File
@@ -20,4 +20,4 @@ tap.test('should end the service', async (tools) => {
tools.delayFor(1000).then(() => process.exit());
});
tap.start();
export default tap.start();
+32 -1
View File
@@ -1 +1,32 @@
echo 'hi'
#!/usr/bin/env bash
set -euo pipefail
node --input-type=module <<'NODE'
import fs from 'node:fs';
const readJson = (path) => JSON.parse(fs.readFileSync(path, 'utf8'));
const checks = {
packageVersion: readJson('/app/package.json').version,
interfacesVersion: readJson('/app/node_modules/@serve.zone/interfaces/package.json').version,
apiVersion: readJson('/app/node_modules/@serve.zone/api/package.json').version,
hasDistServe: fs.existsSync('/app/dist_serve/index.html'),
};
await import('/app/dist_ts/index.js');
if (checks.packageVersion !== '5.3.0') {
throw new Error(`Unexpected Cloudly package version ${checks.packageVersion}`);
}
if (checks.interfacesVersion !== '5.4.6') {
throw new Error(`Unexpected interfaces version ${checks.interfacesVersion}`);
}
if (checks.apiVersion !== '5.3.4') {
throw new Error(`Unexpected api version ${checks.apiVersion}`);
}
if (!checks.hasDistServe) {
throw new Error('Missing dist_serve/index.html');
}
console.log(JSON.stringify(checks));
NODE
+1 -1
View File
@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@serve.zone/cloudly',
version: '5.2.0',
version: '5.9.0',
description: 'A comprehensive tool for managing containerized applications across multiple cloud providers using Docker Swarmkit, featuring web, CLI, and API interfaces.'
}
+5 -1
View File
@@ -6,8 +6,12 @@ export const demoImages: plugins.servezoneInterfaces.data.IImage[] = [
data: {
name: 'DemoImage1',
description: 'DemoImage1',
location: {
internal: true,
externalRegistryId: '',
externalImageTag: '',
},
versions: [],
}
}
];
+61 -6
View File
@@ -16,16 +16,26 @@ import { MongodbConnector } from './connector.mongodb/connector.js';
// processes
import { CloudlyCoreflowManager } from './manager.coreflow/coreflowmanager.js';
import { ClusterManager } from './manager.cluster/classes.clustermanager.js';
import { CloudlyTaskmanager } from './manager.task/taskmanager.js';
import { CloudlyTaskManager } from './manager.task/classes.taskmanager.js';
import { CloudlySecretManager } from './manager.secret/classes.secretmanager.js';
import { CloudlyNodeManager } from './manager.node/classes.nodemanager.js';
import { CloudlyBaremetalManager } from './manager.baremetal/classes.baremetalmanager.js';
import { ExternalApiManager } from './manager.status/statusmanager.js';
import { ExternalRegistryManager } from './manager.externalregistry/index.js';
import { CloudlyRegistryManager } from './manager.registry/index.js';
import { ImageManager } from './manager.image/classes.imagemanager.js';
import { ServiceManager } from './manager.service/classes.servicemanager.js';
import { DeploymentManager } from './manager.deployment/classes.deploymentmanager.js';
import { DnsManager } from './manager.dns/classes.dnsmanager.js';
import { DomainManager } from './manager.domain/classes.domainmanager.js';
import { logger } from './logger.js';
import { CloudlyAuthManager } from './manager.auth/classes.authmanager.js';
import { CloudlySettingsManager } from './manager.settings/classes.settingsmanager.js';
import { CloudlyPlatformManager } from './manager.platform/classes.platformmanager.js';
import { CloudlyBackupManager } from './manager.backup/classes.backupmanager.js';
import { CloudlyBaseOsManager } from './manager.baseos/classes.baseosmanager.js';
import { CloudlyAppCatalogManager } from './manager.appcatalog/classes.appcatalogmanager.js';
import { CloudlyJumpManager } from './manager.jump/classes.jumpmanager.js';
/**
* Cloudly class can be used to instantiate a cloudly server.
@@ -55,18 +65,28 @@ export class Cloudly {
public authManager: CloudlyAuthManager;
public secretManager: CloudlySecretManager;
public settingsManager: CloudlySettingsManager;
public platformManager: CloudlyPlatformManager;
public clusterManager: ClusterManager;
public coreflowManager: CloudlyCoreflowManager;
public externalApiManager: ExternalApiManager;
public externalRegistryManager: ExternalRegistryManager;
public registryManager: CloudlyRegistryManager;
public imageManager: ImageManager;
public taskManager: CloudlyTaskmanager;
public serviceManager: ServiceManager;
public deploymentManager: DeploymentManager;
public dnsManager: DnsManager;
public domainManager: DomainManager;
public taskManager: CloudlyTaskManager;
public backupManager: CloudlyBackupManager;
public nodeManager: CloudlyNodeManager;
public baremetalManager: CloudlyBaremetalManager;
public baseOsManager: CloudlyBaseOsManager;
public appCatalogManager: CloudlyAppCatalogManager;
public jumpManager: CloudlyJumpManager;
private readyDeferred = new plugins.smartpromise.Deferred();
private configOptions: plugins.servezoneInterfaces.data.ICloudlyConfig;
private configOptions?: plugins.servezoneInterfaces.data.ICloudlyConfig;
constructor(configArg?: plugins.servezoneInterfaces.data.ICloudlyConfig) {
this.configOptions = configArg;
this.cloudlyInfo = new CloudlyInfo(this);
@@ -84,15 +104,25 @@ export class Cloudly {
// managers
this.authManager = new CloudlyAuthManager(this);
this.settingsManager = new CloudlySettingsManager(this);
this.platformManager = new CloudlyPlatformManager(this);
this.clusterManager = new ClusterManager(this);
this.coreflowManager = new CloudlyCoreflowManager(this);
this.externalApiManager = new ExternalApiManager(this);
this.externalRegistryManager = new ExternalRegistryManager(this);
this.registryManager = new CloudlyRegistryManager(this);
this.imageManager = new ImageManager(this);
this.taskManager = new CloudlyTaskmanager(this);
this.serviceManager = new ServiceManager(this);
this.deploymentManager = new DeploymentManager(this);
this.dnsManager = new DnsManager(this);
this.domainManager = new DomainManager(this);
this.taskManager = new CloudlyTaskManager(this);
this.backupManager = new CloudlyBackupManager(this);
this.baseOsManager = new CloudlyBaseOsManager(this);
this.secretManager = new CloudlySecretManager(this);
this.appCatalogManager = new CloudlyAppCatalogManager(this);
this.nodeManager = new CloudlyNodeManager(this);
this.baremetalManager = new CloudlyBaremetalManager(this);
this.jumpManager = new CloudlyJumpManager(this);
}
/**
@@ -114,15 +144,28 @@ export class Cloudly {
await this.secretManager.start();
await this.nodeManager.start();
await this.baremetalManager.start();
await this.jumpManager.start();
await this.serviceManager.start();
await this.platformManager.start();
await this.deploymentManager.start();
await this.taskManager.init();
await this.backupManager.start();
await this.baseOsManager.start();
await this.appCatalogManager.start();
await this.registryManager.start();
await this.domainManager.init();
await this.cloudflareConnector.init();
await this.letsencryptConnector.init();
if (this.config.data.sslMode === 'letsencrypt') {
await this.letsencryptConnector.init();
}
await this.clusterManager.init();
await this.server.start();
this.readyDeferred.resolve();
// start the managers
this.imageManager.start();
this.externalRegistryManager.start();
}
/**
@@ -130,8 +173,20 @@ export class Cloudly {
*/
public async stop() {
await this.server.stop();
await this.letsencryptConnector.stop();
if (this.config.data.sslMode === 'letsencrypt') {
await this.letsencryptConnector.stop();
}
await this.mongodbConnector.stop();
await this.secretManager.stop();
await this.serviceManager.stop();
await this.platformManager.stop();
await this.deploymentManager.stop();
await this.jumpManager.stop();
await this.taskManager.stop();
await this.backupManager.stop();
await this.baseOsManager.stop();
await this.registryManager.stop();
await this.appCatalogManager.stop();
await this.externalRegistryManager.stop();
}
}
+1 -1
View File
@@ -9,5 +9,5 @@ export class CloudlyInfo {
this.cloudlyRef = cloudlyRefArg;
}
public projectInfo = new plugins.projectinfo.ProjectInfo(paths.packageDir);
public projectInfo = plugins.projectinfo.ProjectInfo.create(paths.packageDir);
}
+5 -4
View File
@@ -8,8 +8,8 @@ import type { Cloudly } from './classes.cloudly.js';
*/
export class CloudlyConfig {
public cloudlyRef: Cloudly;
public appData: plugins.npmextra.AppData<plugins.servezoneInterfaces.data.ICloudlyConfig>;
public data: plugins.servezoneInterfaces.data.ICloudlyConfig;
public appData!: plugins.smartconfig.AppData<plugins.servezoneInterfaces.data.ICloudlyConfig>;
public data!: plugins.servezoneInterfaces.data.ICloudlyConfig;
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
@@ -17,12 +17,12 @@ export class CloudlyConfig {
public async init(configArg?: plugins.servezoneInterfaces.data.ICloudlyConfig) {
this.appData =
await plugins.npmextra.AppData.createAndInit<plugins.servezoneInterfaces.data.ICloudlyConfig>(
await plugins.smartconfig.AppData.createAndInit<plugins.servezoneInterfaces.data.ICloudlyConfig>(
{
envMapping: {
environment: 'SERVEZONE_ENVIRONMENT' as 'production' | 'integration',
letsEncryptEmail: 'hard:domains@lossless.org',
letsEncryptPrivateKey: null,
letsEncryptPrivateKey: undefined,
publicUrl: 'SERVEZONE_URL',
publicPort: 'SERVEZONE_PORT',
mongoDescriptor: {
@@ -50,6 +50,7 @@ export class CloudlyConfig {
'sslMode',
'environment',
'mongoDescriptor',
's3Descriptor',
],
overwriteObject: configArg,
},
+60 -27
View File
@@ -11,13 +11,13 @@ export class CloudlyServer {
* a reference to the cloudly instance
*/
public cloudlyRef: Cloudly;
public additionalHandlers: plugins.typedserver.servertools.Handler[] = [];
public additionalHandlers: plugins.typedserver.IRouteHandler[] = [];
/**
* the smartexpress server handling the actual requests
*/
public typedServer: plugins.typedserver.TypedServer;
public typedsocketServer: plugins.typedsocket.TypedSocket;
public typedServer!: plugins.typedserver.TypedServer;
public typedsocketServer!: plugins.typedsocket.TypedSocket;
/**
* typedrouter
@@ -39,13 +39,13 @@ export class CloudlyServer {
*/
public async start() {
logger.log('info', `cloudly domain is ${this.cloudlyRef.config.data.publicUrl}`);
let sslCert: plugins.smartacme.Cert;
let sslCert: plugins.smartacme.Cert | undefined;
if (this.cloudlyRef.config.data.sslMode === 'letsencrypt') {
logger.log('info', `Using letsencrypt for ssl mode. Trying to obtain a certificate...`);
logger.log('info', `This might take 10 minutes...`);
sslCert = await this.cloudlyRef.letsencryptConnector.getCertificateForDomain(
this.cloudlyRef.config.data.publicUrl,
this.cloudlyRef.config.data.publicUrl!,
);
logger.log(
'success',
@@ -58,23 +58,6 @@ export class CloudlyServer {
);
}
interface IRequestGuardData {
req: plugins.typedserver.Request;
res: plugins.typedserver.Response;
}
// guards
const guardIp = new plugins.smartguard.Guard<IRequestGuardData>(async (dataArg) => {
if (true) {
return true;
} else {
dataArg.res.status(500);
dataArg.res.send(`Not allowed to perform this operation!`);
dataArg.res.end();
return false;
}
});
// server
this.typedServer = new plugins.typedserver.TypedServer({
cors: true,
@@ -88,14 +71,64 @@ export class CloudlyServer {
: {}),
injectReload: true,
serveDir: paths.distServeDir,
spaFallback: true,
watch: true,
enableCompression: true,
preferredCompressionMethod: 'gzip',
compression: {
enabled: true,
algorithms: ['gzip'],
},
});
this.typedsocketServer = this.typedServer.typedsocket;
this.typedServer.typedrouter.addTypedRouter(this.typedrouter);
this.typedServer.server.addRoute(
this.typedServer.addRoute(
'/v2',
'ALL',
async (ctx) => this.cloudlyRef.registryManager.handleHttpRequest(ctx),
);
this.typedServer.addRoute(
'/v2/*',
'ALL',
async (ctx) => this.cloudlyRef.registryManager.handleHttpRequest(ctx),
);
this.typedServer.addRoute(
'/curlfresh/:scriptname',
this.cloudlyRef.nodeManager.curlfreshInstance.handler,
'ALL',
async (ctx) => this.cloudlyRef.nodeManager.curlfreshInstance.handleRequest(ctx),
);
this.typedServer.addRoute(
'/jump/v1/claim',
'POST',
async (ctx) => this.cloudlyRef.jumpManager.handleClaimHttpRequest(ctx),
);
this.typedServer.addRoute(
'/jump/:code/setup.sh',
'GET',
async (ctx) => this.cloudlyRef.jumpManager.handleSetupScriptHttpRequest(ctx),
);
this.typedServer.addRoute(
'/jump/:code',
'GET',
async (ctx) => this.cloudlyRef.jumpManager.handleJumpHttpRequest(ctx),
);
this.typedServer.addRoute(
'/baseos/v1/nodes/register',
'POST',
async (ctx) => this.cloudlyRef.baseOsManager.handleRegisterHttpRequest(ctx),
);
this.typedServer.addRoute(
'/baseos/v1/nodes/heartbeat',
'POST',
async (ctx) => this.cloudlyRef.baseOsManager.handleHeartbeatHttpRequest(ctx),
);
this.typedServer.addRoute(
'/spark/v1/nodes/heartbeat',
'POST',
async (ctx) => this.cloudlyRef.nodeManager.handleSparkHeartbeatHttpRequest(ctx),
);
this.typedServer.addRoute(
'/baseos/v1/images/:buildId/download',
'GET',
async (ctx) => this.cloudlyRef.baseOsManager.handleImageDownloadHttpRequest(ctx),
);
await this.typedServer.start();
}
@@ -104,6 +137,6 @@ export class CloudlyServer {
* stop the reception instance
*/
public async stop() {
await this.typedServer.stop();
await this.typedServer?.stop();
}
}
+1 -1
View File
@@ -6,7 +6,7 @@ import { Cloudly } from '../classes.cloudly.js';
*/
export class CloudflareConnector {
private cloudlyRef: Cloudly;
public cloudflare: plugins.cloudflare.CloudflareAccount;
public cloudflare?: plugins.cloudflare.CloudflareAccount;
constructor(cloudlyArg: Cloudly) {
this.cloudlyRef = cloudlyArg;
+9 -5
View File
@@ -3,7 +3,7 @@ import { Cloudly } from '../classes.cloudly.js';
export class LetsencryptConnector {
private cloudlyRef: Cloudly;
private smartacme: plugins.smartacme.SmartAcme;
private smartacme!: plugins.smartacme.SmartAcme;
constructor(cloudlyArg: Cloudly) {
this.cloudlyRef = cloudlyArg;
@@ -18,6 +18,10 @@ export class LetsencryptConnector {
* inits letsencrypt
*/
public async init() {
if (!this.cloudlyRef.cloudflareConnector.cloudflare) {
throw new Error('Cloudflare token is required for letsencrypt DNS-01 challenges');
}
// Create DNS-01 challenge handler using Cloudflare
const dnsHandler = new plugins.smartacme.handlers.Dns01Handler(
this.cloudlyRef.cloudflareConnector.cloudflare
@@ -25,13 +29,13 @@ export class LetsencryptConnector {
// Create MongoDB certificate manager
const certManager = new plugins.smartacme.certmanagers.MongoCertManager(
this.cloudlyRef.config.data.mongoDescriptor
this.cloudlyRef.config.data.mongoDescriptor!
);
this.smartacme = new plugins.smartacme.SmartAcme({
accountEmail: this.cloudlyRef.config.data.letsEncryptEmail,
accountEmail: this.cloudlyRef.config.data.letsEncryptEmail!,
accountPrivateKey: this.cloudlyRef.config.data.letsEncryptPrivateKey,
environment: this.cloudlyRef.config.data.environment,
environment: this.cloudlyRef.config.data.environment!,
certManager: certManager,
challengeHandlers: [dnsHandler],
});
@@ -45,6 +49,6 @@ export class LetsencryptConnector {
* stops the instance
*/
public async stop() {
await this.smartacme.stop();
await this.smartacme?.stop();
}
}
+2 -2
View File
@@ -4,7 +4,7 @@ import { Cloudly } from '../classes.cloudly.js';
export class MongodbConnector {
// INSTANCE
private cloudlyRef: Cloudly;
public smartdataDb: plugins.smartdata.SmartdataDb;
public smartdataDb!: plugins.smartdata.SmartdataDb;
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
@@ -12,7 +12,7 @@ export class MongodbConnector {
public async init() {
this.smartdataDb = new plugins.smartdata.SmartdataDb(
this.cloudlyRef.config.data.mongoDescriptor,
this.cloudlyRef.config.data.mongoDescriptor!,
);
await this.smartdataDb.init();
}
+6 -3
View File
@@ -11,7 +11,7 @@ early.stop();
* starts the cloudly instance
*/
const runCli = async () => {
logger.log('info', process.env.SERVEZONE_ENVIRONMENT);
logger.log('info', process.env.SERVEZONE_ENVIRONMENT || '');
const cloudlyInstance = new Cloudly();
logger.log(
@@ -20,8 +20,11 @@ const runCli = async () => {
);
await cloudlyInstance.start();
const demoMod = await import('./00demo/index.js');
demoMod.installDemoData(cloudlyInstance);
if (process.env.SERVEZONE_INSTALL_DEMO_DATA === 'true') {
logger.log('warn', 'SERVEZONE_INSTALL_DEMO_DATA=true: installing destructive demo data');
const demoMod = await import('./00demo/index.js');
await demoMod.installDemoData(cloudlyInstance);
}
};
export { runCli, Cloudly };
+6 -6
View File
@@ -3,12 +3,12 @@ import * as paths from './paths.js';
export const logger = new plugins.smartlog.Smartlog({
logContext: {
company: null,
environment: null,
runtime: null,
zone: null,
companyunit: null,
containerName: null,
company: undefined,
environment: undefined,
runtime: undefined,
zone: undefined,
companyunit: undefined,
containerName: undefined,
},
});
logger.enableConsole({
@@ -0,0 +1,306 @@
import type { Cloudly } from '../classes.cloudly.js';
import * as plugins from '../plugins.js';
import { Image } from '../manager.image/classes.image.js';
import { Service } from '../manager.service/classes.service.js';
import { SecretBundle } from '../manager.secret/classes.secretbundle.js';
import { PlatformBinding } from '../manager.platform/classes.platformbinding.js';
type ICatalogApp = plugins.servezoneInterfaces.appcatalog.ICatalogApp;
type ICatalog = plugins.servezoneInterfaces.appcatalog.ICatalog;
type IAppMeta = plugins.servezoneInterfaces.appcatalog.IAppMeta;
type IAppVersionConfig = plugins.servezoneInterfaces.appcatalog.IAppVersionConfig;
type IInstallOptions = plugins.servezoneInterfaces.appcatalog.IAppInstallRequest;
type IUpgradeableCatalogService = plugins.servezoneInterfaces.appcatalog.IUpgradeableAppService;
export class CloudlyAppCatalogManager {
public typedrouter = new plugins.typedrequest.TypedRouter();
private catalogCache: ICatalog | null = null;
private lastFetchTime = 0;
private readonly repoBaseUrl = process.env.APPCATALOG_URL || 'https://code.foss.global/serve.zone/appstore-apptemplates/raw/branch/main';
private readonly cacheTtlMs = 5 * 60 * 1000;
constructor(private cloudlyRef: Cloudly) {
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
this.registerHandlers();
}
public async start() {}
public async stop() {}
private registerHandlers() {
const addCatalogListHandler = (methodArg: string) => {
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>(methodArg, async (dataArg) => {
await this.passAdminIdentity(dataArg);
return { apps: await this.getApps() };
}),
);
};
addCatalogListHandler('getAppCatalogTemplates');
addCatalogListHandler('getAppTemplates');
const addConfigHandler = (methodArg: string) => {
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>(methodArg, async (dataArg) => {
await this.passAdminIdentity(dataArg);
return {
config: await this.getAppVersionConfig(dataArg.appId, dataArg.version),
appMeta: await this.getAppMeta(dataArg.appId),
};
}),
);
};
addConfigHandler('getAppCatalogConfig');
addConfigHandler('getAppConfig');
const addInstallHandler = (methodArg: string) => {
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>(methodArg, async (dataArg) => {
await this.passAdminIdentity(dataArg);
const service = await this.installApp(dataArg.install || dataArg);
return { service: await service.createSavableObject() };
}),
);
};
addInstallHandler('installAppCatalogApp');
addInstallHandler('installAppTemplate');
const addUpgradeableHandler = (methodArg: string) => {
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>(methodArg, async (dataArg) => {
await this.passAdminIdentity(dataArg);
return { services: await this.getUpgradeableServices() };
}),
);
};
addUpgradeableHandler('getUpgradeableAppCatalogServices');
addUpgradeableHandler('getUpgradeableServices');
}
public async getCatalog(): Promise<ICatalog> {
const now = Date.now();
if (this.catalogCache && now - this.lastFetchTime < this.cacheTtlMs) {
return this.catalogCache;
}
const catalog = await this.fetchJson('catalog.json') as ICatalog;
if (!catalog || !Array.isArray(catalog.apps)) {
throw new Error('Invalid app catalog format');
}
this.catalogCache = catalog;
this.lastFetchTime = now;
return catalog;
}
public async getApps(): Promise<ICatalogApp[]> {
return (await this.getCatalog()).apps;
}
public async getAppMeta(appIdArg: string): Promise<IAppMeta> {
return await this.fetchJson(`apps/${appIdArg}/app.json`) as IAppMeta;
}
public async getAppVersionConfig(appIdArg: string, versionArg?: string): Promise<IAppVersionConfig> {
if (!versionArg) {
versionArg = (await this.getAppMeta(appIdArg)).latestVersion;
}
return await this.fetchJson(`apps/${appIdArg}/versions/${versionArg}/config.json`) as IAppVersionConfig;
}
public async getUpgradeableServices(): Promise<IUpgradeableCatalogService[]> {
const catalog = await this.getCatalog();
const services = await this.cloudlyRef.serviceManager.CService.getInstances({});
const upgradeableServices: IUpgradeableCatalogService[] = [];
for (const service of services) {
const serviceData = service.data as plugins.servezoneInterfaces.data.IService['data'] & {
appTemplateId?: string;
appTemplateVersion?: string;
};
if (!serviceData.appTemplateId || !serviceData.appTemplateVersion) {
continue;
}
const catalogApp = catalog.apps.find((appArg) => appArg.id === serviceData.appTemplateId);
if (!catalogApp || catalogApp.latestVersion === serviceData.appTemplateVersion) {
continue;
}
upgradeableServices.push({
serviceName: serviceData.name,
appTemplateId: serviceData.appTemplateId,
currentVersion: serviceData.appTemplateVersion,
latestVersion: catalogApp.latestVersion,
hasMigration: false,
});
}
return upgradeableServices;
}
public async installApp(optionsArg: IInstallOptions): Promise<Service> {
const appMeta = await this.getAppMeta(optionsArg.appId);
const version = optionsArg.version || appMeta.latestVersion;
const config = await this.getAppVersionConfig(optionsArg.appId, version);
const webPort = optionsArg.port || config.port;
this.assertSupportedPlatformRequirements(config);
const envVars = this.getCatalogEnvVars(config, optionsArg.envVars || {});
if (this.requiresTemplateValue(envVars, 'SERVICE_DOMAIN') && !optionsArg.domain) {
throw new Error('A domain is required because the app template uses ${SERVICE_DOMAIN}');
}
const image = await this.createCatalogImage(optionsArg.serviceName, config.image, appMeta.description);
const secretBundle = await this.createServiceSecretBundle(optionsArg.serviceName, image.id);
const serviceData = {
name: optionsArg.serviceName,
description: appMeta.description,
imageId: image.id,
imageVersion: this.getImageTag(config.image),
deployOnPush: false,
appTemplateId: optionsArg.appId,
appTemplateVersion: version,
environment: envVars,
secretBundleId: secretBundle.id,
additionalSecretBundleIds: [],
serviceCategory: 'workload',
deploymentStrategy: 'limited-replicas',
maxReplicas: 1,
antiAffinity: false,
scaleFactor: 1,
balancingStrategy: 'round-robin',
ports: { web: webPort },
volumes: this.normalizeVolumes(config.volumes),
domains: optionsArg.domain ? [{ name: optionsArg.domain, port: webPort, protocol: 'https' }] : [],
deploymentIds: [],
} as plugins.servezoneInterfaces.data.IService['data'] & {
appTemplateId: string;
appTemplateVersion: string;
};
const service = await Service.createService(serviceData);
secretBundle.data.serviceId = service.id;
await secretBundle.save();
await this.createPlatformBindings(service, config);
await this.cloudlyRef.coreflowManager.pushClusterConfigToConnectedCoreflows();
return service;
}
private async createCatalogImage(serviceNameArg: string, imageRefArg: string, descriptionArg: string): Promise<Image> {
const image = new Image();
image.id = await Image.getNewId();
image.data = {
name: `${serviceNameArg}-catalog-image`,
description: descriptionArg,
location: {
internal: false,
externalRegistryId: '',
externalImageTag: imageRefArg,
externalImageRef: imageRefArg,
},
versions: [{
versionString: this.getImageTag(imageRefArg),
source: 'registry',
registryRepository: imageRefArg,
registryTag: this.getImageTag(imageRefArg),
size: 0,
createdAt: Date.now(),
}],
};
await image.save();
return image;
}
private async createServiceSecretBundle(serviceNameArg: string, imageIdArg: string): Promise<SecretBundle> {
const secretBundle = new SecretBundle();
secretBundle.id = plugins.smartunique.shortId(8);
secretBundle.data = {
name: `${serviceNameArg} catalog secrets`,
description: `Generated catalog secret bundle for ${serviceNameArg}`,
type: 'service',
includedSecretGroupIds: [],
includedTags: [],
imageClaims: [{ imageId: imageIdArg, permissions: ['read'] }],
authorizations: [],
};
await secretBundle.save();
return secretBundle;
}
private async createPlatformBindings(serviceArg: Service, configArg: IAppVersionConfig) {
const requirements = configArg.platformRequirements || {};
if (requirements.mongodb) {
await PlatformBinding.upsertBinding({
id: await PlatformBinding.getNewId(),
serviceId: serviceArg.id,
capability: 'database',
desiredState: 'enabled',
status: 'requested',
});
}
if (requirements.s3) {
await PlatformBinding.upsertBinding({
id: await PlatformBinding.getNewId(),
serviceId: serviceArg.id,
capability: 'objectstorage',
desiredState: 'enabled',
status: 'requested',
});
}
}
private normalizeVolumes(volumesArg: IAppVersionConfig['volumes'] = []) {
return volumesArg.map((volumeArg) => {
if (typeof volumeArg === 'string') {
return { mountPath: volumeArg };
}
return volumeArg;
}).filter((volumeArg) => Boolean(volumeArg.mountPath));
}
private getCatalogEnvVars(configArg: IAppVersionConfig, overridesArg: Record<string, string>): Record<string, string> {
const envVars: Record<string, string> = {};
const missingRequiredEnvVars: string[] = [];
for (const envVar of configArg.envVars || []) {
const value = overridesArg[envVar.key] ?? envVar.value ?? '';
if (envVar.required && !value) {
missingRequiredEnvVars.push(envVar.key);
}
envVars[envVar.key] = value;
}
Object.assign(envVars, overridesArg);
if (missingRequiredEnvVars.length > 0) {
throw new Error(`Missing required app env var(s): ${missingRequiredEnvVars.join(', ')}`);
}
return envVars;
}
private requiresTemplateValue(envVarsArg: Record<string, string>, templateNameArg: string): boolean {
return Object.values(envVarsArg).some((value) => value.includes(`\${${templateNameArg}}`));
}
private assertSupportedPlatformRequirements(configArg: IAppVersionConfig) {
const unsupported = Object.entries(configArg.platformRequirements || {})
.filter(([key, enabled]) => enabled && key !== 'mongodb' && key !== 's3')
.map(([key]) => key);
if (unsupported.length > 0) {
throw new Error(`Cloudly catalog install does not yet support platform requirement(s): ${unsupported.join(', ')}`);
}
}
private getImageTag(imageRefArg: string) {
const lastSlashIndex = imageRefArg.lastIndexOf('/');
const lastColonIndex = imageRefArg.lastIndexOf(':');
return lastColonIndex > lastSlashIndex ? imageRefArg.slice(lastColonIndex + 1) || 'latest' : 'latest';
}
private async passAdminIdentity(dataArg: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
}
private async fetchJson(pathArg: string): Promise<unknown> {
const url = `${this.repoBaseUrl.replace(/\/+$/, '')}/${pathArg}`;
const response = await fetch(url);
if (!response.ok) {
throw new Error(`HTTP ${response.status} for ${url}`);
}
return response.json();
}
}
+102 -29
View File
@@ -11,6 +11,17 @@ export interface IJwtData {
expiresAt: number;
}
interface IReq_AdminValidateIdentity {
method: 'adminValidateIdentity';
request: {
identity: plugins.servezoneInterfaces.data.IIdentity;
};
response: {
valid: boolean;
reason?: string;
};
}
export class CloudlyAuthManager {
cloudlyRef: Cloudly;
public get db() {
@@ -20,7 +31,7 @@ export class CloudlyAuthManager {
public CAuthorization = plugins.smartdata.setDefaultManagerForDoc(this, Authorization);
public typedrouter = new plugins.typedrequest.TypedRouter();
public smartjwtInstance: plugins.smartjwt.SmartJwt<IJwtData>;
public smartjwtInstance!: plugins.smartjwt.SmartJwt<IJwtData>;
constructor(cloudlyRef: Cloudly) {
this.cloudlyRef = cloudlyRef;
@@ -49,8 +60,10 @@ export class CloudlyAuthManager {
this.smartjwtInstance.setKeyPairAsJson(existingJwtKeys);
}
await this.bootstrapInitialAdmin();
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.secret.IReq_Admin_LoginWithUsernameAndPassword>(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.admin.IReq_Admin_LoginWithUsernameAndPassword>(
'adminLoginWithUsernameAndPassword',
async (dataArg) => {
let jwt: string;
@@ -71,7 +84,7 @@ export class CloudlyAuthManager {
identity: {
jwt,
userId: user.id,
name: user.data.username,
name: user.data.username || user.id,
expiresAt: expiresAtTimestamp,
role: user.data.role,
type: user.data.type,
@@ -80,6 +93,52 @@ export class CloudlyAuthManager {
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<IReq_AdminValidateIdentity>('adminValidateIdentity', async (dataArg) => {
const valid = await this.adminIdentityGuard.exec(dataArg).catch(() => false);
return {
valid,
reason: valid ? undefined : 'identity is not valid',
};
}),
);
}
private async bootstrapInitialAdmin() {
const users = await this.CUser.getInstances({});
const hasHumanUser = users.some((userArg) => userArg.data?.type === 'human');
if (hasHumanUser) {
return;
}
const adminAccount = this.cloudlyRef.config.data.servezoneAdminaccount;
if (!adminAccount) {
throw new Error('SERVEZONE_ADMINACCOUNT is required for first-run Cloudly bootstrap');
}
const separatorIndex = adminAccount.indexOf(':');
if (separatorIndex <= 0 || separatorIndex === adminAccount.length - 1) {
throw new Error('SERVEZONE_ADMINACCOUNT must use username:password format');
}
const username = adminAccount.slice(0, separatorIndex).trim();
const password = adminAccount.slice(separatorIndex + 1);
if (!username || !password) {
throw new Error('SERVEZONE_ADMINACCOUNT must include a non-empty username and password');
}
const user = new this.CUser({
id: await this.CUser.getNewId(),
data: {
type: 'human',
username,
password,
role: 'admin',
},
});
await user.save();
logger.log('success', `created initial admin user ${username}`);
}
public async stop() {}
@@ -88,28 +147,22 @@ export class CloudlyAuthManager {
identity: plugins.servezoneInterfaces.data.IIdentity;
}>(
async (dataArg) => {
const jwt = dataArg.identity.jwt;
const jwtData: IJwtData = await this.smartjwtInstance.verifyJWTAndGetData(jwt);
const expired = jwtData.expiresAt < Date.now();
plugins.smartexpect
.expect(jwtData.status)
.setFailMessage('user not logged in')
.toEqual('loggedIn');
plugins.smartexpect.expect(expired).setFailMessage(`jwt expired`).toBeFalse();
plugins.smartexpect
.expect(dataArg.identity.expiresAt)
.setFailMessage(
`expiresAt >>identity valid until:${dataArg.identity.expiresAt}, but jwt says: ${jwtData.expiresAt}<< has been tampered with`,
)
.toEqual(jwtData.expiresAt);
plugins.smartexpect
.expect(dataArg.identity.userId)
.setFailMessage('userId has been tampered with')
.toEqual(jwtData.userId);
if (expired) {
throw new Error('identity is expired');
try {
const jwt = dataArg.identity?.jwt;
if (!jwt) {
return false;
}
const jwtData: IJwtData = await this.smartjwtInstance.verifyJWTAndGetData(jwt);
const expired = jwtData.expiresAt < Date.now();
return (
jwtData.status === 'loggedIn' &&
!expired &&
dataArg.identity.expiresAt === jwtData.expiresAt &&
dataArg.identity.userId === jwtData.userId
);
} catch {
return false;
}
return true;
},
{
failedHint: 'identity is not valid.',
@@ -121,17 +174,37 @@ export class CloudlyAuthManager {
identity: plugins.servezoneInterfaces.data.IIdentity;
}>(
async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [this.validIdentityGuard]);
const validIdentity = await this.validIdentityGuard.exec(dataArg);
if (!validIdentity) {
return false;
}
const jwt = dataArg.identity.jwt;
const jwtData: IJwtData = await this.smartjwtInstance.verifyJWTAndGetData(jwt);
const user = await this.CUser.getInstance({ id: jwtData.userId });
const isAdminBool = user.data.role === 'admin';
console.log(`user is admin: ${isAdminBool}`);
return isAdminBool;
return user?.data.role === 'admin';
},
{
failedHint: 'user is not admin.',
failedHint: 'identity is not valid or user is not admin.',
name: 'adminIdentityGuard',
},
);
public adminOrClusterIdentityGuard = new plugins.smartguard.Guard<{
identity: plugins.servezoneInterfaces.data.IIdentity;
}>(
async (dataArg) => {
const validIdentity = await this.validIdentityGuard.exec(dataArg);
if (!validIdentity) {
return false;
}
const jwt = dataArg.identity.jwt;
const jwtData: IJwtData = await this.smartjwtInstance.verifyJWTAndGetData(jwt);
const user = await this.CUser.getInstance({ id: jwtData.userId });
return user?.data.role === 'admin' || user?.data.role === 'cluster';
},
{
failedHint: 'identity is not valid or user is not admin or cluster.',
name: 'adminOrClusterIdentityGuard',
},
);
}
+2 -2
View File
@@ -45,8 +45,8 @@ export class User extends plugins.smartdata.SmartDataDbDoc<
// INSTANCE
@plugins.smartdata.unI()
public id: string;
public id!: string;
@plugins.smartdata.svDb()
public data: plugins.servezoneInterfaces.data.IUser['data'];
public data!: plugins.servezoneInterfaces.data.IUser['data'];
}
+593
View File
@@ -0,0 +1,593 @@
import type { Cloudly } from '../classes.cloudly.js';
import * as plugins from '../plugins.js';
import { BackupRecord } from './classes.backuprecord.js';
import { createBackupTargetWriterFromEnv, type IBackupTargetWriter } from './classes.replicationtarget.js';
export type TBackupStatus =
| 'pending'
| 'running'
| 'replicating'
| 'replicated'
| 'ready'
| 'failed'
| 'restoring'
| 'restored';
export type TBackupResourceType = 'volume' | 'database' | 'objectstorage';
export type TBackupReplicationTargetType = 's3' | 'smb';
export interface IBackupArchiveObject {
path: string;
size: number;
sha256: string;
}
export interface IBackupArchiveManifest {
version: 1;
backupId: string;
createdAt: number;
objects: IBackupArchiveObject[];
totalSize: number;
}
export interface IBackupReplicationResult {
targetType: TBackupReplicationTargetType;
targetPath: string;
manifestPath: string;
manifestSha256: string;
objectCount: number;
totalSize: number;
completedAt: number;
}
export interface IBackupSnapshotData {
type: TBackupResourceType;
snapshotId: string;
snapshotName?: string;
originalSize: number;
storedSize: number;
createdAt: number;
tags?: Record<string, string>;
volumeName?: string;
mountPath?: string;
resourceName?: string;
databaseName?: string;
bucketName?: string;
}
export interface IBackupRecordData {
id: string;
serviceId: string;
serviceName?: string;
clusterId?: string;
status: TBackupStatus;
trigger: 'manual' | 'scheduled';
snapshots: IBackupSnapshotData[];
replication?: IBackupReplicationResult;
createdAt: number;
updatedAt: number;
completedAt?: number;
requestedBy?: string;
errorText?: string;
restoreHistory?: Array<{
restoredAt: number;
status: 'restored' | 'failed';
errorText?: string;
}>;
tags?: Record<string, string>;
}
export class CloudlyBackupManager {
public typedrouter = new plugins.typedrequest.TypedRouter();
public cloudlyRef: Cloudly;
private backupTargetWriter?: IBackupTargetWriter;
get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
}
public CBackupRecord = plugins.smartdata.setDefaultManagerForDoc(this, BackupRecord);
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('createServiceBackup', async (requestArg) => {
await this.passAdminIdentity(requestArg);
return {
backup: await this.createServiceBackup(requestArg),
};
}),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('getServiceBackups', async (requestArg) => {
await this.passValidIdentity(requestArg);
return {
backups: await this.getBackups({
...(requestArg.serviceId ? { serviceId: requestArg.serviceId } : {}),
...(requestArg.status ? { status: requestArg.status } : {}),
}),
};
}),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('getBackupById', async (requestArg) => {
await this.passValidIdentity(requestArg);
return {
backup: await this.getBackupById(requestArg.backupId),
};
}),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('restoreServiceBackup', async (requestArg) => {
await this.passAdminIdentity(requestArg);
return {
backup: await this.restoreServiceBackup(requestArg),
};
}),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('prepareBackupReplication', async (requestArg) => {
await this.passClusterIdentity(requestArg);
return await this.prepareBackupReplication(requestArg);
}),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('uploadBackupArchiveObject', async (requestArg) => {
await this.passClusterIdentity(requestArg);
return await this.uploadBackupArchiveObject(requestArg);
}),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('completeBackupReplication', async (requestArg) => {
await this.passClusterIdentity(requestArg);
return await this.completeBackupReplication(requestArg);
}),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('getBackupArchiveManifest', async (requestArg) => {
await this.passClusterIdentity(requestArg);
return await this.getBackupArchiveManifest(requestArg);
}),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>('downloadBackupArchiveObject', async (requestArg) => {
await this.passClusterIdentity(requestArg);
return await this.downloadBackupArchiveObject(requestArg);
}),
);
}
public async start() {
const schedule = process.env.CLOUDLY_BACKUP_CRON;
this.cloudlyRef.taskManager.registerTask(
'backup-all-services',
new plugins.taskbuffer.Task({
name: 'backup-all-services',
taskFunction: async () => await this.backupAllServices(),
}),
{
description: 'Create backups for every workload service with backup-enabled resources.',
category: 'backup',
schedule,
enabled: Boolean(schedule),
},
);
}
public async stop() {}
public async getBackups(queryArg: Record<string, unknown> = {}) {
const backups = await this.CBackupRecord.getInstances(queryArg);
return await Promise.all(backups.map((backupArg) => backupArg.createSavableObject()));
}
public async getBackupById(backupIdArg: string) {
const backup = await BackupRecord.getInstance({ id: backupIdArg });
if (!backup) {
throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} not found`);
}
return await backup.createSavableObject();
}
public async backupAllServices() {
const services = await this.cloudlyRef.serviceManager.CService.getInstances({});
const results: Array<{ serviceId: string; backupId?: string; errorText?: string }> = [];
for (const service of services) {
if (service.data.serviceCategory && service.data.serviceCategory !== 'workload') {
continue;
}
try {
const backup = await this.createServiceBackup({
identity: {
name: 'cloudly-backup-scheduler',
role: 'admin',
type: 'machine',
userId: 'system',
expiresAt: Date.now() + 3600 * 1000,
jwt: '',
},
serviceId: service.id,
tags: {
trigger: 'scheduled',
},
});
results.push({ serviceId: service.id, backupId: backup.id });
} catch (error) {
results.push({ serviceId: service.id, errorText: (error as Error).message });
}
}
return { results };
}
public async createServiceBackup(requestArg: {
identity: plugins.servezoneInterfaces.data.IIdentity;
serviceId: string;
clusterId?: string;
tags?: Record<string, string>;
}) {
const service = await this.cloudlyRef.serviceManager.CService.getInstance({
id: requestArg.serviceId,
});
if (!service) {
throw new plugins.typedrequest.TypedResponseError(`Service ${requestArg.serviceId} not found`);
}
const now = Date.now();
const backup = new BackupRecord();
backup.id = await BackupRecord.getNewId();
backup.serviceId = service.id;
backup.serviceName = service.data.name;
backup.clusterId = requestArg.clusterId || (requestArg.identity as any).clusterId;
backup.status = 'running';
backup.trigger = 'manual';
backup.snapshots = [];
backup.createdAt = now;
backup.updatedAt = now;
backup.requestedBy = requestArg.identity.userId;
backup.tags = requestArg.tags;
await backup.save();
const replicationEnabled = (requestArg as any).replicate !== false && !!process.env.CLOUDLY_BACKUP_TARGET_TYPE;
try {
const result = await this.fireCoreflowRequest('executeServiceBackup', {
backupId: backup.id,
service: await service.createSavableObject(),
tags: requestArg.tags,
replication: {
enabled: replicationEnabled,
},
}, backup.clusterId);
backup.snapshots = result.snapshots || [];
if (replicationEnabled && !result.replication) {
throw new Error('Coreflow did not complete remote backup replication');
}
backup.replication = result.replication;
backup.status = 'replicated';
backup.completedAt = Date.now();
backup.updatedAt = Date.now();
await backup.save();
await this.applyRetention(backup.serviceId);
} catch (error) {
backup.status = 'failed';
backup.errorText = (error as Error).message;
backup.completedAt = Date.now();
backup.updatedAt = Date.now();
await backup.save();
throw error;
}
return await backup.createSavableObject();
}
private async applyRetention(serviceIdArg: string) {
const keepLast = Number(process.env.CLOUDLY_BACKUP_KEEP_LAST || '24');
if (!Number.isInteger(keepLast) || keepLast <= 0) {
return;
}
const backups = await this.CBackupRecord.getInstances({
serviceId: serviceIdArg,
});
const completedBackups = backups
.filter((backupArg) => backupArg.status === 'replicated' || backupArg.status === 'restored' || backupArg.status === 'failed')
.sort((a, b) => (b.createdAt || 0) - (a.createdAt || 0));
for (const backup of completedBackups.slice(keepLast)) {
await backup.delete();
}
}
public async restoreServiceBackup(requestArg: {
identity: plugins.servezoneInterfaces.data.IIdentity;
backupId: string;
clear?: boolean;
resourceTypes?: TBackupResourceType[];
}) {
const backup = await BackupRecord.getInstance({ id: requestArg.backupId });
if (!backup) {
throw new plugins.typedrequest.TypedResponseError(`Backup ${requestArg.backupId} not found`);
}
if (backup.status !== 'replicated' && backup.status !== 'restored') {
throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} is not restorable in status ${backup.status}`);
}
const service = await this.cloudlyRef.serviceManager.CService.getInstance({
id: backup.serviceId,
});
if (!service) {
throw new plugins.typedrequest.TypedResponseError(`Service ${backup.serviceId} not found`);
}
const previousStatus = backup.status;
backup.status = 'restoring';
backup.updatedAt = Date.now();
await backup.save();
try {
await this.fireCoreflowRequest('executeServiceRestore', {
backupId: backup.id,
service: await service.createSavableObject(),
snapshots: backup.snapshots || [],
clear: requestArg.clear,
resourceTypes: requestArg.resourceTypes,
replication: {
enabled: true,
},
}, backup.clusterId);
backup.status = 'restored';
backup.restoreHistory = [
...(backup.restoreHistory || []),
{
restoredAt: Date.now(),
status: 'restored',
},
];
backup.updatedAt = Date.now();
await backup.save();
} catch (error) {
backup.status = previousStatus;
backup.restoreHistory = [
...(backup.restoreHistory || []),
{
restoredAt: Date.now(),
status: 'failed',
errorText: (error as Error).message,
},
];
backup.updatedAt = Date.now();
await backup.save();
throw error;
}
return await backup.createSavableObject();
}
private getBackupTargetWriter() {
if (!this.backupTargetWriter) {
this.backupTargetWriter = createBackupTargetWriterFromEnv();
}
return this.backupTargetWriter;
}
private normalizeTargetPath(pathArg: string) {
const normalized = plugins.path.posix
.normalize(String(pathArg || '').replace(/\\/g, '/').trim())
.replace(/^\/+/, '');
if (!normalized || normalized === '.' || normalized.startsWith('../') || normalized.includes('/../')) {
throw new Error(`Invalid backup target path ${pathArg}`);
}
return normalized;
}
private getBackupTargetPath(backupArg: BackupRecord) {
return this.normalizeTargetPath([
process.env.CLOUDLY_BACKUP_TARGET_PREFIX || 'serve.zone-backups',
'clusters',
backupArg.clusterId || 'default',
'services',
backupArg.serviceId,
'backups',
backupArg.id,
].filter(Boolean).join('/'));
}
private getArchiveObjectTargetPath(backupArg: BackupRecord, objectPathArg: string) {
return this.normalizeTargetPath(`${this.getBackupTargetPath(backupArg)}/archive/${objectPathArg}`);
}
private getManifestTargetPath(backupArg: BackupRecord) {
return this.normalizeTargetPath(`${this.getBackupTargetPath(backupArg)}/manifest.json`);
}
private getSha256(contentsArg: Buffer) {
return plugins.crypto.createHash('sha256').update(contentsArg).digest('hex');
}
private assertObjectMatches(objectArg: IBackupArchiveObject, contentsArg: Buffer) {
if (contentsArg.length !== objectArg.size || this.getSha256(contentsArg) !== objectArg.sha256) {
throw new Error(`Backup archive object checksum mismatch for ${objectArg.path}`);
}
}
private createManifestBuffer(backupArg: BackupRecord, manifestArg: IBackupArchiveManifest) {
return Buffer.from(`${JSON.stringify({
version: 1,
backupId: backupArg.id,
serviceId: backupArg.serviceId,
serviceName: backupArg.serviceName,
clusterId: backupArg.clusterId,
archive: manifestArg,
}, null, 2)}\n`);
}
private async getBackupForClusterRequest(backupIdArg: string, identityArg: plugins.servezoneInterfaces.data.IIdentity) {
const backup = await BackupRecord.getInstance({ id: backupIdArg });
if (!backup) {
throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} not found`);
}
const identityClusterId = (identityArg as any).clusterId;
if (backup.clusterId && identityClusterId && backup.clusterId !== identityClusterId) {
throw new plugins.typedrequest.TypedResponseError(`Backup ${backupIdArg} does not belong to this cluster`);
}
return backup;
}
public async prepareBackupReplication(requestArg: {
identity: plugins.servezoneInterfaces.data.IIdentity;
backupId: string;
manifest: IBackupArchiveManifest;
}) {
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
const targetWriter = this.getBackupTargetWriter();
const missingObjects: IBackupArchiveObject[] = [];
for (const object of requestArg.manifest.objects || []) {
const targetPath = this.getArchiveObjectTargetPath(backup, object.path);
if (!await targetWriter.hasObject(targetPath, object)) {
missingObjects.push(object);
}
}
backup.status = 'replicating';
backup.updatedAt = Date.now();
await backup.save();
return { missingObjects };
}
public async uploadBackupArchiveObject(requestArg: {
identity: plugins.servezoneInterfaces.data.IIdentity;
backupId: string;
object: IBackupArchiveObject;
contentsBase64: string;
}) {
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
const contents = Buffer.from(requestArg.contentsBase64 || '', 'base64');
this.assertObjectMatches(requestArg.object, contents);
await this.getBackupTargetWriter().putObject(
this.getArchiveObjectTargetPath(backup, requestArg.object.path),
requestArg.object,
contents,
);
return { accepted: true };
}
public async completeBackupReplication(requestArg: {
identity: plugins.servezoneInterfaces.data.IIdentity;
backupId: string;
manifest: IBackupArchiveManifest;
}) {
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
const targetWriter = this.getBackupTargetWriter();
for (const object of requestArg.manifest.objects || []) {
const targetPath = this.getArchiveObjectTargetPath(backup, object.path);
if (!await targetWriter.hasObject(targetPath, object)) {
throw new Error(`Remote backup target is missing archive object ${object.path}`);
}
}
const manifestPath = this.getManifestTargetPath(backup);
const manifestBuffer = this.createManifestBuffer(backup, requestArg.manifest);
const manifestObject = {
path: 'manifest.json',
size: manifestBuffer.length,
sha256: this.getSha256(manifestBuffer),
};
await targetWriter.putObject(manifestPath, manifestObject, manifestBuffer);
const replication: IBackupReplicationResult = {
targetType: targetWriter.targetType,
targetPath: this.getBackupTargetPath(backup),
manifestPath,
manifestSha256: manifestObject.sha256,
objectCount: requestArg.manifest.objects.length,
totalSize: requestArg.manifest.totalSize,
completedAt: Date.now(),
};
backup.replication = replication;
backup.status = 'replicated';
backup.completedAt = replication.completedAt;
backup.updatedAt = replication.completedAt;
await backup.save();
return { replication };
}
public async getBackupArchiveManifest(requestArg: {
identity: plugins.servezoneInterfaces.data.IIdentity;
backupId: string;
}) {
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
if (!backup.replication) {
throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} has not been replicated`);
}
const manifestBuffer = await this.getBackupTargetWriter().readObject(backup.replication.manifestPath);
if (this.getSha256(manifestBuffer) !== backup.replication.manifestSha256) {
throw new Error(`Remote manifest checksum mismatch for backup ${backup.id}`);
}
const parsedManifest = JSON.parse(manifestBuffer.toString('utf8'));
return { manifest: parsedManifest.archive as IBackupArchiveManifest };
}
public async downloadBackupArchiveObject(requestArg: {
identity: plugins.servezoneInterfaces.data.IIdentity;
backupId: string;
object: IBackupArchiveObject;
}) {
const backup = await this.getBackupForClusterRequest(requestArg.backupId, requestArg.identity);
if (!backup.replication) {
throw new plugins.typedrequest.TypedResponseError(`Backup ${backup.id} has not been replicated`);
}
const contents = await this.getBackupTargetWriter().readObject(
this.getArchiveObjectTargetPath(backup, requestArg.object.path),
);
this.assertObjectMatches(requestArg.object, contents);
return {
object: requestArg.object,
contentsBase64: contents.toString('base64'),
};
}
private async fireCoreflowRequest(methodArg: string, payloadArg: Record<string, unknown>, clusterIdArg?: string) {
const typedsocket = this.cloudlyRef.server.typedServer?.typedsocket;
if (!typedsocket) {
throw new Error('Cloudly TypedSocket server is not running');
}
const connections = await typedsocket.findAllTargetConnections(async (connectionArg) => {
const identityTag = await connectionArg.getTagById('identity');
const identity = identityTag?.payload as plugins.servezoneInterfaces.data.IIdentity | undefined;
return identity?.role === 'cluster' && (!clusterIdArg || (identity as any).clusterId === clusterIdArg);
});
if (connections.length === 0) {
throw new Error(clusterIdArg
? `No connected coreflow for cluster ${clusterIdArg}`
: 'No connected coreflow');
}
const request = typedsocket.createTypedRequest<any>(methodArg, connections[0]);
return await request.fire(payloadArg as any);
}
private async passValidIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
await plugins.smartguard.passGuardsOrReject(requestData, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
}
private async passAdminIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
await plugins.smartguard.passGuardsOrReject(requestData, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
}
private async passClusterIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
await this.passValidIdentity(requestData);
if (requestData.identity.role !== 'cluster') {
throw new plugins.typedrequest.TypedResponseError('Cluster identity required');
}
}
}
+54
View File
@@ -0,0 +1,54 @@
import * as plugins from '../plugins.js';
import type { CloudlyBackupManager, IBackupRecordData } from './classes.backupmanager.js';
@plugins.smartdata.managed()
export class BackupRecord extends plugins.smartdata.SmartDataDbDoc<
BackupRecord,
IBackupRecordData,
CloudlyBackupManager
> {
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public serviceId!: string;
@plugins.smartdata.svDb()
public serviceName?: string;
@plugins.smartdata.svDb()
public clusterId?: string;
@plugins.smartdata.svDb()
public status!: IBackupRecordData['status'];
@plugins.smartdata.svDb()
public trigger!: IBackupRecordData['trigger'];
@plugins.smartdata.svDb()
public snapshots!: IBackupRecordData['snapshots'];
@plugins.smartdata.svDb()
public replication?: IBackupRecordData['replication'];
@plugins.smartdata.svDb()
public createdAt!: number;
@plugins.smartdata.svDb()
public updatedAt!: number;
@plugins.smartdata.svDb()
public completedAt?: number;
@plugins.smartdata.svDb()
public requestedBy?: string;
@plugins.smartdata.svDb()
public errorText?: string;
@plugins.smartdata.svDb()
public restoreHistory?: IBackupRecordData['restoreHistory'];
@plugins.smartdata.svDb()
public tags?: Record<string, string>;
}
@@ -0,0 +1,197 @@
import * as plugins from '../plugins.js';
type TArchiveObject = {
path: string;
size: number;
sha256: string;
};
type TTargetType = 's3' | 'smb';
export interface IBackupTargetWriter {
targetType: TTargetType;
hasObject(pathArg: string, objectArg: TArchiveObject): Promise<boolean>;
putObject(pathArg: string, objectArg: TArchiveObject, contentsArg: Buffer): Promise<void>;
readObject(pathArg: string): Promise<Buffer>;
}
const requiredEnv = (nameArg: string) => {
const value = process.env[nameArg];
if (!value) {
throw new Error(`Missing required backup target env ${nameArg}`);
}
return value;
};
const normalizeRemotePath = (pathArg: string) => {
const normalized = plugins.path.posix
.normalize(String(pathArg || '').replace(/\\/g, '/').trim())
.replace(/^\/+/, '');
if (!normalized || normalized === '.' || normalized.startsWith('../') || normalized.includes('/../')) {
throw new Error(`Invalid backup target path ${pathArg}`);
}
return normalized;
};
const getBufferSha256 = (contentsArg: Buffer) => {
return plugins.crypto.createHash('sha256').update(contentsArg).digest('hex');
};
const assertObjectMatches = (objectArg: TArchiveObject, contentsArg: Buffer, labelArg: string) => {
const sha256 = getBufferSha256(contentsArg);
if (contentsArg.length !== objectArg.size || sha256 !== objectArg.sha256) {
throw new Error(`Backup target checksum mismatch for ${labelArg}`);
}
};
const objectMatches = (objectArg: TArchiveObject, contentsArg: Buffer) => {
return contentsArg.length === objectArg.size && getBufferSha256(contentsArg) === objectArg.sha256;
};
class S3BackupTargetWriter implements IBackupTargetWriter {
public targetType: TTargetType = 's3';
private bucketPromise?: Promise<any>;
private async getBucket() {
if (!this.bucketPromise) {
this.bucketPromise = (async () => {
const smartBucket = new plugins.smartbucket.SmartBucket({
endpoint: requiredEnv('CLOUDLY_BACKUP_S3_ENDPOINT'),
accessKey: requiredEnv('CLOUDLY_BACKUP_S3_ACCESS_KEY'),
accessSecret: requiredEnv('CLOUDLY_BACKUP_S3_SECRET_KEY'),
region: process.env.CLOUDLY_BACKUP_S3_REGION || 'us-east-1',
...(process.env.CLOUDLY_BACKUP_S3_PORT
? { port: Number(process.env.CLOUDLY_BACKUP_S3_PORT) }
: {}),
...(process.env.CLOUDLY_BACKUP_S3_USE_SSL
? { useSsl: process.env.CLOUDLY_BACKUP_S3_USE_SSL !== 'false' }
: {}),
} as any);
const bucketName = requiredEnv('CLOUDLY_BACKUP_S3_BUCKET');
if (await smartBucket.bucketExists(bucketName)) {
return await smartBucket.getBucketByName(bucketName);
}
return await smartBucket.createBucket(bucketName);
})();
}
return await this.bucketPromise;
}
public async hasObject(pathArg: string, objectArg: TArchiveObject) {
try {
return objectMatches(objectArg, await this.readObject(pathArg));
} catch {
return false;
}
}
public async putObject(pathArg: string, objectArg: TArchiveObject, contentsArg: Buffer) {
const targetPath = normalizeRemotePath(pathArg);
assertObjectMatches(objectArg, contentsArg, targetPath);
const bucket = await this.getBucket();
const tempPath = `${targetPath}.upload-${Date.now()}-${plugins.smartunique.shortId()}.tmp`;
try {
await bucket.fastPut({ path: tempPath, contents: contentsArg, overwrite: true });
assertObjectMatches(objectArg, await bucket.fastGet({ path: tempPath }), tempPath);
await bucket.fastMove({ sourcePath: tempPath, destinationPath: targetPath, overwrite: true });
assertObjectMatches(objectArg, await bucket.fastGet({ path: targetPath }), targetPath);
} finally {
await bucket.fastRemove({ path: tempPath }).catch(() => {});
}
}
public async readObject(pathArg: string) {
const bucket = await this.getBucket();
return await bucket.fastGet({ path: normalizeRemotePath(pathArg) });
}
}
class SmbBackupTargetWriter implements IBackupTargetWriter {
public targetType: TTargetType = 'smb';
private clientPromise?: Promise<plugins.smartsamba.SambaClient>;
private async getClient() {
if (!this.clientPromise) {
this.clientPromise = (async () => {
const client = new plugins.smartsamba.SambaClient({
host: requiredEnv('CLOUDLY_BACKUP_SMB_HOST'),
...(process.env.CLOUDLY_BACKUP_SMB_PORT
? { port: Number(process.env.CLOUDLY_BACKUP_SMB_PORT) }
: {}),
auth: {
...(process.env.CLOUDLY_BACKUP_SMB_USERNAME
? { username: process.env.CLOUDLY_BACKUP_SMB_USERNAME }
: {}),
...(process.env.CLOUDLY_BACKUP_SMB_PASSWORD
? { password: process.env.CLOUDLY_BACKUP_SMB_PASSWORD }
: {}),
...(process.env.CLOUDLY_BACKUP_SMB_DOMAIN
? { domain: process.env.CLOUDLY_BACKUP_SMB_DOMAIN }
: {}),
},
});
await client.start();
return client;
})();
}
return await this.clientPromise;
}
private getShare() {
return requiredEnv('CLOUDLY_BACKUP_SMB_SHARE');
}
private async ensureParentDirectory(pathArg: string) {
const client = await this.getClient();
const parent = plugins.path.posix.dirname(pathArg);
if (!parent || parent === '.') {
return;
}
const parts = parent.split('/').filter(Boolean);
let current = '';
for (const part of parts) {
current = current ? `${current}/${part}` : part;
await client.createDirectory(this.getShare(), current).catch(() => {});
}
}
public async hasObject(pathArg: string, objectArg: TArchiveObject) {
try {
return objectMatches(objectArg, await this.readObject(pathArg));
} catch {
return false;
}
}
public async putObject(pathArg: string, objectArg: TArchiveObject, contentsArg: Buffer) {
const targetPath = normalizeRemotePath(pathArg);
assertObjectMatches(objectArg, contentsArg, targetPath);
const client = await this.getClient();
const share = this.getShare();
const tempPath = `${targetPath}.upload-${Date.now()}-${plugins.smartunique.shortId()}.tmp`;
await this.ensureParentDirectory(targetPath);
try {
await client.writeFile(share, tempPath, contentsArg);
assertObjectMatches(objectArg, await client.readFile(share, tempPath), tempPath);
await client.deleteFile(share, targetPath).catch(() => {});
await client.rename(share, tempPath, targetPath);
assertObjectMatches(objectArg, await client.readFile(share, targetPath), targetPath);
} finally {
await client.deleteFile(share, tempPath).catch(() => {});
}
}
public async readObject(pathArg: string) {
return await (await this.getClient()).readFile(this.getShare(), normalizeRemotePath(pathArg));
}
}
export const createBackupTargetWriterFromEnv = (): IBackupTargetWriter => {
const targetType = process.env.CLOUDLY_BACKUP_TARGET_TYPE as TTargetType | undefined;
if (targetType === 's3') {
return new S3BackupTargetWriter();
}
if (targetType === 'smb') {
return new SmbBackupTargetWriter();
}
throw new Error('No remote backup target configured. Set CLOUDLY_BACKUP_TARGET_TYPE to s3 or smb.');
};
+22 -13
View File
@@ -12,29 +12,38 @@ export class BareMetal extends plugins.smartdata.SmartDataDbDoc<
public static async createFromHetznerServer(
hetznerServerArg: plugins.hetznercloud.HetznerServer,
) {
const serverData = hetznerServerArg.data;
if (!serverData) {
throw new Error('Hetzner server response is missing server data');
}
const ipv4 = serverData.public_net.ipv4;
if (!ipv4) {
throw new Error(`Hetzner server ${serverData.id} has no primary IPv4 address`);
}
const newBareMetal = new BareMetal();
newBareMetal.id = plugins.smartunique.shortId(8);
const data: plugins.servezoneInterfaces.data.IBareMetal['data'] = {
hostname: hetznerServerArg.data.name,
primaryIp: hetznerServerArg.data.public_net.ipv4.ip,
hostname: serverData.name,
primaryIp: ipv4.ip,
provider: 'hetzner',
location: hetznerServerArg.data.datacenter.name,
location: serverData.datacenter.name,
specs: {
cpuModel: hetznerServerArg.data.server_type.cpu_type,
cpuCores: hetznerServerArg.data.server_type.cores,
memoryGB: hetznerServerArg.data.server_type.memory,
storageGB: hetznerServerArg.data.server_type.disk,
cpuModel: serverData.server_type.cpu_type,
cpuCores: serverData.server_type.cores,
memoryGB: serverData.server_type.memory,
storageGB: serverData.server_type.disk,
storageType: 'nvme',
},
powerState: hetznerServerArg.data.status === 'running' ? 'on' : 'off',
powerState: serverData.status === 'running' ? 'on' : 'off',
osInfo: {
name: 'Debian',
version: '12',
},
assignedNodeIds: [],
providerMetadata: {
hetznerServerId: hetznerServerArg.data.id,
hetznerServerName: hetznerServerArg.data.name,
hetznerServerId: serverData.id,
hetznerServerName: serverData.name,
},
};
Object.assign(newBareMetal, { data });
@@ -44,10 +53,10 @@ export class BareMetal extends plugins.smartdata.SmartDataDbDoc<
// INSTANCE
@plugins.smartdata.unI()
public id: string;
public id!: string;
@plugins.smartdata.svDb()
public data: plugins.servezoneInterfaces.data.IBareMetal['data'];
public data!: plugins.servezoneInterfaces.data.IBareMetal['data'];
constructor() {
super();
@@ -101,4 +110,4 @@ export class BareMetal extends plugins.smartdata.SmartDataDbDoc<
}
return false;
}
}
}
@@ -7,7 +7,7 @@ export class CloudlyBaremetalManager {
public cloudlyRef: Cloudly;
public typedRouter = new plugins.typedrequest.TypedRouter();
public hetznerAccount: plugins.hetznercloud.HetznerAccount;
public hetznerAccount?: plugins.hetznercloud.HetznerAccount;
public get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
@@ -119,18 +119,22 @@ export class CloudlyBaremetalManager {
* Create baremetal from Hetzner server
*/
public async createBaremetalFromHetznerServer(hetznerServer: plugins.hetznercloud.HetznerServer): Promise<BareMetal> {
const serverData = hetznerServer.data;
if (!serverData) {
throw new Error('Hetzner server response is missing server data');
}
// Check if baremetal already exists for this Hetzner server
const existingBaremetals = await this.CBareMetal.getInstances({});
for (const baremetal of existingBaremetals) {
if (baremetal.data.providerMetadata?.hetznerServerId === hetznerServer.data.id) {
logger.log('info', `BareMetal already exists for Hetzner server ${hetznerServer.data.id}`);
if (baremetal.data.providerMetadata?.hetznerServerId === serverData.id) {
logger.log('info', `BareMetal already exists for Hetzner server ${serverData.id}`);
return baremetal;
}
}
// Create new baremetal
const newBaremetal = await BareMetal.createFromHetznerServer(hetznerServer);
logger.log('success', `Created new BareMetal ${newBaremetal.id} from Hetzner server ${hetznerServer.data.id}`);
logger.log('success', `Created new BareMetal ${newBaremetal.id} from Hetzner server ${serverData.id}`);
return newBaremetal;
}
@@ -173,4 +177,4 @@ export class CloudlyBaremetalManager {
throw new Error(`Provider ${options.provider} not supported or not configured`);
}
}
}
@@ -0,0 +1,87 @@
import * as plugins from '../plugins.js';
export type TBaseOsImageArchitecture = 'amd64' | 'arm64' | 'rpi';
export type TBaseOsImageKind = 'ubuntu-iso' | 'balena-raw';
export type TBaseOsImageSourcePreset =
| 'balena-generic-amd64'
| 'balena-generic-aarch64'
| 'balena-raspberrypi4-64';
export type TBaseOsImageBuildStatus = 'queued' | 'building' | 'ready' | 'failed' | 'cancelled';
export interface IBaseOsImageArtifact {
bucketName: string;
key: string;
filename: string;
contentType: string;
size: number;
sha256: string;
createdAt: number;
}
export interface IBaseOsImageBuildPublic {
id: string;
data: {
status: TBaseOsImageBuildStatus;
architecture: TBaseOsImageArchitecture;
imageKind?: TBaseOsImageKind;
cloudlyUrl: string;
sourceImageUrl?: string;
sourceImagePreset?: TBaseOsImageSourcePreset;
balenaOsVersion?: string;
ubuntuVersion?: string;
hostname?: string;
wifiSsid?: string;
sshPublicKey?: string;
artifact?: IBaseOsImageArtifact;
errorText?: string;
logs: string[];
createdAt: number;
updatedAt: number;
startedAt?: number;
completedAt?: number;
expiresAt?: number;
};
}
@plugins.smartdata.managed()
export class BaseOsImageBuild extends plugins.smartdata.SmartDataDbDoc<
BaseOsImageBuild,
IBaseOsImageBuildPublic
> {
constructor(optionsArg?: IBaseOsImageBuildPublic & {
provisioningTokenHash?: string;
provisioningTokenConsumedAt?: number;
downloadTokenHash?: string;
downloadTokenExpiresAt?: number;
}) {
super();
if (optionsArg) {
Object.assign(this, optionsArg);
}
}
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public provisioningTokenHash!: string;
@plugins.smartdata.svDb()
public provisioningTokenConsumedAt?: number;
@plugins.smartdata.svDb()
public downloadTokenHash?: string;
@plugins.smartdata.svDb()
public downloadTokenExpiresAt?: number;
@plugins.smartdata.svDb()
public data!: IBaseOsImageBuildPublic['data'];
public toPublicBuild(): IBaseOsImageBuildPublic {
return {
id: this.id,
data: this.data,
};
}
}
File diff suppressed because it is too large Load Diff
+67
View File
@@ -0,0 +1,67 @@
import * as plugins from '../plugins.js';
export type TBaseOsRuntimeLevel = 'app-layer' | 'host-os' | 'target-state';
export type TBaseOsCloudlyConnectionStatus =
| 'not-configured'
| 'connecting'
| 'connected'
| 'failed';
export interface IBaseOsRuntimeInfo {
runtime: 'baseos';
runtimeLevel: TBaseOsRuntimeLevel;
nodeId: string;
cloudlyUrl?: string;
cloudlyConnectionStatus: TBaseOsCloudlyConnectionStatus;
supervisorAvailable: boolean;
supervisorAddress?: string;
deviceState?: Record<string, unknown>;
stateStatus?: Record<string, unknown>;
checkedAt: number;
}
export interface IBaseOsDesiredState {
release?: string;
targetState?: Record<string, unknown>;
updatedAt?: number;
}
export interface IBaseOsNodeData {
runtimeInfo: IBaseOsRuntimeInfo;
desiredState?: IBaseOsDesiredState;
createdAt: number;
updatedAt: number;
lastHeartbeatAt?: number;
}
export interface IBaseOsNodePublic {
id: string;
data: IBaseOsNodeData;
}
@plugins.smartdata.managed()
export class BaseOsNode extends plugins.smartdata.SmartDataDbDoc<BaseOsNode, IBaseOsNodePublic> {
constructor(optionsArg?: IBaseOsNodePublic & { nodeToken?: string }) {
super();
if (optionsArg) {
Object.assign(this, optionsArg);
}
}
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public nodeToken!: string;
@plugins.smartdata.svDb()
public data!: IBaseOsNodeData;
public toPublicNode(): IBaseOsNodePublic {
return {
id: this.id,
data: this.data,
};
}
}
+2 -2
View File
@@ -17,10 +17,10 @@ export class Cluster extends plugins.smartdata.SmartDataDbDoc<
// INSTANCE
@plugins.smartdata.unI()
public id: string;
public id!: string;
@plugins.smartdata.svDb()
public data: plugins.servezoneInterfaces.data.ICluster['data'];
public data!: plugins.servezoneInterfaces.data.ICluster['data'];
constructor() {
super();
+43 -11
View File
@@ -22,16 +22,19 @@ export class ClusterManager {
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.cluster.IRequest_CreateCluster>(
new plugins.typedrequest.TypedHandler('createCluster', async (dataArg) => {
// TODO: guards
new plugins.typedrequest.TypedHandler('createCluster', async (dataArg, toolsArg) => {
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], dataArg);
const setupMode = dataArg.setupMode || 'manual'; // Default to manual if not specified
const cluster = await this.createCluster({
id: plugins.smartunique.uniSimple('cluster'),
data: {
userId: null, // this is created by the createCluster method
userId: '', // this is created by the createCluster method
name: dataArg.clusterName,
setupMode: setupMode,
acmeInfo: null,
acmeInfo: {
serverAddress: '',
serverSecret: '',
},
cloudlyUrl: `https://${this.cloudlyRef.config.data.publicUrl}:${this.cloudlyRef.config.data.publicPort}/`,
nodes: [],
sshKeys: [],
@@ -51,8 +54,8 @@ export class ClusterManager {
);
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.cluster.IReq_Any_Cloudly_GetClusters>(
new plugins.typedrequest.TypedHandler('getClusters', async (dataArg) => {
// TODO: do authentication here
new plugins.typedrequest.TypedHandler('getClusters', async (dataArg, toolsArg) => {
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], dataArg);
const clusters = await this.getAllClusters();
return {
clusters: await Promise.all(
@@ -62,10 +65,41 @@ export class ClusterManager {
}),
);
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.cluster.IReq_Any_Cloudly_GetClusterById>(
new plugins.typedrequest.TypedHandler('getClusterById', async (dataArg, toolsArg) => {
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], dataArg);
const cluster = await this.CCluster.getInstance({ id: (dataArg as any).clusterId });
if (!cluster) {
throw new plugins.typedrequest.TypedResponseError('Cluster not found');
}
return {
cluster: await cluster.createSavableObject(),
};
}),
);
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.cluster.IReq_Any_Cloudly_UpdateCluster>(
new plugins.typedrequest.TypedHandler('updateCluster', async (dataArg, toolsArg) => {
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], dataArg);
const cluster = await this.CCluster.getInstance({ id: (dataArg as any).clusterId });
if (!cluster) {
throw new plugins.typedrequest.TypedResponseError('Cluster not found');
}
cluster.data = {
...cluster.data,
...dataArg.clusterData,
};
await cluster.save();
return {
resultCluster: await cluster.createSavableObject(),
};
}),
);
// delete cluster
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.cluster.IReq_Any_Cloudly_DeleteClusterById>(
new plugins.typedrequest.TypedHandler('deleteClusterById', async (reqDataArg, toolsArg) => {
await toolsArg.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], reqDataArg);
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], reqDataArg);
await this.deleteCluster(reqDataArg.clusterId);
return {
ok: true,
@@ -134,11 +168,11 @@ export class ClusterManager {
* @param configObjectArg
*/
public async createCluster(configObjectArg: plugins.servezoneInterfaces.data.ICluster) {
// TODO: guards
// lets create the cluster user
const clusterUser = new this.cloudlyRef.authManager.CUser({
id: await this.cloudlyRef.authManager.CUser.getNewId(),
data: {
username: `cluster-${configObjectArg.id}`,
role: 'cluster',
type: 'machine',
tokens: [
@@ -151,9 +185,7 @@ export class ClusterManager {
},
});
await clusterUser.save();
Object.assign(configObjectArg, {
userId: clusterUser.id,
});
configObjectArg.data.userId = clusterUser.id;
const clusterInstance = await Cluster.fromConfigObject(configObjectArg);
await clusterInstance.save();
return clusterInstance;
+175 -7
View File
@@ -1,6 +1,31 @@
import * as plugins from '../plugins.js';
import { Cloudly } from '../classes.cloudly.js';
import type { Cluster } from '../manager.cluster/classes.cluster.js';
import { logger } from '../logger.js';
type TCoreflowDeploymentRequest =
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_RestartDeployment
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_KillDeployment
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_DeploymentWorkspaceReadFile
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_DeploymentWorkspaceWriteFile
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_DeploymentWorkspaceReadDir
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_DeploymentWorkspaceMkdir
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_DeploymentWorkspaceRm
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_DeploymentWorkspaceExists
| plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_DeploymentWorkspaceExec;
type TCoreflowDeploymentActionMethod =
| 'coreflowRestartDeployment'
| 'coreflowKillDeployment';
type TCoreflowDeploymentActionRequest = Extract<TCoreflowDeploymentRequest, {
method: TCoreflowDeploymentActionMethod;
}>;
export type TCoreflowDeploymentWorkspaceMethod = Exclude<
TCoreflowDeploymentRequest['method'],
TCoreflowDeploymentActionMethod
>;
/**
* in charge of talking to coreflow services on clusters
@@ -35,14 +60,14 @@ export class CloudlyCoreflowManager {
'The supplied token is not valid. The user is not a machine.'
);
}
let cluster: Cluster;
let cluster: Cluster | undefined;
if (user.data.role === 'cluster') {
cluster = await this.cloudlyRef.clusterManager.getClusterBy_UserId(user.id);
}
const expiryTimestamp = Date.now() + 3600 * 1000 * 24 * 365;
return {
identity: {
name: user.data.username,
name: user.data.username || user.id,
role: user.data.role,
type: 'machine', // if someone authenticates by token, they are a machine, no matter what.
userId: user.id,
@@ -71,12 +96,9 @@ export class CloudlyCoreflowManager {
const identity = dataArg.identity;
console.log('trying to get clusterConfigSet');
console.log(dataArg);
const cluster = await this.cloudlyRef.clusterManager.getClusterBy_Identity(identity);
const clusterConfig = await this.getClusterConfigPayloadForIdentity(identity);
console.log('got cluster config and sending it back to coreflow');
return {
configData: await cluster.createSavableObject(),
services: [],
};
return clusterConfig;
}
)
);
@@ -98,4 +120,150 @@ export class CloudlyCoreflowManager {
)
);
}
public async getClusterConfigPayloadForIdentity(
identityArg: plugins.servezoneInterfaces.data.IIdentity,
): Promise<plugins.servezoneInterfaces.requests.config.IRequest_Cloudly_Coreflow_PushClusterConfig['request']> {
const cluster = await this.cloudlyRef.clusterManager.getClusterBy_Identity(identityArg);
const services = await this.cloudlyRef.serviceManager.CService.getInstances({});
const platformDesiredState = await this.cloudlyRef.platformManager.getPlatformDesiredState();
const settings = await this.cloudlyRef.settingsManager.getSettings();
const targetPort = Number(settings.dcrouterTargetPort || '80');
const externalGateway = settings.dcrouterGatewayUrl && settings.dcrouterGatewayApiToken
? {
url: settings.dcrouterGatewayUrl,
apiToken: settings.dcrouterGatewayApiToken,
workHosterType: 'cloudly' as const,
workHosterId: settings.dcrouterWorkHosterId || cluster.id,
targetHost: settings.dcrouterTargetHost,
targetPort: Number.isInteger(targetPort) && targetPort > 0 ? targetPort : 80,
}
: undefined;
const payload: plugins.servezoneInterfaces.requests.config.IRequest_Cloudly_Coreflow_PushClusterConfig['request'] & {
externalGateway?: typeof externalGateway;
} = {
configData: await cluster.createSavableObject(),
services: await Promise.all(services.map((service) => service.createSavableObject())),
platformProviderConfigs: platformDesiredState.providerConfigs,
platformBindings: platformDesiredState.bindings,
};
if (externalGateway) {
payload.externalGateway = externalGateway;
}
return payload;
}
public async pushClusterConfigToConnectedCoreflows() {
const typedsocket = this.cloudlyRef.server.typedServer?.typedsocket;
if (!typedsocket) {
return 0;
}
const connections = await typedsocket.findAllTargetConnections(async (connectionArg) => {
const identityTag = await connectionArg.getTagById('identity');
const identity = identityTag?.payload as plugins.servezoneInterfaces.data.IIdentity | undefined;
return identity?.role === 'cluster' && !!identity.userId;
});
await Promise.all(
connections.map(async (connectionArg) => {
const identityTag = await connectionArg.getTagById('identity');
const identity = identityTag?.payload as plugins.servezoneInterfaces.data.IIdentity;
try {
const pushClusterConfig = typedsocket.createTypedRequest<plugins.servezoneInterfaces.requests.config.IRequest_Cloudly_Coreflow_PushClusterConfig>(
'pushClusterConfig',
connectionArg,
);
await pushClusterConfig.fire(await this.getClusterConfigPayloadForIdentity(identity));
} catch (error) {
logger.log('error', `failed to push cluster config to coreflow ${identity.userId}: ${(error as Error).message}`);
}
}),
);
return connections.length;
}
public async getRuntimeDeploymentsForService(
serviceArg: plugins.servezoneInterfaces.data.IService,
): Promise<plugins.servezoneInterfaces.data.IDeployment[]> {
const connections = await this.getConnectedCoreflowConnections();
const deployments: plugins.servezoneInterfaces.data.IDeployment[] = [];
for (const connection of connections) {
try {
const request = this.cloudlyRef.server.typedServer.typedsocket.createTypedRequest<plugins.servezoneInterfaces.requests.deployment.IReq_Cloudly_Coreflow_GetServiceDeployments>(
'coreflowGetServiceDeployments',
connection,
);
const response = await request.fire({ service: serviceArg });
deployments.push(...(response.deployments || []));
} catch (error) {
logger.log('warn', `failed to query coreflow deployments: ${(error as Error).message}`);
}
}
return deployments;
}
public async fireDeploymentRuntimeAction(
methodArg: TCoreflowDeploymentActionMethod,
deploymentIdArg: string,
): Promise<{ deployment: plugins.servezoneInterfaces.data.IDeployment }> {
const response = await this.fireCoreflowRequestUntilFound<TCoreflowDeploymentActionRequest>(methodArg, {
deploymentId: deploymentIdArg,
});
if (!response.deployment) {
throw new plugins.typedrequest.TypedResponseError('Coreflow did not return deployment data');
}
return { deployment: response.deployment };
}
public async fireDeploymentWorkspaceRequest(
methodArg: TCoreflowDeploymentWorkspaceMethod,
payloadArg: Extract<TCoreflowDeploymentRequest, { method: typeof methodArg }>['request'],
) {
return await this.fireCoreflowRequestUntilFound(methodArg, payloadArg);
}
private async fireCoreflowRequestUntilFound<TRequest extends TCoreflowDeploymentRequest>(
methodArg: TRequest['method'],
payloadArg: TRequest['request'],
): Promise<TRequest['response']> {
const connections = await this.getConnectedCoreflowConnections();
if (connections.length === 0) {
throw new plugins.typedrequest.TypedResponseError('No connected coreflow');
}
let lastError: Error | undefined;
for (const connection of connections) {
try {
const request = this.cloudlyRef.server.typedServer.typedsocket.createTypedRequest<TRequest>(
methodArg,
connection,
);
const response = await request.fire(payloadArg);
if (response?.found) {
return response;
}
} catch (error) {
lastError = error as Error;
}
}
throw new plugins.typedrequest.TypedResponseError(
lastError?.message || 'No connected coreflow found the requested deployment',
);
}
private async getConnectedCoreflowConnections() {
const typedsocket = this.cloudlyRef.server.typedServer?.typedsocket;
if (!typedsocket) {
return [];
}
return await typedsocket.findAllTargetConnections(async (connectionArg) => {
const identityTag = await connectionArg.getTagById('identity');
const identity = identityTag?.payload as plugins.servezoneInterfaces.data.IIdentity | undefined;
return identity?.role === 'cluster' && !!identity.userId;
});
}
}
@@ -0,0 +1,98 @@
import * as plugins from '../plugins.js';
@plugins.smartdata.managed()
export class Deployment extends plugins.smartdata.SmartDataDbDoc<
Deployment,
plugins.servezoneInterfaces.data.IDeployment
> {
@plugins.smartdata.unI()
public id: string = plugins.smartunique.uniSimple('deployment');
@plugins.smartdata.svDb()
public serviceId!: string;
@plugins.smartdata.svDb()
public nodeId!: string;
@plugins.smartdata.svDb()
public containerId?: string;
@plugins.smartdata.svDb()
public usedImageId!: string;
@plugins.smartdata.svDb()
public version!: string;
@plugins.smartdata.svDb()
public deployedAt!: number;
@plugins.smartdata.svDb()
public deploymentLog: string[] = [];
@plugins.smartdata.svDb()
public status!: 'scheduled' | 'starting' | 'running' | 'stopping' | 'stopped' | 'failed';
@plugins.smartdata.svDb()
public healthStatus?: 'healthy' | 'unhealthy' | 'unknown';
@plugins.smartdata.svDb()
public resourceUsage?: {
cpuUsagePercent: number;
memoryUsedMB: number;
lastUpdated: number;
};
public static async createDeployment(
deploymentData: Partial<plugins.servezoneInterfaces.data.IDeployment>
): Promise<Deployment> {
const deployment = new Deployment();
if (deploymentData.serviceId) deployment.serviceId = deploymentData.serviceId;
if (deploymentData.nodeId) deployment.nodeId = deploymentData.nodeId;
if (deploymentData.containerId) deployment.containerId = deploymentData.containerId;
if (deploymentData.usedImageId) deployment.usedImageId = deploymentData.usedImageId;
if (deploymentData.version) deployment.version = deploymentData.version;
if (deploymentData.deployedAt) deployment.deployedAt = deploymentData.deployedAt;
if (deploymentData.deploymentLog) deployment.deploymentLog = deploymentData.deploymentLog;
if (deploymentData.status) deployment.status = deploymentData.status;
if (deploymentData.healthStatus) deployment.healthStatus = deploymentData.healthStatus;
if (deploymentData.resourceUsage) deployment.resourceUsage = deploymentData.resourceUsage;
await deployment.save();
return deployment;
}
public async updateHealthStatus(healthStatus: 'healthy' | 'unhealthy' | 'unknown') {
this.healthStatus = healthStatus;
await this.save();
}
public async updateResourceUsage(cpuUsagePercent: number, memoryUsedMB: number) {
this.resourceUsage = {
cpuUsagePercent,
memoryUsedMB,
lastUpdated: Date.now(),
};
await this.save();
}
public async addLogEntry(entry: string) {
this.deploymentLog.push(entry);
await this.save();
}
public async createSavableObject(): Promise<plugins.servezoneInterfaces.data.IDeployment> {
return {
id: this.id,
serviceId: this.serviceId,
nodeId: this.nodeId,
containerId: this.containerId,
usedImageId: this.usedImageId,
version: this.version,
deployedAt: this.deployedAt,
deploymentLog: this.deploymentLog,
status: this.status,
healthStatus: this.healthStatus,
resourceUsage: this.resourceUsage,
};
}
}
@@ -0,0 +1,379 @@
import type { Cloudly } from '../classes.cloudly.js';
import * as plugins from '../plugins.js';
import { Deployment } from './classes.deployment.js';
import type { TCoreflowDeploymentWorkspaceMethod } from '../manager.coreflow/coreflowmanager.js';
export class DeploymentManager {
public typedrouter = new plugins.typedrequest.TypedRouter();
public cloudlyRef: Cloudly;
get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
}
public CDeployment = plugins.smartdata.setDefaultManagerForDoc(this, Deployment);
constructor(cloudlyRef: Cloudly) {
this.cloudlyRef = cloudlyRef;
// Connect typedrouter to main router
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
// Get all deployments
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_GetDeployments>(
'getDeployments',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const deployments = await this.CDeployment.getInstances({});
return {
deployments: await Promise.all(
deployments.map((deployment) => deployment.createSavableObject())
),
};
}
)
);
// Get deployment by ID
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_GetDeploymentById>(
'getDeploymentById',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const deployment = await this.CDeployment.getInstance({
id: reqArg.deploymentId,
});
if (!deployment) {
throw new Error('Deployment not found');
}
return {
deployment: await deployment.createSavableObject(),
};
}
)
);
// Get deployments by service
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_GetDeploymentsByService>(
'getDeploymentsByService',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const service = await this.cloudlyRef.serviceManager.CService.getInstance({
id: reqArg.serviceId,
});
if (service) {
const runtimeDeployments = await this.cloudlyRef.coreflowManager.getRuntimeDeploymentsForService(
await service.createSavableObject(),
);
if (runtimeDeployments.length > 0) {
return { deployments: runtimeDeployments };
}
}
const deployments = await this.CDeployment.getInstances({
serviceId: reqArg.serviceId,
});
return {
deployments: await Promise.all(
deployments.map((deployment) => deployment.createSavableObject())
),
};
}
)
);
// Get deployments by node
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_GetDeploymentsByNode>(
'getDeploymentsByNode',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const deployments = await this.CDeployment.getInstances({
nodeId: reqArg.nodeId,
});
return {
deployments: await Promise.all(
deployments.map((deployment) => deployment.createSavableObject())
),
};
}
)
);
// Create deployment
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_CreateDeployment>(
'createDeployment',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const deployment = await Deployment.createDeployment(reqArg.deploymentData);
return {
deployment: await deployment.createSavableObject(),
};
}
)
);
// Update deployment
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_UpdateDeployment>(
'updateDeployment',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const deployment = await this.CDeployment.getInstance({
id: reqArg.deploymentId,
});
if (!deployment) {
throw new Error('Deployment not found');
}
// Update fields
if (reqArg.deploymentData.status !== undefined) {
deployment.status = reqArg.deploymentData.status;
}
if (reqArg.deploymentData.healthStatus !== undefined) {
deployment.healthStatus = reqArg.deploymentData.healthStatus;
}
if (reqArg.deploymentData.containerId !== undefined) {
deployment.containerId = reqArg.deploymentData.containerId;
}
if (reqArg.deploymentData.resourceUsage !== undefined) {
deployment.resourceUsage = reqArg.deploymentData.resourceUsage;
}
await deployment.save();
return {
deployment: await deployment.createSavableObject(),
};
}
)
);
// Delete deployment
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_DeleteDeploymentById>(
'deleteDeploymentById',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const deployment = await this.CDeployment.getInstance({
id: reqArg.deploymentId,
});
if (!deployment) {
throw new Error('Deployment not found');
}
const serviceId = deployment.serviceId;
await deployment.delete();
// Check if this was the last deployment for the service
const remainingDeployments = await this.getDeploymentsForService(serviceId);
if (remainingDeployments.length === 0) {
// Deactivate DNS entries if no more deployments exist
await this.cloudlyRef.dnsManager.deactivateServiceDnsEntries(serviceId);
}
return {
success: true,
};
}
)
);
// Restart deployment
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_RestartDeployment>(
'restartDeployment',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
const result = await this.cloudlyRef.coreflowManager.fireDeploymentRuntimeAction(
'coreflowRestartDeployment',
reqArg.deploymentId,
);
return {
success: true,
deployment: result.deployment,
};
}
)
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_KillDeployment>(
'killDeployment',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
const result = await this.cloudlyRef.coreflowManager.fireDeploymentRuntimeAction(
'coreflowKillDeployment',
reqArg.deploymentId,
);
return {
success: true,
deployment: result.deployment,
};
},
),
);
// Scale deployment
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.deployment.IReq_Any_Cloudly_ScaleDeployment>(
'scaleDeployment',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
// TODO: Implement scaling logic
// This would create/delete deployment instances based on replicas count
const deployment = await this.CDeployment.getInstance({
id: reqArg.deploymentId,
});
if (!deployment) {
throw new Error('Deployment not found');
}
return {
success: true,
deployment: await deployment.createSavableObject(),
};
}
)
);
const addDeploymentWorkspaceHandler = (methodArg: string, coreflowMethodArg: TCoreflowDeploymentWorkspaceMethod) => {
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<any>(methodArg, async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
const { identity: _identity, ...payload } = reqArg;
const response = await this.cloudlyRef.coreflowManager.fireDeploymentWorkspaceRequest(
coreflowMethodArg,
payload,
);
const { found: _found, ...publicResponse } = response;
return publicResponse;
}),
);
};
addDeploymentWorkspaceHandler('deploymentWorkspaceReadFile', 'coreflowDeploymentWorkspaceReadFile');
addDeploymentWorkspaceHandler('deploymentWorkspaceWriteFile', 'coreflowDeploymentWorkspaceWriteFile');
addDeploymentWorkspaceHandler('deploymentWorkspaceReadDir', 'coreflowDeploymentWorkspaceReadDir');
addDeploymentWorkspaceHandler('deploymentWorkspaceMkdir', 'coreflowDeploymentWorkspaceMkdir');
addDeploymentWorkspaceHandler('deploymentWorkspaceRm', 'coreflowDeploymentWorkspaceRm');
addDeploymentWorkspaceHandler('deploymentWorkspaceExists', 'coreflowDeploymentWorkspaceExists');
addDeploymentWorkspaceHandler('deploymentWorkspaceExec', 'coreflowDeploymentWorkspaceExec');
}
/**
* Get all deployments
*/
public async getAllDeployments(): Promise<Deployment[]> {
return await this.CDeployment.getInstances({});
}
/**
* Get deployments for a specific service
*/
public async getDeploymentsForService(serviceId: string): Promise<Deployment[]> {
return await this.CDeployment.getInstances({
serviceId,
});
}
/**
* Get deployments for a specific node
*/
public async getDeploymentsForNode(nodeId: string): Promise<Deployment[]> {
return await this.CDeployment.getInstances({
nodeId,
});
}
/**
* Create a new deployment
*/
public async createDeployment(
serviceId: string,
nodeId: string,
version: string = 'latest'
): Promise<Deployment> {
const deployment = await Deployment.createDeployment({
serviceId,
nodeId,
version,
status: 'scheduled',
deployedAt: Date.now(),
deploymentLog: [`Deployment created at ${new Date().toISOString()}`],
});
// Activate DNS entries for the service
await this.cloudlyRef.dnsManager.activateServiceDnsEntries(serviceId);
// Get the node's IP address and update DNS entries
const node = await this.cloudlyRef.nodeManager.CClusterNode.getInstance({
id: nodeId,
});
if (node?.data.baremetalId) {
const baremetal = await this.cloudlyRef.baremetalManager.CBareMetal.getInstance({
id: node.data.baremetalId,
});
if (baremetal?.data.primaryIp) {
await this.cloudlyRef.dnsManager.updateServiceDnsEntriesIp(serviceId, baremetal.data.primaryIp);
}
}
return deployment;
}
public async start() {
// DeploymentManager is ready - handlers are already registered in constructor
console.log('DeploymentManager started');
}
public async stop() {
// Cleanup if needed
console.log('DeploymentManager stopped');
}
}
+149
View File
@@ -0,0 +1,149 @@
import * as plugins from '../plugins.js';
import { DnsManager } from './classes.dnsmanager.js';
@plugins.smartdata.managed()
export class DnsEntry extends plugins.smartdata.SmartDataDbDoc<
DnsEntry,
plugins.servezoneInterfaces.data.IDnsEntry,
DnsManager
> {
// STATIC
public static async getDnsEntryById(dnsEntryIdArg: string) {
const dnsEntry = await this.getInstance({
id: dnsEntryIdArg,
});
return dnsEntry;
}
public static async getDnsEntries(filterArg?: { zone?: string }) {
const filter: any = {};
if (filterArg?.zone) {
filter['data.zone'] = filterArg.zone;
}
const dnsEntries = await this.getInstances(filter);
return dnsEntries;
}
public static async getDnsZones() {
const dnsEntries = await this.getInstances({});
const zones = new Set<string>();
for (const entry of dnsEntries) {
if (entry.data.zone) {
zones.add(entry.data.zone);
}
}
return Array.from(zones).sort();
}
public static async createDnsEntry(dnsEntryDataArg: plugins.servezoneInterfaces.data.IDnsEntry['data']) {
const dnsEntry = new DnsEntry();
dnsEntry.id = await DnsEntry.getNewId();
dnsEntry.data = {
...dnsEntryDataArg,
ttl: dnsEntryDataArg.ttl || 3600, // Default TTL: 1 hour
active: dnsEntryDataArg.active !== false, // Default to active
createdAt: Date.now(),
updatedAt: Date.now(),
};
await dnsEntry.save();
return dnsEntry;
}
public static async updateDnsEntry(
dnsEntryIdArg: string,
dnsEntryDataArg: Partial<plugins.servezoneInterfaces.data.IDnsEntry['data']>
) {
const dnsEntry = await this.getInstance({
id: dnsEntryIdArg,
});
if (!dnsEntry) {
throw new Error(`DNS entry with id ${dnsEntryIdArg} not found`);
}
Object.assign(dnsEntry.data, dnsEntryDataArg, {
updatedAt: Date.now(),
});
await dnsEntry.save();
return dnsEntry;
}
public static async deleteDnsEntry(dnsEntryIdArg: string) {
const dnsEntry = await this.getInstance({
id: dnsEntryIdArg,
});
if (!dnsEntry) {
throw new Error(`DNS entry with id ${dnsEntryIdArg} not found`);
}
await dnsEntry.delete();
return true;
}
// INSTANCE
@plugins.smartdata.svDb()
public id!: string;
@plugins.smartdata.svDb()
public data!: plugins.servezoneInterfaces.data.IDnsEntry['data'];
/**
* Validates the DNS entry data
*/
public validateData(): boolean {
const { type, name, value, zone } = this.data;
// Basic validation
if (!type || !name || !value || !zone) {
return false;
}
// Type-specific validation
switch (type) {
case 'A':
// Validate IPv4 address
return /^(\d{1,3}\.){3}\d{1,3}$/.test(value);
case 'AAAA':
// Validate IPv6 address (simplified)
return /^([0-9a-fA-F]{0,4}:){7}[0-9a-fA-F]{0,4}$/.test(value);
case 'MX':
// MX records must have priority
return this.data.priority !== undefined && this.data.priority >= 0;
case 'SRV':
// SRV records must have priority, weight, and port
return (
this.data.priority !== undefined &&
this.data.weight !== undefined &&
this.data.port !== undefined
);
case 'CNAME':
case 'NS':
case 'PTR':
// These should point to valid domain names
return /^[a-zA-Z0-9.-]+$/.test(value);
case 'TXT':
case 'CAA':
case 'SOA':
// These can contain any text
return true;
default:
return false;
}
}
/**
* Get a formatted string representation of the DNS entry
*/
public toFormattedString(): string {
const { type, name, value, ttl, priority } = this.data;
let result = `${name} ${ttl} IN ${type}`;
if (priority !== undefined) {
result += ` ${priority}`;
}
if (type === 'SRV' && this.data.weight !== undefined && this.data.port !== undefined) {
result += ` ${this.data.weight} ${this.data.port}`;
}
result += ` ${value}`;
return result;
}
}
+267
View File
@@ -0,0 +1,267 @@
import type { Cloudly } from '../classes.cloudly.js';
import * as plugins from '../plugins.js';
import { DnsEntry } from './classes.dnsentry.js';
export class DnsManager {
public typedrouter = new plugins.typedrequest.TypedRouter();
public cloudlyRef: Cloudly;
get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
}
public CDnsEntry = plugins.smartdata.setDefaultManagerForDoc(this, DnsEntry);
constructor(cloudlyRef: Cloudly) {
this.cloudlyRef = cloudlyRef;
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
// Get all DNS entries
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.dns.IRequest_Any_Cloudly_GetDnsEntries>(
'getDnsEntries',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const dnsEntries = await this.CDnsEntry.getDnsEntries(
reqArg.zone ? { zone: reqArg.zone } : undefined
);
return {
dnsEntries: await Promise.all(
dnsEntries.map((entry) => entry.createSavableObject())
),
};
}
)
);
// Get DNS entry by ID
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.dns.IRequest_Any_Cloudly_GetDnsEntryById>(
'getDnsEntryById',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const dnsEntry = await this.CDnsEntry.getDnsEntryById(reqArg.dnsEntryId);
if (!dnsEntry) {
throw new Error(`DNS entry with id ${reqArg.dnsEntryId} not found`);
}
return {
dnsEntry: await dnsEntry.createSavableObject(),
};
}
)
);
// Create DNS entry
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.dns.IRequest_Any_Cloudly_CreateDnsEntry>(
'createDnsEntry',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
// Validate domain exists and is activated if domainId is provided
if (reqArg.dnsEntryData.domainId) {
const domain = await this.cloudlyRef.domainManager.CDomain.getDomainById(reqArg.dnsEntryData.domainId);
if (!domain) {
throw new Error(`Domain with id ${reqArg.dnsEntryData.domainId} not found`);
}
if ((domain.data as any).activationState !== 'activated') {
throw new Error(`Domain ${domain.data.name} is not activated; DNS changes are not allowed.`);
}
// Set the zone from the domain name
reqArg.dnsEntryData.zone = domain.data.name;
}
const dnsEntry = await this.CDnsEntry.createDnsEntry(reqArg.dnsEntryData);
return {
dnsEntry: await dnsEntry.createSavableObject(),
};
}
)
);
// Update DNS entry
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.dns.IRequest_Any_Cloudly_UpdateDnsEntry>(
'updateDnsEntry',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
// Validate domain exists and is activated if domainId is provided
if (reqArg.dnsEntryData.domainId) {
const domain = await this.cloudlyRef.domainManager.CDomain.getDomainById(reqArg.dnsEntryData.domainId);
if (!domain) {
throw new Error(`Domain with id ${reqArg.dnsEntryData.domainId} not found`);
}
if ((domain.data as any).activationState !== 'activated') {
throw new Error(`Domain ${domain.data.name} is not activated; DNS changes are not allowed.`);
}
// Set the zone from the domain name
reqArg.dnsEntryData.zone = domain.data.name;
}
const dnsEntry = await this.CDnsEntry.updateDnsEntry(
reqArg.dnsEntryId,
reqArg.dnsEntryData
);
return {
dnsEntry: await dnsEntry.createSavableObject(),
};
}
)
);
// Delete DNS entry
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.dns.IRequest_Any_Cloudly_DeleteDnsEntry>(
'deleteDnsEntry',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const success = await this.CDnsEntry.deleteDnsEntry(reqArg.dnsEntryId);
return {
success,
};
}
)
);
// Get DNS zones
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.dns.IRequest_Any_Cloudly_GetDnsZones>(
'getDnsZones',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const zones = await this.CDnsEntry.getDnsZones();
return {
zones,
};
}
)
);
}
/**
* Create a DNS entry for a service
* @param dnsEntryData The DNS entry data
*/
public async createServiceDnsEntry(dnsEntryData: plugins.servezoneInterfaces.data.IDnsEntry['data']) {
// If domainId is provided, get the domain and set the zone
if (dnsEntryData.domainId) {
const domain = await this.cloudlyRef.domainManager.CDomain.getInstance({
id: dnsEntryData.domainId,
});
if (domain) {
dnsEntryData.zone = domain.data.name;
}
}
// Create the DNS entry
const dnsEntry = await this.CDnsEntry.createDnsEntry(dnsEntryData);
return dnsEntry;
}
/**
* Activate DNS entries for a service when it's deployed
* @param serviceId The service ID
*/
public async activateServiceDnsEntries(serviceId: string) {
const dnsEntries = await this.CDnsEntry.getInstances({
'data.sourceServiceId': serviceId,
'data.sourceType': 'service',
});
for (const entry of dnsEntries) {
entry.data.active = true;
entry.data.updatedAt = Date.now();
await entry.save();
}
}
/**
* Deactivate DNS entries for a service when it's undeployed
* @param serviceId The service ID
*/
public async deactivateServiceDnsEntries(serviceId: string) {
const dnsEntries = await this.CDnsEntry.getInstances({
'data.sourceServiceId': serviceId,
'data.sourceType': 'service',
});
for (const entry of dnsEntries) {
entry.data.active = false;
entry.data.updatedAt = Date.now();
await entry.save();
}
}
/**
* Remove all DNS entries for a service
* @param serviceId The service ID
*/
public async removeServiceDnsEntries(serviceId: string) {
const dnsEntries = await this.CDnsEntry.getInstances({
'data.sourceServiceId': serviceId,
'data.sourceType': 'service',
});
for (const entry of dnsEntries) {
await entry.delete();
}
}
/**
* Update DNS entry values when deployment happens
* @param serviceId The service ID
* @param ipAddress The IP address to set for the DNS entries
*/
public async updateServiceDnsEntriesIp(serviceId: string, ipAddress: string) {
const dnsEntries = await this.CDnsEntry.getInstances({
'data.sourceServiceId': serviceId,
'data.sourceType': 'service',
});
for (const entry of dnsEntries) {
if (entry.data.type === 'A' || entry.data.type === 'AAAA') {
entry.data.value = ipAddress;
entry.data.updatedAt = Date.now();
await entry.save();
}
}
}
/**
* Initialize the DNS manager
*/
public async init() {
console.log('DNS Manager initialized');
}
/**
* Stop the DNS manager
*/
public async stop() {
console.log('DNS Manager stopped');
}
}
+211
View File
@@ -0,0 +1,211 @@
import * as plugins from '../plugins.js';
import { DomainManager } from './classes.domainmanager.js';
@plugins.smartdata.managed()
export class Domain extends plugins.smartdata.SmartDataDbDoc<
Domain,
plugins.servezoneInterfaces.data.IDomain,
DomainManager
> {
// STATIC
public static async getDomainById(domainIdArg: string) {
const domain = await this.getInstance({
id: domainIdArg,
});
return domain;
}
public static async getDomainByName(domainNameArg: string) {
const domain = await this.getInstance({
'data.name': domainNameArg,
});
return domain;
}
public static async getDomains() {
const domains = await this.getInstances({});
return domains;
}
public static async createDomain(domainDataArg: plugins.servezoneInterfaces.data.IDomain['data']) {
const domain = new Domain();
domain.id = await Domain.getNewId();
domain.data = {
...domainDataArg,
status: domainDataArg.status || 'pending',
verificationStatus: domainDataArg.verificationStatus || 'pending',
nameservers: domainDataArg.nameservers || [],
autoRenew: domainDataArg.autoRenew !== false,
activationState: domainDataArg.activationState || 'available',
syncSource: domainDataArg.syncSource ?? null,
lastSyncAt: domainDataArg.lastSyncAt,
createdAt: Date.now(),
updatedAt: Date.now(),
};
await domain.save();
return domain;
}
public static async updateDomain(
domainIdArg: string,
domainDataArg: Partial<plugins.servezoneInterfaces.data.IDomain['data']>
) {
const domain = await this.getInstance({
id: domainIdArg,
});
if (!domain) {
throw new Error(`Domain with id ${domainIdArg} not found`);
}
// Merge updates and respect incoming activationState when provided
Object.assign(domain.data, domainDataArg);
domain.data.updatedAt = Date.now();
// Ensure activationState has a sensible default if still missing
if (!domain.data.activationState) {
(domain.data as any).activationState = 'available';
}
await domain.save();
return domain;
}
public static async deleteDomain(domainIdArg: string) {
const domain = await this.getInstance({
id: domainIdArg,
});
if (!domain) {
throw new Error(`Domain with id ${domainIdArg} not found`);
}
// Check if there are DNS entries for this domain
const dnsManager = domain.manager.cloudlyRef.dnsManager;
const dnsEntries = await dnsManager.CDnsEntry.getInstances({
'data.zone': domain.data.name,
});
if (dnsEntries.length > 0) {
console.log(`Warning: Deleting domain ${domain.data.name} with ${dnsEntries.length} DNS entries`);
// Optionally delete associated DNS entries
for (const dnsEntry of dnsEntries) {
await dnsEntry.delete();
}
}
await domain.delete();
return true;
}
// INSTANCE
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public data!: plugins.servezoneInterfaces.data.IDomain['data'];
/**
* Verify domain ownership
*/
public async verifyDomain(methodArg?: 'dns' | 'http' | 'email' | 'manual') {
const method = methodArg || this.data.verificationMethod || 'dns';
// Generate verification token if not exists
if (!this.data.verificationToken) {
this.data.verificationToken = plugins.smartunique.shortId();
await this.save();
}
let verificationResult = {
success: false,
message: '',
details: {} as any,
};
switch (method) {
case 'dns':
// Check for TXT record with verification token
verificationResult = await this.verifyViaDns();
break;
case 'http':
// Check for file at well-known URL
verificationResult = await this.verifyViaHttp();
break;
case 'email':
// Send verification email
verificationResult = await this.verifyViaEmail();
break;
case 'manual':
// Manual verification
verificationResult.success = true;
verificationResult.message = 'Manually verified';
break;
}
// Update verification status
if (verificationResult.success) {
this.data.verificationStatus = 'verified';
this.data.lastVerificationAt = Date.now();
this.data.verificationMethod = method;
} else {
this.data.verificationStatus = 'failed';
this.data.lastVerificationAt = Date.now();
}
await this.save();
return verificationResult;
}
private async verifyViaDns(): Promise<{ success: boolean; message: string; details: any }> {
// TODO: Implement DNS verification
// Look for TXT record _cloudly-verify.{domain} with value {verificationToken}
return {
success: false,
message: 'DNS verification not yet implemented',
details: {
expectedRecord: `_cloudly-verify.${this.data.name}`,
expectedValue: this.data.verificationToken,
},
};
}
private async verifyViaHttp(): Promise<{ success: boolean; message: string; details: any }> {
// TODO: Implement HTTP verification
// Check for file at http://{domain}/.well-known/cloudly-verify.txt
return {
success: false,
message: 'HTTP verification not yet implemented',
details: {
expectedUrl: `http://${this.data.name}/.well-known/cloudly-verify.txt`,
expectedContent: this.data.verificationToken,
},
};
}
private async verifyViaEmail(): Promise<{ success: boolean; message: string; details: any }> {
// TODO: Implement email verification
return {
success: false,
message: 'Email verification not yet implemented',
details: {},
};
}
/**
* Check if domain is expiring soon
*/
public isExpiringSoon(daysThreshold: number = 30): boolean {
if (!this.data.expiresAt) {
return false;
}
const daysUntilExpiry = (this.data.expiresAt - Date.now()) / (1000 * 60 * 60 * 24);
return daysUntilExpiry <= daysThreshold;
}
/**
* Get all DNS entries for this domain
*/
public async getDnsEntries() {
const dnsManager = this.manager.cloudlyRef.dnsManager;
const dnsEntries = await dnsManager.CDnsEntry.getInstances({
'data.zone': this.data.name,
});
return dnsEntries;
}
}
+258
View File
@@ -0,0 +1,258 @@
import type { Cloudly } from '../classes.cloudly.js';
import * as plugins from '../plugins.js';
import { Domain } from './classes.domain.js';
interface IWorkHosterDomain {
name: string;
nameservers?: string[];
capabilities?: {
canCreateSubdomains: boolean;
canManageDnsRecords: boolean;
canIssueCertificates: boolean;
canHostEmail: boolean;
};
}
export class DomainManager {
public typedrouter = new plugins.typedrequest.TypedRouter();
public cloudlyRef: Cloudly;
get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
}
public CDomain = plugins.smartdata.setDefaultManagerForDoc(this, Domain);
constructor(cloudlyRef: Cloudly) {
this.cloudlyRef = cloudlyRef;
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
// Get all domains
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.domain.IRequest_Any_Cloudly_GetDomains>(
'getDomains',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const domains = await this.CDomain.getDomains();
return {
domains: await Promise.all(
domains.map((domain) => domain.createSavableObject())
),
};
}
)
);
// Get domain by ID
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.domain.IRequest_Any_Cloudly_GetDomainById>(
'getDomainById',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const domain = await this.CDomain.getDomainById(reqArg.domainId);
if (!domain) {
throw new Error(`Domain with id ${reqArg.domainId} not found`);
}
return {
domain: await domain.createSavableObject(),
};
}
)
);
// Create domain
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.domain.IRequest_Any_Cloudly_CreateDomain>(
'createDomain',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
// Check if domain already exists
const existingDomain = await this.CDomain.getDomainByName(reqArg.domainData.name);
if (existingDomain) {
throw new Error(`Domain ${reqArg.domainData.name} already exists`);
}
const domain = await this.CDomain.createDomain(reqArg.domainData);
return {
domain: await domain.createSavableObject(),
};
}
)
);
// Update domain
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.domain.IRequest_Any_Cloudly_UpdateDomain>(
'updateDomain',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const domain = await this.CDomain.updateDomain(
reqArg.domainId,
reqArg.domainData
);
return {
domain: await domain.createSavableObject(),
};
}
)
);
// Delete domain
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.domain.IRequest_Any_Cloudly_DeleteDomain>(
'deleteDomain',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const success = await this.CDomain.deleteDomain(reqArg.domainId);
return {
success,
};
}
)
);
// Verify domain
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.domain.IRequest_Any_Cloudly_VerifyDomain>(
'verifyDomain',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const domain = await this.CDomain.getDomainById(reqArg.domainId);
if (!domain) {
throw new Error(`Domain with id ${reqArg.domainId} not found`);
}
const verificationResult = await domain.verifyDomain(reqArg.verificationMethod);
return {
domain: await domain.createSavableObject(),
verificationResult,
};
}
)
);
}
/**
* Initialize the domain manager
*/
public async init() {
await this.syncExternalGatewayDomains().catch((error) => {
console.log(`External gateway domain sync failed: ${(error as Error).message}`);
});
console.log('Domain Manager initialized');
}
public async syncExternalGatewayDomains(): Promise<number> {
const settings = await this.cloudlyRef.settingsManager.getSettings();
if (!settings.dcrouterGatewayUrl || !settings.dcrouterGatewayApiToken) {
return 0;
}
const typedRequest = new plugins.typedrequest.TypedRequest<any>(
`${settings.dcrouterGatewayUrl.replace(/\/+$/, '')}/typedrequest`,
'getWorkHosterDomains',
);
const response = await typedRequest.fire({
apiToken: settings.dcrouterGatewayApiToken,
}) as { domains: IWorkHosterDomain[] };
const activeDomainNames = new Set<string>();
for (const gatewayDomain of response.domains) {
const domainName = gatewayDomain.name.trim().toLowerCase();
if (!domainName) continue;
activeDomainNames.add(domainName);
const existingDomain = await this.CDomain.getDomainByName(domainName);
const tags = Array.from(new Set([...(existingDomain?.data.tags || []), 'dcrouter']));
const domainData: Partial<plugins.servezoneInterfaces.data.IDomain['data']> = {
name: domainName,
status: 'active',
verificationStatus: 'not_required',
nameservers: gatewayDomain.nameservers || existingDomain?.data.nameservers || [],
autoRenew: gatewayDomain.capabilities?.canIssueCertificates !== false,
activationState: 'available',
syncSource: 'manual',
lastSyncAt: Date.now(),
isExternal: true,
tags,
};
if (existingDomain) {
await this.CDomain.updateDomain(existingDomain.id, domainData);
} else {
await this.CDomain.createDomain(domainData as plugins.servezoneInterfaces.data.IDomain['data']);
}
}
const knownDomains = await this.CDomain.getDomains();
for (const domain of knownDomains) {
if (domain.data.tags?.includes('dcrouter') && !activeDomainNames.has(domain.data.name)) {
await this.CDomain.updateDomain(domain.id, {
activationState: 'ignored',
lastSyncAt: Date.now(),
});
}
}
console.log(`Synced ${activeDomainNames.size} domain(s) from external dcrouter gateway`);
return activeDomainNames.size;
}
/**
* Stop the domain manager
*/
public async stop() {
console.log('Domain Manager stopped');
}
/**
* Get all active domains
*/
public async getActiveDomains() {
const domains = await this.CDomain.getInstances({
'data.status': 'active',
});
return domains;
}
/**
* Get domains that are expiring soon
*/
public async getExpiringDomains(daysThreshold: number = 30) {
const domains = await this.CDomain.getDomains();
return domains.filter(domain => domain.isExpiringSoon(daysThreshold));
}
/**
* Check if a domain name is available (not in our system)
*/
public async isDomainAvailable(domainName: string): Promise<boolean> {
const existingDomain = await this.CDomain.getDomainByName(domainName);
return !existingDomain;
}
}
@@ -1,8 +1,9 @@
import * as plugins from '../plugins.js';
import * as paths from '../paths.js';
import type { Cloudly } from 'ts/classes.cloudly.js';
import type { Cloudly } from '../classes.cloudly.js';
import type { ExternalRegistryManager } from './classes.externalregistrymanager.js';
@plugins.smartdata.managed()
export class ExternalRegistry extends plugins.smartdata.SmartDataDbDoc<ExternalRegistry, plugins.servezoneInterfaces.data.IExternalRegistry, ExternalRegistryManager> {
// STATIC
public static async getRegistryById(registryIdArg: string) {
@@ -17,24 +18,190 @@ export class ExternalRegistry extends plugins.smartdata.SmartDataDbDoc<ExternalR
return externalRegistries;
}
public static async getDefaultRegistry(type: 'docker' | 'npm' = 'docker') {
const defaultRegistry = await this.getInstance({
'data.type': type,
'data.isDefault': true,
});
return defaultRegistry;
}
public static async createExternalRegistry(registryDataArg: Partial<plugins.servezoneInterfaces.data.IExternalRegistry['data']>) {
const externalRegistry = new ExternalRegistry();
externalRegistry.id = await ExternalRegistry.getNewId();
Object.assign(externalRegistry, registryDataArg);
externalRegistry.id = await this.getNewId();
externalRegistry.data = {
type: registryDataArg.type || 'docker',
name: registryDataArg.name || '',
url: registryDataArg.url || '',
username: registryDataArg.username,
password: registryDataArg.password,
description: registryDataArg.description,
isDefault: registryDataArg.isDefault || false,
authType: registryDataArg.authType || (registryDataArg.username || registryDataArg.password ? 'basic' : 'none'),
insecure: registryDataArg.insecure || false,
namespace: registryDataArg.namespace,
proxy: registryDataArg.proxy,
config: registryDataArg.config,
status: 'unverified',
createdAt: Date.now(),
updatedAt: Date.now(),
};
// If this is set as default, unset other defaults of the same type
if (externalRegistry.data.isDefault) {
const existingDefaults = await this.getInstances({
'data.type': externalRegistry.data.type,
'data.isDefault': true,
});
for (const existingDefault of existingDefaults) {
existingDefault.data.isDefault = false;
await existingDefault.save();
}
}
await externalRegistry.save();
return externalRegistry;
}
public static async updateExternalRegistry(
registryIdArg: string,
registryDataArg: Partial<plugins.servezoneInterfaces.data.IExternalRegistry['data']>
) {
const externalRegistry = await this.getRegistryById(registryIdArg);
if (!externalRegistry) {
throw new Error(`Registry with id ${registryIdArg} not found`);
}
// If setting as default, unset other defaults of the same type
if (registryDataArg.isDefault && !externalRegistry.data.isDefault) {
const existingDefaults = await this.getInstances({
'data.type': externalRegistry.data.type,
'data.isDefault': true,
});
for (const existingDefault of existingDefaults) {
if (existingDefault.id !== registryIdArg) {
existingDefault.data.isDefault = false;
await existingDefault.save();
}
}
}
// Update fields
Object.assign(externalRegistry.data, registryDataArg, {
updatedAt: Date.now(),
});
await externalRegistry.save();
return externalRegistry;
}
public static async deleteExternalRegistry(registryIdArg: string) {
const externalRegistry = await this.getRegistryById(registryIdArg);
if (!externalRegistry) {
return false;
}
await externalRegistry.delete();
return true;
}
// INSTANCE
@plugins.smartdata.svDb()
public id: string;
public id!: string;
@plugins.smartdata.svDb()
public data: plugins.servezoneInterfaces.data.IExternalRegistry['data'];
public data!: plugins.servezoneInterfaces.data.IExternalRegistry['data'];
constructor() {
super();
}
/**
* Verify the registry connection
*/
public async verifyConnection(): Promise<{ success: boolean; message?: string }> {
try {
// For Docker registries, try to access the v2 API
if (this.data.type === 'docker') {
const registryUrl = this.data.url.replace(/\/$/, ''); // Remove trailing slash
// Build headers based on auth type
const headers: any = {};
if (this.data.authType === 'basic' && this.data.username && this.data.password) {
headers['Authorization'] = 'Basic ' + Buffer.from(`${this.data.username}:${this.data.password}`).toString('base64');
} else if (this.data.authType === 'token' && this.data.password) {
// For token auth, password field contains the token
headers['Authorization'] = `Bearer ${this.data.password}`;
}
// For 'none' auth type or missing credentials, no auth header is added
// Try to access the Docker Registry v2 API
const response = await fetch(`${registryUrl}/v2/`, {
headers,
// Allow insecure if configured
...(this.data.insecure ? { rejectUnauthorized: false } : {}),
}).catch(err => {
throw new Error(`Failed to connect: ${err.message}`);
});
if (response.status === 200) {
// 200 means successful (either public or authenticated)
this.data.status = 'active';
this.data.lastVerified = Date.now();
this.data.lastError = undefined;
await this.save();
return { success: true, message: 'Registry connection successful' };
} else if (response.status === 401 && this.data.authType === 'none') {
// 401 with no auth means registry exists but needs auth
throw new Error('Registry requires authentication');
} else if (response.status === 401) {
throw new Error('Authentication failed - check credentials');
} else {
throw new Error(`Registry returned status ${response.status}`);
}
}
// For npm registries, implement npm-specific verification
if (this.data.type === 'npm') {
// TODO: Implement npm registry verification
this.data.status = 'unverified';
return { success: false, message: 'NPM registry verification not yet implemented' };
}
return { success: false, message: 'Unknown registry type' };
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
this.data.status = 'error';
this.data.lastError = errorMessage;
await this.save();
return { success: false, message: errorMessage };
}
}
/**
* Get the full registry URL with namespace if applicable
*/
public getFullRegistryUrl(): string {
let url = this.data.url.replace(/\/$/, ''); // Remove trailing slash
if (this.data.namespace) {
url = `${url}/${this.data.namespace}`;
}
return url;
}
/**
* Get Docker auth config for this registry
*/
public getDockerAuthConfig() {
if (this.data.type !== 'docker') {
return null;
}
return {
username: this.data.username,
password: this.data.password,
email: this.data.config?.dockerConfig?.email,
serveraddress: this.data.config?.dockerConfig?.serverAddress || this.data.url,
};
}
}
@@ -13,24 +13,35 @@ export class ExternalRegistryManager {
constructor(cloudlyRef: Cloudly) {
this.cloudlyRef = cloudlyRef;
}
public async start() {
// lets set up a typedrouter
this.typedrouter.addTypedRouter(this.typedrouter);
// Add typedrouter to cloudly's main router
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
// Get registry by ID
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.externalRegistry.IReq_GetRegistryById>(
new plugins.typedrequest.TypedHandler('getExternalRegistryById', async (dataArg) => {
const registry = await ExternalRegistry.getRegistryById(dataArg.id);
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const registry = await this.CExternalRegistry.getRegistryById(dataArg.id);
if (!registry) {
throw new Error(`Registry with id ${dataArg.id} not found`);
}
return {
registry: await registry.createSavableObject(),
};
})
);
// Get all registries
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.externalRegistry.IReq_GetRegistries>(
new plugins.typedrequest.TypedHandler('getExternalRegistries', async (dataArg) => {
const registries = await ExternalRegistry.getRegistries();
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const registries = await this.CExternalRegistry.getRegistries();
return {
registries: await Promise.all(
registries.map((registry) => registry.createSavableObject())
@@ -39,13 +50,81 @@ export class ExternalRegistryManager {
})
);
// Create registry
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.externalRegistry.IReq_CreateRegistry>(
new plugins.typedrequest.TypedHandler('createExternalRegistry', async (dataArg) => {
const registry = await ExternalRegistry.createExternalRegistry(dataArg.registryData);
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const registry = await this.CExternalRegistry.createExternalRegistry(dataArg.registryData);
return {
registry: await registry.createSavableObject(),
};
})
);
// Update registry
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.externalRegistry.IReq_UpdateRegistry>(
new plugins.typedrequest.TypedHandler('updateExternalRegistry', async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const registry = await this.CExternalRegistry.updateExternalRegistry(
dataArg.registryId,
dataArg.registryData
);
return {
resultRegistry: await registry.createSavableObject(),
};
})
);
// Delete registry
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.externalRegistry.IReq_DeleteRegistryById>(
new plugins.typedrequest.TypedHandler('deleteExternalRegistryById', async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const success = await this.CExternalRegistry.deleteExternalRegistry(dataArg.registryId);
return {
ok: success,
};
})
);
// Verify registry connection
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.externalRegistry.IReq_VerifyRegistry>(
new plugins.typedrequest.TypedHandler('verifyExternalRegistry', async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const registry = await this.CExternalRegistry.getRegistryById(dataArg.registryId);
if (!registry) {
return {
success: false,
message: `Registry with id ${dataArg.registryId} not found`,
};
}
const result = await registry.verifyConnection();
return {
success: result.success,
message: result.message,
registry: await registry.createSavableObject(),
};
})
);
}
public async start() {
console.log('External Registry Manager started');
}
public async stop() {
console.log('External Registry Manager stopped');
}
}
+2 -2
View File
@@ -26,10 +26,10 @@ export class Image extends plugins.smartdata.SmartDataDbDoc<
}
@plugins.smartdata.unI()
public id: string;
public id!: string;
@plugins.smartdata.svDb()
public data: plugins.servezoneInterfaces.data.IImage['data'];
public data!: plugins.servezoneInterfaces.data.IImage['data'];
public async getVersions() {}
+45 -17
View File
@@ -7,9 +7,9 @@ import { Image } from './classes.image.js';
export class ImageManager {
cloudlyRef: Cloudly;
public typedrouter = new plugins.typedrequest.TypedRouter();
public smartbucketInstance: plugins.smartbucket.SmartBucket;
public imageDir: plugins.smartbucket.Directory;
public dockerImageStore: plugins.docker.DockerImageStore;
public smartbucketInstance!: plugins.smartbucket.SmartBucket;
public imageDir!: plugins.smartbucket.Directory;
public dockerImageStore!: plugins.docker.DockerImageStore;
get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
@@ -26,7 +26,7 @@ export class ImageManager {
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.image.IRequest_CreateImage>(
'createImage',
async (reqArg, toolsArg) => {
await toolsArg.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], reqArg);
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], reqArg);
const image = await this.CImage.create({
name: reqArg.name,
description: reqArg.description,
@@ -41,7 +41,7 @@ export class ImageManager {
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.image.IRequest_GetImage>(
new plugins.typedrequest.TypedHandler('getImage', async (reqArg, toolsArg) => {
await toolsArg.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], reqArg);
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminOrClusterIdentityGuard], reqArg);
const image = await this.CImage.getInstance({
id: reqArg.imageId,
});
@@ -55,7 +55,7 @@ export class ImageManager {
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.image.IRequest_DeleteImage>(
'deleteImage',
async (reqArg, toolsArg) => {
await toolsArg.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], reqArg);
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], reqArg);
const image = await this.CImage.getInstance({
id: reqArg.imageId,
});
@@ -69,7 +69,7 @@ export class ImageManager {
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.image.IRequest_GetAllImages>(
'getAllImages',
async (requestArg, toolsArg) => {
await toolsArg.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], requestArg);
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], requestArg);
const images = await this.CImage.getInstances({});
return {
images: await Promise.all(
@@ -96,9 +96,24 @@ export class ImageManager {
throw new plugins.typedrequest.TypedResponseError('Image not found');
}
const imageVersion = reqArg.versionString;
if (!imageVersion) {
throw new plugins.typedrequest.TypedResponseError('versionString is required');
}
console.log(
`got request to push image version ${imageVersion} for image ${refImage.data.name}`,
);
const storagePath = await refImage.getStoragePath(imageVersion);
refImage.data.versions = [
...refImage.data.versions.filter((version) => version.versionString !== imageVersion),
{
versionString: imageVersion,
source: 'upload',
storagePath,
size: 0,
createdAt: Date.now(),
},
];
await refImage.save();
const imagePushStream = reqArg.imageStream;
(async () => {
const smartWebDuplex = new plugins.smartstream.webstream.WebDuplexStream<
@@ -112,10 +127,12 @@ export class ImageManager {
});
imagePushStream.writeToWebstream(smartWebDuplex.writable);
await this.dockerImageStore.storeImage(
refImage.id,
storagePath,
plugins.smartstream.SmartDuplex.fromWebReadableStream(smartWebDuplex.readable),
);
})();
})().catch((error) => {
console.error(`failed to store image ${refImage.id}:${imageVersion}`, error);
});
return {
allowed: true,
};
@@ -133,13 +150,21 @@ export class ImageManager {
const imageVersion = image.data.versions.find(
(version) => version.versionString === reqArg.versionString,
);
const readable = this.imageDir.fastGetStream(
if (!imageVersion) {
throw new plugins.typedrequest.TypedResponseError('Image version not found');
}
const readable = await this.imageDir.fastGetStream(
{
path: await image.getStoragePath(reqArg.versionString),
path: imageVersion.storagePath || await image.getStoragePath(reqArg.versionString),
},
'webstream',
);
const imageVirtualStream = new plugins.typedrequest.VirtualStream();
(async () => {
await imageVirtualStream.readFromWebstream(readable);
})().catch((error) => {
console.error(`failed to stream image ${image.id}:${reqArg.versionString}`, error);
});
return {
imageStream: imageVirtualStream,
};
@@ -150,21 +175,24 @@ export class ImageManager {
public async start() {
// lets setup s3
const s3Descriptor: plugins.tsclass.storage.IS3Descriptor =
await this.cloudlyRef.config.appData.waitForAndGetKey('s3Descriptor');
const s3Descriptor =
await this.cloudlyRef.config.appData.waitForAndGetKey('s3Descriptor') as plugins.tsclass.storage.IS3Descriptor;
console.log(this.cloudlyRef.config.data.s3Descriptor);
this.smartbucketInstance = new plugins.smartbucket.SmartBucket(
this.cloudlyRef.config.data.s3Descriptor,
this.cloudlyRef.config.data.s3Descriptor!,
);
const bucket = await this.smartbucketInstance.getBucketByName(s3Descriptor.bucketName);
await bucket.fastPut({ path: 'images/00init', contents: 'init' });
const bucketName = s3Descriptor.bucketName!;
const bucket = await this.smartbucketInstance.bucketExists(bucketName)
? await this.smartbucketInstance.getBucketByName(bucketName)
: await this.smartbucketInstance.createBucket(bucketName);
await bucket.fastPut({ path: 'images/00init', contents: 'init', overwrite: true });
this.imageDir = await bucket.getDirectoryFromPath({
path: '/images',
});
// lets setup dockerstore
await plugins.smartfile.fs.ensureDir(paths.dockerImageStoreDir);
await plugins.fsPromises.mkdir(paths.dockerImageStoreDir, { recursive: true });
this.dockerImageStore = new plugins.docker.DockerImageStore({
localDirPath: paths.dockerImageStoreDir,
bucketDir: this.imageDir,
+43
View File
@@ -0,0 +1,43 @@
import * as plugins from '../plugins.js';
export interface IJumpCodeData {
clusterId: string;
createdBy: string;
role: plugins.servezoneInterfaces.data.IClusterNode['data']['role'];
nodeType: plugins.servezoneInterfaces.data.IClusterNode['data']['nodeType'];
createdAt: number;
expiresAt: number;
consumedAt?: number;
consumedByNodeId?: string;
}
export interface IJumpCodePublic {
id: string;
data: IJumpCodeData;
}
@plugins.smartdata.Manager()
export class JumpCode extends plugins.smartdata.SmartDataDbDoc<JumpCode, IJumpCodePublic> {
constructor(optionsArg?: IJumpCodePublic & { tokenHash?: string }) {
super();
if (optionsArg) {
Object.assign(this, optionsArg);
}
}
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public tokenHash!: string;
@plugins.smartdata.svDb()
public data!: IJumpCodeData;
public toPublicObject(): IJumpCodePublic {
return {
id: this.id,
data: this.data,
};
}
}
+379
View File
@@ -0,0 +1,379 @@
import * as plugins from '../plugins.js';
import type { Cloudly } from '../classes.cloudly.js';
import { logger } from '../logger.js';
import { JumpCode } from './classes.jumpcode.js';
type IReqCreateNodeJumpCommand = plugins.servezoneInterfaces.requests.node.IReq_Any_Cloudly_CreateNodeJumpCommand['request'];
type IResCreateNodeJumpCommand = plugins.servezoneInterfaces.requests.node.IReq_Any_Cloudly_CreateNodeJumpCommand['response'];
interface IClaimJumpCodeRequest {
jumpCode?: string;
hostname?: string;
}
interface IClaimJumpCodeResponse {
accepted: boolean;
message?: string;
nodeId?: string;
sparkNodeToken?: string;
cloudlyUrl?: string;
coreflowJumpCode?: string;
}
export class CloudlyJumpManager {
public cloudlyRef: Cloudly;
public typedRouter = new plugins.typedrequest.TypedRouter();
public CJumpCode = plugins.smartdata.setDefaultManagerForDoc(this, JumpCode);
public get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
}
private defaultTtlMs = 1000 * 60 * 30;
private maxTtlMs = 1000 * 60 * 60 * 24;
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
this.cloudlyRef.typedrouter.addTypedRouter(this.typedRouter);
this.typedRouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.node.IReq_Any_Cloudly_CreateNodeJumpCommand>('createNodeJumpCommand', async (requestDataArg) => {
await plugins.smartguard.passGuardsOrReject(
{ identity: requestDataArg.identity },
[this.cloudlyRef.authManager.adminIdentityGuard],
);
return await this.createNodeJumpCommand(requestDataArg);
}),
);
}
public async start() {
logger.log('info', 'Jump manager started');
}
public async stop() {
logger.log('info', 'Jump manager stopped');
}
public async createNodeJumpCommand(optionsArg: IReqCreateNodeJumpCommand): Promise<IResCreateNodeJumpCommand> {
const cluster = await this.cloudlyRef.clusterManager.CCluster.getInstance({
id: optionsArg.clusterId,
});
if (!cluster) {
throw new plugins.typedrequest.TypedResponseError(`Cluster ${optionsArg.clusterId} not found`);
}
const now = Date.now();
const ttlMs = this.normalizeTtl(optionsArg.ttlMs);
const jumpCode = this.createJumpCode();
const jumpCodeDoc = new this.CJumpCode({
id: await this.CJumpCode.getNewId(),
tokenHash: this.hashSecret(jumpCode),
data: {
clusterId: cluster.id,
createdBy: optionsArg.identity.userId,
role: optionsArg.role || 'worker',
nodeType: optionsArg.nodeType || 'baremetal',
createdAt: now,
expiresAt: now + ttlMs,
},
});
await jumpCodeDoc.save();
const jumpUrl = `${this.getPublicCloudlyUrl()}/jump/${encodeURIComponent(jumpCode)}`;
const setupUrl = `${jumpUrl}/setup.sh`;
return {
jumpCode,
jumpUrl,
setupUrl,
command: `curl -fsSL '${jumpUrl}' | sudo bash`,
expiresAt: jumpCodeDoc.data.expiresAt,
};
}
public async handleJumpHttpRequest(ctxArg: plugins.typedserver.IRequestContext): Promise<Response> {
const jumpCode = this.getCodeFromContext(ctxArg);
if (this.shouldRenderHtml(ctxArg)) {
return await this.createLandingPageResponse(jumpCode);
}
return await this.createSetupScriptResponse(jumpCode);
}
public async handleSetupScriptHttpRequest(ctxArg: plugins.typedserver.IRequestContext): Promise<Response> {
return await this.createSetupScriptResponse(this.getCodeFromContext(ctxArg));
}
public async handleClaimHttpRequest(ctxArg: plugins.typedserver.IRequestContext): Promise<Response> {
try {
const requestData = await this.readJsonBody<IClaimJumpCodeRequest>(ctxArg);
const response = await this.claimJumpCode(requestData);
return this.createJsonResponse(200, response);
} catch (error) {
return this.createJsonResponse(400, {
accepted: false,
message: (error as Error).message,
} satisfies IClaimJumpCodeResponse);
}
}
public async claimJumpCode(requestDataArg: IClaimJumpCodeRequest): Promise<IClaimJumpCodeResponse> {
if (!requestDataArg.jumpCode) {
throw new Error('Jump code is missing');
}
const jumpCodeDoc = await this.getJumpCodeByCode(requestDataArg.jumpCode);
if (!jumpCodeDoc) {
throw new Error('Jump code is invalid');
}
if (jumpCodeDoc.data.consumedAt) {
throw new Error('Jump code has already been used');
}
if (jumpCodeDoc.data.expiresAt <= Date.now()) {
throw new Error('Jump code has expired');
}
const cluster = await this.cloudlyRef.clusterManager.CCluster.getInstance({
id: jumpCodeDoc.data.clusterId,
});
if (!cluster) {
throw new Error('Jump code references a missing cluster');
}
const clusterUser = await this.cloudlyRef.authManager.CUser.getInstance({
id: cluster.data.userId,
});
const coreflowJumpCode = clusterUser?.data.tokens?.find((tokenArg) => tokenArg.expiresAt > Date.now())?.token;
if (!coreflowJumpCode) {
throw new Error('Cluster runtime token is missing or expired');
}
const nodeId = plugins.smartunique.shortId(8);
const now = Date.now();
const sparkNodeToken = await this.cloudlyRef.authManager.createNewSecureToken();
const node = new this.cloudlyRef.nodeManager.CClusterNode();
node.id = nodeId;
node.sparkNodeTokenHash = this.hashSecret(sparkNodeToken);
node.data = {
clusterId: cluster.id,
nodeType: jumpCodeDoc.data.nodeType,
status: 'initializing',
role: jumpCodeDoc.data.role,
joinedAt: now,
lastHealthCheck: now,
sshKeys: [],
requiredDebianPackages: [],
};
await node.save();
cluster.data.nodes = [
...(cluster.data.nodes || []).filter((nodeArg) => nodeArg.id !== node.id),
await node.createSavableObject(),
];
await cluster.save();
jumpCodeDoc.data = {
...jumpCodeDoc.data,
consumedAt: now,
consumedByNodeId: node.id,
};
await jumpCodeDoc.save();
return {
accepted: true,
nodeId: node.id,
sparkNodeToken,
cloudlyUrl: cluster.data.cloudlyUrl || `${this.getPublicCloudlyUrl()}/`,
coreflowJumpCode,
};
}
private async createLandingPageResponse(jumpCodeArg: string) {
const jumpCodeDoc = await this.getJumpCodeByCode(jumpCodeArg);
let clusterName = 'Unknown cluster';
let isUsable = false;
if (jumpCodeDoc && !jumpCodeDoc.data.consumedAt && jumpCodeDoc.data.expiresAt > Date.now()) {
const cluster = await this.cloudlyRef.clusterManager.CCluster.getInstance({
id: jumpCodeDoc.data.clusterId,
});
clusterName = cluster?.data.name || jumpCodeDoc.data.clusterId;
isUsable = true;
}
const jumpUrl = `${this.getPublicCloudlyUrl()}/jump/${encodeURIComponent(jumpCodeArg)}`;
const command = `curl -fsSL '${jumpUrl}' | sudo bash`;
const html = `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Cloudly Jump</title>
<style>
body { margin: 0; font-family: Inter, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #0d1117; color: #f0f6fc; }
main { max-width: 760px; margin: 10vh auto; padding: 32px; }
.card { border: 1px solid #30363d; border-radius: 18px; background: #161b22; padding: 28px; box-shadow: 0 24px 80px rgba(0, 0, 0, 0.35); }
.label { color: #8b949e; font-size: 13px; text-transform: uppercase; letter-spacing: 0.08em; }
h1 { margin: 8px 0 12px; font-size: 34px; }
p { color: #c9d1d9; line-height: 1.55; }
pre { white-space: pre-wrap; word-break: break-all; background: #0d1117; border: 1px solid #30363d; border-radius: 12px; padding: 16px; color: #7ee787; }
.status { display: inline-block; margin-top: 16px; padding: 6px 10px; border-radius: 999px; background: ${isUsable ? '#17391f' : '#3d1d1d'}; color: ${isUsable ? '#7ee787' : '#ff7b72'}; }
</style>
</head>
<body>
<main>
<div class="card">
<div class="label">Cloudly Jump</div>
<h1>Connect System</h1>
<p>Cluster: <strong>${this.escapeHtml(clusterName)}</strong></p>
<p>Run this command on the Linux system you want to connect:</p>
<pre>${this.escapeHtml(command)}</pre>
<div class="status">${isUsable ? 'Ready to use' : 'This jump code is invalid, expired, or already used'}</div>
</div>
</main>
</body>
</html>`;
return new Response(html, {
status: isUsable ? 200 : 404,
headers: {
'Content-Type': 'text/html; charset=utf-8',
},
});
}
private async createSetupScriptResponse(jumpCodeArg: string) {
if (!jumpCodeArg || !(await this.isJumpCodeUsable(jumpCodeArg))) {
return new Response('jump code is invalid, expired, or already used\n', {
status: 404,
headers: {
'Content-Type': 'text/plain; charset=utf-8',
},
});
}
return new Response(this.createSetupScript(jumpCodeArg), {
headers: {
'Content-Type': 'application/x-sh; charset=utf-8',
},
});
}
private createSetupScript(jumpCodeArg: string) {
const claimUrl = `${this.getPublicCloudlyUrl()}/jump/v1/claim`;
return `#!/usr/bin/env bash
set -euo pipefail
if [ "$(id -u)" -ne 0 ]; then
echo "Cloudly jump setup must run as root. Re-run with sudo." >&2
exit 1
fi
export DEBIAN_FRONTEND=noninteractive
export JUMP_CODE='${this.escapeShellValue(jumpCodeArg)}'
export CLAIM_URL='${this.escapeShellValue(claimUrl)}'
echo "Preparing system for Cloudly jump..."
apt-get update
apt-get install -y --force-yes curl ca-certificates git
if ! command -v docker >/dev/null 2>&1; then
curl -sSL https://get.docker.com/ | sh
fi
if ! command -v node >/dev/null 2>&1; then
curl -sL https://deb.nodesource.com/setup_18.x | bash
apt-get install -y --force-yes nodejs
fi
if ! command -v pnpm >/dev/null 2>&1; then
curl -fsSL https://get.pnpm.io/install.sh | sh -
fi
export PNPM_HOME="\${PNPM_HOME:-/root/.local/share/pnpm}"
export PATH="\${PNPM_HOME}:\${PATH}"
pnpm install -g @serve.zone/spark
REQUEST_BODY="$(node -e 'process.stdout.write(JSON.stringify({ jumpCode: process.env.JUMP_CODE, hostname: require("os").hostname() }))')"
CLAIM_RESPONSE="$(curl -fsSL -X POST "\${CLAIM_URL}" -H 'content-type: application/json' --data "\${REQUEST_BODY}")"
export CLAIM_RESPONSE
CLOUDLY_URL="$(node -e 'const data = JSON.parse(process.env.CLAIM_RESPONSE); if (!data.accepted) { throw new Error(data.message || "Cloudly rejected jump code"); } process.stdout.write(data.cloudlyUrl);')"
COREFLOW_JUMPCODE="$(node -e 'const data = JSON.parse(process.env.CLAIM_RESPONSE); if (!data.coreflowJumpCode) { throw new Error("Cloudly did not return a Coreflow jump code"); } process.stdout.write(data.coreflowJumpCode);')"
SPARK_NODE_ID="$(node -e 'const data = JSON.parse(process.env.CLAIM_RESPONSE); if (!data.nodeId) { throw new Error("Cloudly did not return a Spark node id"); } process.stdout.write(data.nodeId);')"
SPARK_NODE_TOKEN="$(node -e 'const data = JSON.parse(process.env.CLAIM_RESPONSE); if (!data.sparkNodeToken) { throw new Error("Cloudly did not return a Spark node token"); } process.stdout.write(data.sparkNodeToken);')"
spark installdaemon --mode=coreflow-node --cloudlyUrl="\${CLOUDLY_URL}" --jumpcode="\${COREFLOW_JUMPCODE}" --nodeId="\${SPARK_NODE_ID}" --nodeToken="\${SPARK_NODE_TOKEN}"
echo "Cloudly jump completed. This system is now connected."
`;
}
private async getJumpCodeByCode(jumpCodeArg: string) {
const jumpCodes = await this.CJumpCode.getInstances({
tokenHash: this.hashSecret(jumpCodeArg),
});
return jumpCodes[0] || null;
}
private async isJumpCodeUsable(jumpCodeArg: string) {
const jumpCodeDoc = await this.getJumpCodeByCode(jumpCodeArg);
return Boolean(jumpCodeDoc && !jumpCodeDoc.data.consumedAt && jumpCodeDoc.data.expiresAt > Date.now());
}
private getCodeFromContext(ctxArg: plugins.typedserver.IRequestContext) {
return ctxArg.params.code || ctxArg.url.pathname.split('/').filter(Boolean)[1] || '';
}
private shouldRenderHtml(ctxArg: plugins.typedserver.IRequestContext) {
const acceptHeader = ctxArg.headers.get('accept') || '';
const userAgent = ctxArg.headers.get('user-agent') || '';
return acceptHeader.includes('text/html') && !/(curl|wget|httpie|fetch)/i.test(userAgent);
}
private createJumpCode() {
return plugins.crypto.randomBytes(12).toString('base64url');
}
private normalizeTtl(ttlMsArg?: number) {
if (!ttlMsArg || !Number.isFinite(ttlMsArg)) {
return this.defaultTtlMs;
}
return Math.min(Math.max(ttlMsArg, 1000 * 60), this.maxTtlMs);
}
private hashSecret(secretArg: string) {
return plugins.crypto.createHash('sha256').update(secretArg).digest('hex');
}
private getPublicCloudlyUrl() {
const sslMode = this.cloudlyRef.config.data.sslMode;
const protocol = sslMode === 'none' ? 'http' : 'https';
const port = String(this.cloudlyRef.config.data.publicPort || (protocol === 'https' ? '443' : '80'));
const includePort = !((protocol === 'https' && port === '443') || (protocol === 'http' && port === '80'));
return `${protocol}://${this.cloudlyRef.config.data.publicUrl}${includePort ? `:${port}` : ''}`;
}
private async readJsonBody<T>(ctxArg: plugins.typedserver.IRequestContext): Promise<T> {
const bodyString = (await ctxArg.text()).trim();
return bodyString ? JSON.parse(bodyString) as T : {} as T;
}
private createJsonResponse(statusCodeArg: number, bodyArg: object): Response {
return new Response(JSON.stringify(bodyArg), {
status: statusCodeArg,
headers: {
'Content-Type': 'application/json',
},
});
}
private escapeHtml(valueArg: string) {
return valueArg
.replaceAll('&', '&amp;')
.replaceAll('<', '&lt;')
.replaceAll('>', '&gt;')
.replaceAll('"', '&quot;')
.replaceAll("'", '&#39;');
}
private escapeShellValue(valueArg: string) {
return valueArg.replaceAll("'", "'\\''");
}
}
+19 -2
View File
@@ -34,10 +34,13 @@ export class ClusterNode extends plugins.smartdata.SmartDataDbDoc<
// INSTANCE
@plugins.smartdata.unI()
public id: string;
public id!: string;
@plugins.smartdata.svDb()
public data: plugins.servezoneInterfaces.data.IClusterNode['data'];
public data!: plugins.servezoneInterfaces.data.IClusterNode['data'];
@plugins.smartdata.svDb()
public sparkNodeTokenHash?: string;
constructor() {
super();
@@ -54,6 +57,20 @@ export class ClusterNode extends plugins.smartdata.SmartDataDbDoc<
await this.save();
}
public async updateSparkHeartbeat(
metricsArg: plugins.servezoneInterfaces.data.IClusterNodeMetrics,
runtimeInfoArg: plugins.servezoneInterfaces.data.ISparkNodeRuntimeInfo,
) {
this.data.metrics = metricsArg;
this.data.sparkRuntimeInfo = runtimeInfoArg;
this.data.status = 'online';
this.data.lastHealthCheck = Date.now();
if (typeof runtimeInfoArg.swarmNodeId === 'string' && runtimeInfoArg.swarmNodeId) {
this.data.swarmNodeId = runtimeInfoArg.swarmNodeId;
}
await this.save();
}
public async updateStatus(status: plugins.servezoneInterfaces.data.IClusterNode['data']['status']) {
this.data.status = status;
await this.save();
+25 -14
View File
@@ -1,6 +1,7 @@
import { logger } from '../logger.js';
import * as plugins from '../plugins.js';
import type { CloudlyNodeManager } from './classes.nodemanager.js';
import type { Cluster } from '../manager.cluster/classes.cluster.js';
export class CurlFresh {
public optionsArg = {
@@ -39,33 +40,33 @@ bash -c "npm config set registry ${this.optionsArg.npmRegistry}"
bash -c "pnpm install -g @serve.zone/spark"
# lets install the spark daemon
bash -c "spark installdaemon"
# TODO: start spark with jump code
bash -c "spark installdaemon --mode=coreflow-node --cloudlyUrl='__CLOUDLY_URL__' --jumpcode='__JUMPCODE__'"
`,
};
public nodeManagerRef: CloudlyNodeManager;
public curlFreshRoute: plugins.typedserver.servertools.Route;
public handler = new plugins.typedserver.servertools.Handler('ALL', async (req, res) => {
public async handleRequest(ctx: plugins.typedserver.IRequestContext): Promise<Response> {
logger.log('info', 'curlfresh handler called. a server might be coming online soon :)');
const scriptname = req.params.scriptname;
const scriptname = ctx.params.scriptname;
switch (scriptname) {
case 'setup.sh':
logger.log('info', 'sending setup.sh');
res.type('application/x-sh');
res.send(this.scripts['setup.sh']);
break;
return new Response(this.scripts['setup.sh']
.replaceAll('__CLOUDLY_URL__', ctx.url.searchParams.get('cloudlyUrl') || '')
.replaceAll('__JUMPCODE__', ctx.url.searchParams.get('jumpcode') || ''), {
headers: {
'Content-Type': 'application/x-sh',
},
});
default:
res.send('no script found');
break;
return new Response('no script found', { status: 404 });
}
});
}
constructor(nodeManagerRefArg: CloudlyNodeManager) {
this.nodeManagerRef = nodeManagerRefArg;
}
public async getServerUserData(): Promise<string> {
public async getServerUserData(clusterArg?: Cluster): Promise<string> {
const sslMode =
await this.nodeManagerRef.cloudlyRef.config.appData.waitForAndGetKey('sslMode');
let protocol: 'http' | 'https';
@@ -80,9 +81,19 @@ bash -c "spark installdaemon"
const port =
await this.nodeManagerRef.cloudlyRef.config.appData.waitForAndGetKey('publicPort');
let cloudlyUrl = `${protocol}://${domain}:${port}/`;
let jumpcode = '';
if (clusterArg?.data.userId) {
const clusterUser = await this.nodeManagerRef.cloudlyRef.authManager.CUser.getInstance({
id: clusterArg.data.userId,
});
jumpcode = clusterUser?.data.tokens?.[0]?.token || '';
cloudlyUrl = clusterArg.data.cloudlyUrl || cloudlyUrl;
}
const serverUserData = `#cloud-config
runcmd:
- curl -o- ${protocol}://${domain}:${port}/curlfresh/setup.sh | sh
- curl -o- '${protocol}://${domain}:${port}/curlfresh/setup.sh?cloudlyUrl=${encodeURIComponent(cloudlyUrl)}&jumpcode=${encodeURIComponent(jumpcode)}' | sh
`;
console.log(serverUserData);
return serverUserData;
+134 -6
View File
@@ -4,12 +4,24 @@ import { Cluster } from '../manager.cluster/classes.cluster.js';
import { ClusterNode } from './classes.clusternode.js';
import { CurlFresh } from './classes.curlfresh.js';
interface ISparkHeartbeatRequest {
nodeId?: string;
nodeToken?: string;
metrics?: plugins.servezoneInterfaces.data.IClusterNodeMetrics;
runtimeInfo?: plugins.servezoneInterfaces.data.ISparkNodeRuntimeInfo;
}
interface ISparkHeartbeatResponse {
accepted: boolean;
message?: string;
}
export class CloudlyNodeManager {
public cloudlyRef: Cloudly;
public typedRouter = new plugins.typedrequest.TypedRouter();
public curlfreshInstance = new CurlFresh(this);
public hetznerAccount: plugins.hetznercloud.HetznerAccount;
public hetznerAccount?: plugins.hetznercloud.HetznerAccount;
public get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
@@ -23,7 +35,7 @@ export class CloudlyNodeManager {
* is used be serverconfig module on the node to get the actual node config
*/
this.typedRouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.config.IRequest_Any_Cloudly_GetNodeConfig>(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.node.IRequest_Any_Cloudly_GetNodeConfig>(
'getNodeConfig',
async (requestData) => {
const nodeId = requestData.nodeId;
@@ -51,6 +63,69 @@ export class CloudlyNodeManager {
public async stop() {}
public async handleSparkHeartbeatHttpRequest(
ctxArg: plugins.typedserver.IRequestContext,
): Promise<Response> {
try {
const requestData = await this.readJsonBody<ISparkHeartbeatRequest>(ctxArg);
const response = await this.acceptSparkHeartbeat(requestData);
return this.createJsonResponse(200, response);
} catch (error) {
return this.createJsonResponse(400, {
accepted: false,
message: `Spark heartbeat failed: ${(error as Error).message}`,
} satisfies ISparkHeartbeatResponse);
}
}
public async acceptSparkHeartbeat(
requestDataArg: ISparkHeartbeatRequest,
): Promise<ISparkHeartbeatResponse> {
if (!requestDataArg.nodeId) {
return {
accepted: false,
message: 'Spark node id is missing',
};
}
if (!requestDataArg.nodeToken) {
return {
accepted: false,
message: 'Spark node token is missing',
};
}
if (!this.isSparkMetrics(requestDataArg.metrics)) {
return {
accepted: false,
message: 'Spark metrics are missing or invalid',
};
}
if (!this.isSparkRuntimeInfo(requestDataArg.runtimeInfo, requestDataArg.nodeId)) {
return {
accepted: false,
message: 'Spark runtime info is missing or invalid',
};
}
const node = await this.CClusterNode.getInstance({ id: requestDataArg.nodeId });
if (!node) {
return {
accepted: false,
message: 'Spark node is unknown',
};
}
if (node.sparkNodeTokenHash !== this.hashSecret(requestDataArg.nodeToken)) {
return {
accepted: false,
message: 'Spark node token is invalid',
};
}
await node.updateSparkHeartbeat(requestDataArg.metrics, requestDataArg.runtimeInfo);
return {
accepted: true,
};
}
/**
* creates the node infrastructure on hetzner
* ensures that there are exactly the resources that are needed
@@ -65,13 +140,17 @@ export class CloudlyNodeManager {
console.log(`Skipping node provisioning for cluster ${cluster.id} - setupMode is ${cluster.data.setupMode || 'manual'}`);
continue;
}
const hetznerAccount = this.hetznerAccount;
if (!hetznerAccount) {
throw new Error('Hetzner account is not configured');
}
// get existing nodes
const nodes = await this.getNodesByCluster(cluster);
// if there is no node, create one
if (nodes.length === 0) {
const hetznerServer = await this.hetznerAccount.createServer({
const hetznerServer = await hetznerAccount.createServer({
name: plugins.smartunique.uniSimple('node'),
location: 'nbg1',
type: 'cpx41',
@@ -79,7 +158,7 @@ export class CloudlyNodeManager {
clusterId: cluster.id,
priority: '1',
},
userData: await this.curlfreshInstance.getServerUserData(),
userData: await this.curlfreshInstance.getServerUserData(cluster),
});
// First create BareMetal record
@@ -94,12 +173,12 @@ export class CloudlyNodeManager {
);
// if there is a node, make sure that it exists
for (const node of nodes) {
const hetznerServers = await this.hetznerAccount.getServersByLabel({
const hetznerServers = await hetznerAccount.getServersByLabel({
clusterId: cluster.id,
});
if (!hetznerServers || hetznerServers.length === 0) {
console.log(`node ${node.id} does not exist in the real world. Creating it now...`);
const hetznerServer = await this.hetznerAccount.createServer({
const hetznerServer = await hetznerAccount.createServer({
name: plugins.smartunique.uniSimple('node'),
location: 'nbg1',
type: 'cpx41',
@@ -107,6 +186,7 @@ export class CloudlyNodeManager {
clusterId: cluster.id,
priority: '1',
},
userData: await this.curlfreshInstance.getServerUserData(cluster),
});
// First create BareMetal record
@@ -128,4 +208,52 @@ export class CloudlyNodeManager {
});
return results;
}
private isSparkMetrics(valueArg: unknown): valueArg is plugins.servezoneInterfaces.data.IClusterNodeMetrics {
if (!valueArg || typeof valueArg !== 'object') {
return false;
}
const metrics = valueArg as Partial<plugins.servezoneInterfaces.data.IClusterNodeMetrics>;
return typeof metrics.cpuUsagePercent === 'number'
&& typeof metrics.memoryUsedMB === 'number'
&& typeof metrics.memoryAvailableMB === 'number'
&& typeof metrics.diskUsedGB === 'number'
&& typeof metrics.diskAvailableGB === 'number'
&& typeof metrics.containerCount === 'number'
&& typeof metrics.timestamp === 'number';
}
private isSparkRuntimeInfo(
valueArg: unknown,
nodeIdArg: string,
): valueArg is plugins.servezoneInterfaces.data.ISparkNodeRuntimeInfo {
if (!valueArg || typeof valueArg !== 'object') {
return false;
}
const runtimeInfo = valueArg as Record<string, unknown>;
return runtimeInfo.runtime === 'spark'
&& runtimeInfo.nodeId === nodeIdArg
&& typeof runtimeInfo.checkedAt === 'number';
}
private hashSecret(secretArg: string) {
return plugins.crypto.createHash('sha256').update(secretArg).digest('hex');
}
private async readJsonBody<T>(ctxArg: plugins.typedserver.IRequestContext): Promise<T> {
const bodyString = (await ctxArg.text()).trim();
return bodyString ? JSON.parse(bodyString) as T : {} as T;
}
private createJsonResponse(
statusCodeArg: number,
bodyArg: object,
): Response {
return new Response(JSON.stringify(bodyArg), {
status: statusCodeArg,
headers: {
'Content-Type': 'application/json',
},
});
}
}
@@ -0,0 +1,68 @@
import * as plugins from '../plugins.js';
import type { CloudlyPlatformManager } from './classes.platformmanager.js';
@plugins.smartdata.managed()
export class PlatformBinding extends plugins.smartdata.SmartDataDbDoc<
PlatformBinding,
plugins.servezoneInterfaces.platform.IPlatformBinding,
CloudlyPlatformManager
> {
public static async upsertBinding(
bindingArg: plugins.servezoneInterfaces.platform.IPlatformBinding,
) {
const existingBinding = bindingArg.id
? await this.getInstance({
id: bindingArg.id,
})
: undefined;
const binding = existingBinding || new PlatformBinding();
const timestamp = Date.now();
Object.assign(binding, {
...bindingArg,
id: bindingArg.id || (await this.getNewId()),
status: bindingArg.status || 'requested',
desiredState: bindingArg.desiredState || 'enabled',
createdAt: bindingArg.createdAt || existingBinding?.createdAt || timestamp,
updatedAt: timestamp,
});
await binding.save();
return binding;
}
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public serviceId!: string;
@plugins.smartdata.svDb()
public capability!: plugins.servezoneInterfaces.platform.TPlatformCapability;
@plugins.smartdata.svDb()
public desiredState!: plugins.servezoneInterfaces.platform.TPlatformDesiredState;
@plugins.smartdata.svDb()
public status!: plugins.servezoneInterfaces.platform.TPlatformBindingStatus;
@plugins.smartdata.svDb()
public providerConfigId?: string;
@plugins.smartdata.svDb()
public config?: { [key: string]: plugins.servezoneInterfaces.platform.TPlatformConfigValue };
@plugins.smartdata.svDb()
public endpoints?: plugins.servezoneInterfaces.platform.IPlatformServiceEndpoint[];
@plugins.smartdata.svDb()
public credentials?: plugins.servezoneInterfaces.platform.IPlatformCredentialRef[];
@plugins.smartdata.svDb()
public createdAt?: number;
@plugins.smartdata.svDb()
public updatedAt?: number;
@plugins.smartdata.svDb()
public errorText?: string;
}
@@ -0,0 +1,228 @@
import type { Cloudly } from '../classes.cloudly.js';
import * as plugins from '../plugins.js';
import { PlatformBinding } from './classes.platformbinding.js';
import { PlatformProviderConfig } from './classes.platformproviderconfig.js';
export class CloudlyPlatformManager {
public typedrouter = new plugins.typedrequest.TypedRouter();
public cloudlyRef: Cloudly;
public capabilities: plugins.servezoneInterfaces.platform.IPlatformCapability[] = [
{ id: 'email', title: 'Email', accessMode: 'rpc', defaultProviderType: 'cloudly' },
{ id: 'sms', title: 'SMS', accessMode: 'rpc', defaultProviderType: 'cloudly' },
{ id: 'pushnotification', title: 'Push Notifications', accessMode: 'rpc', defaultProviderType: 'cloudly' },
{ id: 'letter', title: 'Letters', accessMode: 'rpc', defaultProviderType: 'cloudly' },
{ id: 'ai', title: 'AI', accessMode: 'rpc', defaultProviderType: 'cloudly' },
{ id: 'database', title: 'Database', accessMode: 'binding', defaultProviderType: 'docker' },
{ id: 'objectstorage', title: 'Object Storage', accessMode: 'binding', defaultProviderType: 's3' },
{ id: 'logging', title: 'Logging', accessMode: 'sidecar', defaultProviderType: 'corelog' },
{ id: 'backup', title: 'Backup', accessMode: 'internal', defaultProviderType: 'corebackup' },
{ id: 'sip', title: 'SIP', accessMode: 'rpc', defaultProviderType: 'cloudly' },
];
get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
}
public CPlatformProviderConfig = plugins.smartdata.setDefaultManagerForDoc(
this,
PlatformProviderConfig,
);
public CPlatformBinding = plugins.smartdata.setDefaultManagerForDoc(this, PlatformBinding);
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_GetPlatformDesiredState>(
'getPlatformDesiredState',
async (requestData) => {
await this.passValidIdentity(requestData);
return await this.getPlatformDesiredState();
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_GetPlatformCapabilities>(
'getPlatformCapabilities',
async (requestData) => {
await this.passValidIdentity(requestData);
return {
capabilities: this.capabilities,
};
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_GetPlatformProviderConfigs>(
'getPlatformProviderConfigs',
async (requestData) => {
await this.passValidIdentity(requestData);
const query = requestData.capability ? { capability: requestData.capability } : {};
return {
providerConfigs: await this.getProviderConfigs(query),
};
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_UpsertPlatformProviderConfig>(
'upsertPlatformProviderConfig',
async (requestData) => {
await this.passAdminIdentity(requestData);
const providerConfig = await PlatformProviderConfig.upsertProviderConfig(
requestData.providerConfig,
);
return {
providerConfig: await providerConfig.createSavableObject(),
};
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_DeletePlatformProviderConfigById>(
'deletePlatformProviderConfigById',
async (requestData) => {
await this.passAdminIdentity(requestData);
const providerConfig = await PlatformProviderConfig.getInstance({
id: requestData.providerConfigId,
});
if (providerConfig) {
await providerConfig.delete();
}
return {
success: true,
};
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_GetPlatformBindings>(
'getPlatformBindings',
async (requestData) => {
await this.passValidIdentity(requestData);
return {
bindings: await this.getBindings({
...(requestData.serviceId ? { serviceId: requestData.serviceId } : {}),
...(requestData.capability ? { capability: requestData.capability } : {}),
}),
};
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_UpsertPlatformBinding>(
'upsertPlatformBinding',
async (requestData) => {
await this.passAdminIdentity(requestData);
const binding = await PlatformBinding.upsertBinding(requestData.binding);
return {
binding: await binding.createSavableObject(),
};
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_UpdatePlatformBindingStatus>(
'updatePlatformBindingStatus',
async (requestData) => {
await this.passAdminOrClusterIdentity(requestData);
const binding = await PlatformBinding.getInstance({
id: requestData.bindingId,
});
if (!binding) {
throw new plugins.typedrequest.TypedResponseError(
`Platform binding ${requestData.bindingId} not found`,
);
}
binding.status = requestData.status;
binding.updatedAt = Date.now();
if (requestData.endpoints) {
binding.endpoints = requestData.endpoints;
}
if (requestData.credentials) {
binding.credentials = requestData.credentials;
}
if (requestData.errorText !== undefined) {
binding.errorText = requestData.errorText;
}
await binding.save();
return {
binding: await binding.createSavableObject(),
};
},
),
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.platform.IReq_Any_Cloudly_DeletePlatformBindingById>(
'deletePlatformBindingById',
async (requestData) => {
await this.passAdminIdentity(requestData);
const binding = await PlatformBinding.getInstance({
id: requestData.bindingId,
});
if (binding) {
await binding.delete();
}
return {
success: true,
};
},
),
);
}
public async start() {}
public async stop() {}
public async getPlatformDesiredState() {
return {
capabilities: this.capabilities,
providerConfigs: await this.getProviderConfigs(),
bindings: await this.getBindings(),
};
}
public async getProviderConfigs(queryArg: Record<string, unknown> = {}) {
const providerConfigs = await this.CPlatformProviderConfig.getInstances(queryArg);
return await Promise.all(
providerConfigs.map((providerConfig) => providerConfig.createSavableObject()),
);
}
public async getBindings(queryArg: Record<string, unknown> = {}) {
const bindings = await this.CPlatformBinding.getInstances(queryArg);
return await Promise.all(bindings.map((binding) => binding.createSavableObject()));
}
private async passValidIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
await plugins.smartguard.passGuardsOrReject(requestData, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
}
private async passAdminIdentity(requestData: { identity: plugins.servezoneInterfaces.data.IIdentity }) {
await plugins.smartguard.passGuardsOrReject(requestData, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
}
private async passAdminOrClusterIdentity(requestData: {
identity: plugins.servezoneInterfaces.data.IIdentity;
}) {
await this.passValidIdentity(requestData);
if (requestData.identity.role !== 'admin' && requestData.identity.role !== 'cluster') {
throw new plugins.typedrequest.TypedResponseError('identity must be admin or cluster');
}
}
}
@@ -0,0 +1,47 @@
import * as plugins from '../plugins.js';
import type { CloudlyPlatformManager } from './classes.platformmanager.js';
@plugins.smartdata.managed()
export class PlatformProviderConfig extends plugins.smartdata.SmartDataDbDoc<
PlatformProviderConfig,
plugins.servezoneInterfaces.platform.IPlatformProviderConfig,
CloudlyPlatformManager
> {
public static async upsertProviderConfig(
providerConfigArg: plugins.servezoneInterfaces.platform.IPlatformProviderConfig,
) {
const providerConfig =
(providerConfigArg.id &&
(await this.getInstance({
id: providerConfigArg.id,
}))) || new PlatformProviderConfig();
Object.assign(providerConfig, {
...providerConfigArg,
id: providerConfigArg.id || (await this.getNewId()),
});
await providerConfig.save();
return providerConfig;
}
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public capability!: plugins.servezoneInterfaces.platform.TPlatformCapability;
@plugins.smartdata.svDb()
public providerType!: string;
@plugins.smartdata.svDb()
public name!: string;
@plugins.smartdata.svDb()
public enabled!: boolean;
@plugins.smartdata.svDb()
public config?: { [key: string]: plugins.servezoneInterfaces.platform.TPlatformConfigValue };
@plugins.smartdata.svDb()
public secretBundleId?: string;
}
@@ -0,0 +1,416 @@
import type { Cloudly } from '../classes.cloudly.js';
import { logger } from '../logger.js';
import * as plugins from '../plugins.js';
import type { Service } from '../manager.service/classes.service.js';
type TAuthenticatedRegistryUser = {
userId: string;
username: string;
canWrite: boolean;
};
export class CloudlyRegistryManager {
private cloudlyRef: Cloudly;
private smartRegistry!: plugins.smartregistry.SmartRegistry;
private recordedTagDigests = new Map<string, string>();
private started = false;
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
}
public async start() {
const publicRegistryUrl = this.getPublicRegistryUrl();
const registryJwtSecret = JSON.stringify(this.cloudlyRef.authManager.smartjwtInstance.getKeyPairAsJson());
const s3Descriptor = this.cloudlyRef.config.data.s3Descriptor;
if (!s3Descriptor?.bucketName) {
throw new Error('Cloudly registry requires an S3 bucketName');
}
this.smartRegistry = new plugins.smartregistry.SmartRegistry({
storage: s3Descriptor as plugins.smartregistry.IStorageConfig,
storageHooks: {
afterPut: async (contextArg) => {
await this.handleRegistryStorageAfterPut(contextArg);
},
},
auth: {
jwtSecret: registryJwtSecret,
tokenStore: 'memory',
npmTokens: { enabled: false },
ociTokens: {
enabled: true,
realm: `${publicRegistryUrl}/v2/token`,
service: this.cloudlyRef.config.data.publicUrl || 'cloudly',
},
pypiTokens: { enabled: false },
rubygemsTokens: { enabled: false },
},
oci: {
enabled: true,
basePath: '/v2',
registryUrl: publicRegistryUrl,
},
});
await this.smartRegistry.init();
this.started = true;
logger.log('info', `Cloudly OCI registry available at ${publicRegistryUrl}/v2`);
}
public async stop() {
if (this.smartRegistry) {
this.smartRegistry.destroy();
}
this.started = false;
}
public async handleHttpRequest(
ctx: plugins.typedserver.IRequestContext,
): Promise<Response> {
try {
const requestUrl = ctx.url;
if (requestUrl.pathname === '/v2/token') {
return await this.handleTokenRequest(ctx, requestUrl);
}
if (!this.started) {
return new Response('registry is not ready', { status: 503 });
}
const rawBody = Buffer.from(await ctx.request.arrayBuffer());
const response = await this.smartRegistry.handleRequest({
method: ctx.method || 'GET',
path: requestUrl.pathname,
query: Object.fromEntries(requestUrl.searchParams),
headers: this.headersToRecord(ctx.headers),
rawBody: rawBody.length > 0 ? rawBody : undefined,
});
return this.createRegistryResponse(response);
} catch (error) {
logger.log('error', `registry request failed: ${(error as Error).message}`);
return new Response('registry request failed', { status: 500 });
}
}
public getRegistryHost() {
if (!this.cloudlyRef.config.data.publicUrl) {
throw new Error('Cloudly registry requires publicUrl');
}
const publicPort = this.cloudlyRef.config.data.publicPort;
const includePort =
this.cloudlyRef.config.data.sslMode === 'none' && publicPort && !['80', '443'].includes(publicPort);
return `${this.cloudlyRef.config.data.publicUrl}${includePort ? `:${publicPort}` : ''}`;
}
public getServiceRegistryTarget(
serviceArg: Service,
tagArg = 'latest',
): plugins.servezoneInterfaces.data.IRegistryTarget {
const registryHost = this.getRegistryHost();
const repository = this.getServiceRepository(serviceArg);
return {
protocol: 'oci',
registryHost,
repository,
tag: tagArg,
imageUrl: `${registryHost}/${repository}:${tagArg}`,
serviceId: serviceArg.id,
imageId: serviceArg.data?.imageId,
};
}
private async handleRegistryStorageAfterPut(
contextArg: plugins.smartregistry.IStorageHookContext,
) {
try {
if (contextArg.protocol !== 'oci') {
return;
}
if (!contextArg.key.startsWith('oci/tags/') || !contextArg.key.endsWith('/tags.json')) {
return;
}
const repository = contextArg.key.slice('oci/tags/'.length, -'/tags.json'.length);
const tagsBuffer = await this.smartRegistry.getStorage().getObject(contextArg.key);
if (!tagsBuffer) {
return;
}
const tags = JSON.parse(tagsBuffer.toString('utf8')) as Record<string, string>;
for (const [tag, digest] of Object.entries(tags)) {
const tagKey = `${repository}:${tag}`;
if (this.recordedTagDigests.get(tagKey) === digest) {
continue;
}
this.recordedTagDigests.set(tagKey, digest);
await this.recordRegistryPushEvent(repository, tag, digest, contextArg.actor?.userId);
}
} catch (error) {
logger.log('error', `registry push event handling failed: ${(error as Error).message}`);
}
}
private async recordRegistryPushEvent(
repositoryArg: string,
tagArg: string,
digestArg: string,
actorUserIdArg?: string,
) {
const service = await this.getServiceByRegistryRepository(repositoryArg);
if (!service) {
logger.log('info', `registry push for unmapped repository ${repositoryArg}:${tagArg}`);
return;
}
const registryTarget = this.getServiceRegistryTarget(service, tagArg);
const pushEvent: plugins.servezoneInterfaces.data.IRegistryPushEvent = {
protocol: 'oci',
registryHost: registryTarget.registryHost,
repository: repositoryArg,
tag: tagArg,
digest: digestArg,
imageUrl: registryTarget.imageUrl,
pushedAt: Date.now(),
serviceId: service.id,
imageId: service.data.imageId,
actorUserId: actorUserIdArg,
};
service.data = {
...service.data,
...(service.data.deployOnPush === false ? {} : { imageVersion: tagArg }),
registryTarget,
};
await service.save();
await this.recordImagePushEvent(service, pushEvent);
if (service.data.deployOnPush !== false) {
await this.cloudlyRef.coreflowManager.pushClusterConfigToConnectedCoreflows();
}
logger.log('info', `recorded registry push ${repositoryArg}:${tagArg} -> ${digestArg}`);
}
private async recordImagePushEvent(
serviceArg: Service,
pushEventArg: plugins.servezoneInterfaces.data.IRegistryPushEvent,
) {
if (!serviceArg.data.imageId) {
return;
}
const image = await this.cloudlyRef.imageManager.CImage.getInstance({
id: serviceArg.data.imageId,
}).catch(() => null);
if (!image) {
return;
}
image.data.versions = image.data.versions || [];
const existingVersion = image.data.versions.find((versionArg) => {
return versionArg.versionString === pushEventArg.tag;
});
const versionData = {
versionString: pushEventArg.tag,
digest: pushEventArg.digest,
registryRepository: pushEventArg.repository,
registryTag: pushEventArg.tag,
source: 'registry' as const,
size: existingVersion?.size || 0,
createdAt: existingVersion?.createdAt || pushEventArg.pushedAt,
};
if (existingVersion) {
Object.assign(existingVersion, versionData);
} else {
image.data.versions.push(versionData);
}
image.data.lastPushEvent = pushEventArg;
await image.save();
}
private async getServiceByRegistryRepository(repositoryArg: string) {
const services = await this.cloudlyRef.serviceManager.CService.getInstances({});
return services.find((serviceArg) => {
return this.getServiceRepository(serviceArg) === repositoryArg;
});
}
private getServiceRepository(serviceArg: Service) {
const serviceName = this.slugify(serviceArg.data?.name || serviceArg.id);
const serviceId = this.slugify(serviceArg.id).slice(0, 12) || serviceArg.id;
return `workloads/${this.slugify(`${serviceName}-${serviceId}`)}`;
}
private slugify(valueArg: string) {
return valueArg
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/-+/g, '-')
.replace(/^-+|-+$/g, '')
|| 'service';
}
private async handleTokenRequest(
ctx: plugins.typedserver.IRequestContext,
requestUrl: URL,
): Promise<Response> {
const user = await this.authenticateRequest(ctx);
if (!user) {
return new Response('authentication required', {
status: 401,
headers: {
'WWW-Authenticate': 'Basic realm="Cloudly Registry"',
},
});
}
const requestedScopes = this.getRequestedOciScopes(requestUrl.searchParams);
const requestedWriteAccess = requestedScopes.some((scopeArg) => {
const action = scopeArg.split(':').at(-1);
return action === 'push' || action === 'delete';
});
if (requestedWriteAccess && !user.canWrite) {
return new Response('registry write access denied', { status: 403 });
}
const token = await this.smartRegistry.getAuthManager().createOciToken(
user.userId,
requestedScopes,
3600,
);
return new Response(
JSON.stringify({
token,
access_token: token,
expires_in: 3600,
issued_at: new Date().toISOString(),
}),
{
status: 200,
headers: {
'Content-Type': 'application/json',
},
},
);
}
private async authenticateRequest(
ctx: plugins.typedserver.IRequestContext,
): Promise<TAuthenticatedRegistryUser | null> {
const credentials = this.getBasicCredentials(ctx);
if (!credentials) {
return null;
}
const users = await this.cloudlyRef.authManager.CUser.getInstances({});
for (const user of users) {
if (user.data?.username !== credentials.username) {
continue;
}
const passwordMatches = user.data.password === credentials.password;
const matchingToken = user.data.tokens?.find((tokenArg) => {
return tokenArg.token === credentials.password && tokenArg.expiresAt > Date.now();
});
if (!passwordMatches && !matchingToken) {
continue;
}
const assignedRoles = matchingToken?.assignedRoles || [];
return {
userId: user.id,
username: user.data.username,
canWrite: user.data.role === 'admin' || assignedRoles.includes('admin'),
};
}
return null;
}
private getBasicCredentials(ctx: plugins.typedserver.IRequestContext) {
const authHeader = ctx.headers.get('authorization');
if (!authHeader?.startsWith('Basic ')) {
return null;
}
const decoded = Buffer.from(authHeader.slice('Basic '.length), 'base64').toString('utf8');
const separatorIndex = decoded.indexOf(':');
if (separatorIndex <= 0) {
return null;
}
return {
username: decoded.slice(0, separatorIndex),
password: decoded.slice(separatorIndex + 1),
};
}
private getRequestedOciScopes(searchParamsArg: URLSearchParams) {
const scopes: string[] = [];
for (const scope of searchParamsArg.getAll('scope')) {
const [scopeType, scopeName, actionsString] = scope.split(':');
if (scopeType !== 'repository' || !scopeName || !actionsString) {
continue;
}
for (const action of actionsString.split(',')) {
if (action) {
scopes.push(`oci:${scopeType}:${scopeName}:${action}`);
}
}
}
return scopes;
}
private getPublicRegistryUrl() {
return `${this.cloudlyRef.config.data.sslMode === 'none' ? 'http' : 'https'}://${this.getRegistryHost()}`;
}
private headersToRecord(headersArg: Headers) {
const headers: Record<string, string> = {};
headersArg.forEach((value, key) => {
headers[key.toLowerCase()] = value;
});
return headers;
}
private createRegistryResponse(
responseArg: plugins.smartregistry.IResponse,
): Response {
const headers = new Headers();
for (const [key, value] of Object.entries(responseArg.headers)) {
headers.set(key, value);
}
if (!responseArg.body) {
return new Response(null, {
status: responseArg.status,
headers,
});
}
if (responseArg.body instanceof ReadableStream) {
return new Response(responseArg.body, {
status: responseArg.status,
headers,
});
}
if (Buffer.isBuffer(responseArg.body) || typeof responseArg.body === 'string') {
return new Response(responseArg.body as BodyInit, {
status: responseArg.status,
headers,
});
}
if (!headers.has('Content-Type')) {
headers.set('Content-Type', 'application/json');
}
return new Response(JSON.stringify(responseArg.body), {
status: responseArg.status,
headers,
});
}
}
+1
View File
@@ -0,0 +1 @@
export * from './classes.registrymanager.js';
+2 -2
View File
@@ -12,10 +12,10 @@ export class SecretBundle extends plugins.smartdata.SmartDataDbDoc<
// INSTANCE
@plugins.smartdata.unI()
public id: string;
public id!: string;
@plugins.smartdata.svDb()
public data: plugins.servezoneInterfaces.data.ISecretBundle['data'];
public data!: plugins.servezoneInterfaces.data.ISecretBundle['data'];
public async getSecretGroups() {
const secretGroups: SecretGroup[] = [];
+2 -2
View File
@@ -14,8 +14,8 @@ export class SecretGroup extends plugins.smartdata.SmartDataDbDoc<
* the insatnce id. This should be a random id, except for default
*/
@plugins.smartdata.unI()
id: string;
id!: string;
@plugins.smartdata.svDb()
data: plugins.servezoneInterfaces.data.ISecretGroup['data'];
data!: plugins.servezoneInterfaces.data.ISecretGroup['data'];
}
+19 -4
View File
@@ -18,9 +18,9 @@ export class CloudlySecretManager {
// INSTANCE
public cloudlyRef: Cloudly;
public projectinfo = new plugins.projectinfo.ProjectinfoNpm(paths.packageDir);
public projectinfo = plugins.projectinfo.ProjectinfoNpm.create(paths.packageDir);
public serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
public typedrouter: plugins.typedrequest.TypedRouter;
public typedrouter!: plugins.typedrequest.TypedRouter;
get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
@@ -40,7 +40,7 @@ export class CloudlySecretManager {
new plugins.typedrequest.TypedHandler(
'getSecretBundles',
async (dataArg, toolsArg) => {
await toolsArg.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], dataArg);
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], dataArg);
dataArg.identity.jwt;
const secretBundles = await SecretBundle.getInstances({});
return {
@@ -54,6 +54,18 @@ export class CloudlySecretManager {
),
);
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.secretbundle.IReq_GetSecretBundleById>(
new plugins.typedrequest.TypedHandler('getSecretBundleById', async (dataArg, toolsArg) => {
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminOrClusterIdentityGuard], dataArg);
const secretBundle = await SecretBundle.getInstance({
id: dataArg.secretBundleId,
});
return {
secretBundle: await secretBundle.createSavableObject(),
};
}),
);
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.secretbundle.IReq_CreateSecretBundle>(
new plugins.typedrequest.TypedHandler('createSecretBundle', async (dataArg) => {
const secretBundle = new SecretBundle();
@@ -96,7 +108,7 @@ export class CloudlySecretManager {
new plugins.typedrequest.TypedHandler(
'getSecretGroups',
async (dataArg, toolsArg) => {
await toolsArg.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], dataArg);
await toolsArg!.passGuards([this.cloudlyRef.authManager.adminIdentityGuard], dataArg);
dataArg.identity.jwt;
const secretGroups = await SecretGroup.getInstances({});
return {
@@ -164,6 +176,9 @@ export class CloudlySecretManager {
const authorization = await wantedBundle.getAuthorizationFromAuthKey(
dataArg.secretBundleAuthorization.secretAccessKey,
);
if (!authorization) {
throw new plugins.typedrequest.TypedResponseError('secret bundle authorization not found');
}
return {
flatKeyValueObject: await wantedBundle.getKeyValueObjectForEnvironment(
authorization.environment,
+48 -5
View File
@@ -1,7 +1,8 @@
import { SecretBundle } from 'ts/manager.secret/classes.secretbundle.js';
import { SecretBundle } from '../manager.secret/classes.secretbundle.js';
import * as plugins from '../plugins.js';
import { ServiceManager } from './classes.servicemanager.js';
@plugins.smartdata.managed()
export class Service extends plugins.smartdata.SmartDataDbDoc<
Service,
plugins.servezoneInterfaces.data.IService,
@@ -23,17 +24,23 @@ export class Service extends plugins.smartdata.SmartDataDbDoc<
public static async createService(serviceDataArg: Partial<plugins.servezoneInterfaces.data.IService['data']>) {
const service = new Service();
service.id = await Service.getNewId();
Object.assign(service, serviceDataArg);
service.data = serviceDataArg as plugins.servezoneInterfaces.data.IService['data'];
await service.save();
// Create DNS entries if service has web port and domains configured
if (service.data.ports?.web && service.data.domains?.length > 0) {
await service.createDnsEntries();
}
return service;
}
// INSTANCE
@plugins.smartdata.svDb()
public id: string;
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public data: plugins.servezoneInterfaces.data.IService['data'];
public data!: plugins.servezoneInterfaces.data.IService['data'];
/**
* a service runs in a specific environment
@@ -54,4 +61,40 @@ export class Service extends plugins.smartdata.SmartDataDbDoc<
}
return finalFlatObject;
}
/**
* Creates DNS entries for this service (in inactive state)
* These will be activated when the service is deployed
*/
public async createDnsEntries() {
const dnsManager = this.manager.cloudlyRef.dnsManager;
for (const domain of this.data.domains) {
const dnsEntryData: plugins.servezoneInterfaces.data.IDnsEntry['data'] = {
type: 'A', // Default to A record, could be made configurable
name: domain.name,
value: '0.0.0.0', // Placeholder, will be updated on deployment
ttl: 3600,
zone: '', // Will be set based on domainId
domainId: domain.domainId,
active: false, // Created as inactive
description: `Auto-generated DNS entry for service ${this.data.name}`,
createdAt: Date.now(),
isAutoGenerated: true,
sourceServiceId: this.id,
sourceType: 'service',
};
// Create the DNS entry
await dnsManager.createServiceDnsEntry(dnsEntryData);
}
}
/**
* Removes DNS entries for this service
*/
public async removeDnsEntries() {
const dnsManager = this.manager.cloudlyRef.dnsManager;
await dnsManager.removeServiceDnsEntries(this.id);
}
}
@@ -14,6 +14,8 @@ export class ServiceManager {
constructor(cloudlyRef: Cloudly) {
this.cloudlyRef = cloudlyRef;
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_GetServices>(
@@ -36,6 +38,23 @@ export class ServiceManager {
)
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_GetServiceById>(
'getServiceById',
async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const service = await Service.getInstance({
id: dataArg.serviceId,
});
return {
service: await service.createSavableObject(),
};
}
)
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_GetServiceSecretBundlesAsFlatObject>(
'getServiceSecretBundlesAsFlatObject',
@@ -51,11 +70,40 @@ export class ServiceManager {
)
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_GetServiceRegistryTarget>(
'getServiceRegistryTarget',
async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const service = await Service.getInstance({
id: dataArg.serviceId,
});
return {
registryTarget: this.cloudlyRef.registryManager.getServiceRegistryTarget(
service,
dataArg.tag || service.data.imageVersion || 'latest',
),
};
}
)
);
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_CreateService>(
'createService',
async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
const service = await Service.createService(dataArg.serviceData);
service.data.registryTarget = this.cloudlyRef.registryManager.getServiceRegistryTarget(
service,
service.data.imageVersion || 'latest',
);
await service.save();
await this.cloudlyRef.coreflowManager.pushClusterConfigToConnectedCoreflows();
return {
service: await service.createSavableObject(),
};
@@ -67,6 +115,9 @@ export class ServiceManager {
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_UpdateService>(
'updateService',
async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
const service = await Service.getInstance({
id: dataArg.serviceId,
});
@@ -74,7 +125,12 @@ export class ServiceManager {
...service.data,
...dataArg.serviceData,
};
service.data.registryTarget = this.cloudlyRef.registryManager.getServiceRegistryTarget(
service,
service.data.imageVersion || 'latest',
);
await service.save();
await this.cloudlyRef.coreflowManager.pushClusterConfigToConnectedCoreflows();
return {
service: await service.createSavableObject(),
};
@@ -86,9 +142,16 @@ export class ServiceManager {
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_DeleteServiceById>(
'deleteServiceById',
async (dataArg) => {
await plugins.smartguard.passGuardsOrReject(dataArg, [
this.cloudlyRef.authManager.adminIdentityGuard,
]);
const service = await Service.getInstance({
id: dataArg.serviceId,
});
// Remove DNS entries before deleting the service
await service.removeDnsEntries();
await service.delete();
return {
success: true,
@@ -97,4 +160,14 @@ export class ServiceManager {
)
);
}
public async start() {
// ServiceManager is ready - handlers are already registered in constructor
console.log('ServiceManager started');
}
public async stop() {
// Cleanup if needed
console.log('ServiceManager stopped');
}
}
+49 -6
View File
@@ -5,7 +5,7 @@ import * as servezoneInterfaces from '@serve.zone/interfaces';
export class CloudlySettingsManager {
public cloudlyRef: Cloudly;
public readyDeferred = plugins.smartpromise.defer();
public settingsStore: plugins.smartdata.EasyStore<servezoneInterfaces.data.ICloudlySettings>;
public settingsStore!: plugins.smartdata.EasyStore<servezoneInterfaces.data.ICloudlySettings>;
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
@@ -41,7 +41,7 @@ export class CloudlySettingsManager {
const masked: servezoneInterfaces.data.ICloudlySettingsMasked = {};
for (const [key, value] of Object.entries(settings)) {
if (typeof value === 'string' && value.length > 4) {
if (this.isSensitiveSettingKey(key) && typeof value === 'string' && value.length > 4) {
// Mask the token, showing only last 4 characters
masked[key] = '****' + value.slice(-4);
} else {
@@ -51,6 +51,24 @@ export class CloudlySettingsManager {
return masked;
}
private isSensitiveSettingKey(key: string): boolean {
if (key === 'corebuildWorkersJson') {
return true;
}
const normalizedKey = key.toLowerCase();
return [
'token',
'secret',
'apikey',
'accesskey',
'applicationkey',
'consumerkey',
'keyjson',
'privatekey',
'password',
].some((sensitivePart) => normalizedKey.includes(sensitivePart));
}
/**
* Update multiple settings at once
@@ -65,6 +83,29 @@ export class CloudlySettingsManager {
await this.settingsStore.deleteKey(key as keyof servezoneInterfaces.data.ICloudlySettings);
}
}
if (Object.keys(updates).some((key) => this.isExternalGatewaySettingKey(key))) {
this.refreshExternalGatewayConfig().catch((error) => {
console.log(`External gateway settings refresh failed: ${(error as Error).message}`);
});
}
}
private isExternalGatewaySettingKey(key: string): boolean {
return [
'dcrouterGatewayUrl',
'dcrouterGatewayApiToken',
'dcrouterWorkHosterId',
'dcrouterTargetHost',
'dcrouterTargetPort',
].includes(key);
}
private async refreshExternalGatewayConfig(): Promise<void> {
await Promise.all([
this.cloudlyRef.domainManager.syncExternalGatewayDomains(),
this.cloudlyRef.coreflowManager.pushClusterConfigToConnectedCoreflows(),
]);
}
/**
@@ -155,7 +196,7 @@ export class CloudlySettingsManager {
return { success: false, message: `Unknown provider: ${provider}` };
}
} catch (error) {
return { success: false, message: `Connection test failed: ${error.message}` };
return { success: false, message: `Connection test failed: ${error instanceof Error ? error.message : String(error)}` };
}
}
@@ -191,9 +232,10 @@ export class CloudlySettingsManager {
message: 'Settings updated successfully'
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
return {
success: false,
message: `Failed to update settings: ${error.message}`
message: `Failed to update settings: ${errorMessage}`
};
}
}
@@ -213,9 +255,10 @@ export class CloudlySettingsManager {
message: `Setting ${requestData.key} cleared successfully`
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
return {
success: false,
message: `Failed to clear setting: ${error.message}`
message: `Failed to clear setting: ${errorMessage}`
};
}
}
@@ -252,4 +295,4 @@ export class CloudlySettingsManager {
)
);
}
}
}
+165
View File
@@ -0,0 +1,165 @@
import * as plugins from '../plugins.js';
import { CloudlyTaskManager } from './classes.taskmanager.js';
@plugins.smartdata.managed()
export class TaskExecution extends plugins.smartdata.SmartDataDbDoc<
TaskExecution,
plugins.servezoneInterfaces.data.ITaskExecution,
CloudlyTaskManager
> {
// STATIC
public static async getTaskExecutionById(executionIdArg: string) {
const execution = await this.getInstance({
id: executionIdArg,
});
return execution;
}
public static async getTaskExecutions(filterArg?: {
taskName?: string;
status?: string;
startedAfter?: number;
startedBefore?: number;
}) {
const query: any = {};
if (filterArg?.taskName) {
query['data.taskName'] = filterArg.taskName;
}
if (filterArg?.status) {
query['data.status'] = filterArg.status;
}
if (filterArg?.startedAfter || filterArg?.startedBefore) {
query['data.startedAt'] = {};
if (filterArg.startedAfter) {
query['data.startedAt'].$gte = filterArg.startedAfter;
}
if (filterArg.startedBefore) {
query['data.startedAt'].$lte = filterArg.startedBefore;
}
}
const executions = await this.getInstances(query);
return executions;
}
public static async createTaskExecution(
taskName: string,
triggeredBy: 'schedule' | 'manual' | 'system',
userId?: string
) {
const execution = new TaskExecution();
execution.id = await TaskExecution.getNewId();
execution.data = {
taskName,
startedAt: Date.now(),
status: 'running',
triggeredBy,
userId,
logs: [],
};
await execution.save();
return execution;
}
// INSTANCE
@plugins.smartdata.unI()
public id!: string;
@plugins.smartdata.svDb()
public data!: plugins.servezoneInterfaces.data.ITaskExecution['data'];
/**
* Add a log entry to the execution
*/
public async addLog(message: string, severity: 'info' | 'warning' | 'error' | 'success' = 'info') {
this.data.logs.push({
timestamp: Date.now(),
message,
severity,
});
await this.save();
}
/**
* Set a metric for the execution
*/
public async setMetric(key: string, value: any) {
if (!this.data.metrics) {
this.data.metrics = {};
}
this.data.metrics[key] = value;
await this.save();
}
/**
* Mark the execution as completed
*/
public async complete(result?: any) {
this.data.completedAt = Date.now();
this.data.duration = this.data.completedAt - this.data.startedAt;
this.data.status = 'completed';
if (result !== undefined) {
this.data.result = result;
}
await this.save();
}
/**
* Mark the execution as failed
*/
public async fail(error: Error | string) {
this.data.completedAt = Date.now();
this.data.duration = this.data.completedAt - this.data.startedAt;
this.data.status = 'failed';
if (typeof error === 'string') {
this.data.error = {
message: error,
};
} else {
this.data.error = {
message: error.message,
stack: error.stack,
code: (error as any).code,
};
}
await this.save();
}
/**
* Cancel the execution
*/
public async cancel() {
this.data.completedAt = Date.now();
this.data.duration = this.data.completedAt - this.data.startedAt;
this.data.status = 'cancelled';
await this.save();
}
/**
* Get running executions
*/
public static async getRunningExecutions() {
return await this.getInstances({
'data.status': 'running',
});
}
/**
* Clean up old executions
*/
public static async cleanupOldExecutions(olderThanDays: number = 30) {
const cutoffTime = Date.now() - (olderThanDays * 24 * 60 * 60 * 1000);
const oldExecutions = await this.getInstances({
'data.completedAt': { $lt: cutoffTime },
});
for (const execution of oldExecutions) {
await execution.delete();
}
return oldExecutions.length;
}
}
+353
View File
@@ -0,0 +1,353 @@
import * as plugins from '../plugins.js';
import { Cloudly } from '../classes.cloudly.js';
import { TaskExecution } from './classes.taskexecution.js';
import { createPredefinedTasks } from './predefinedtasks.js';
import { logger } from '../logger.js';
export interface ITaskInfo {
name: string;
description: string;
category: 'maintenance' | 'deployment' | 'backup' | 'monitoring' | 'cleanup' | 'system' | 'security';
schedule?: string; // Cron expression if scheduled
lastRun?: number;
enabled: boolean;
}
export class CloudlyTaskManager {
public typedrouter = new plugins.typedrequest.TypedRouter();
public cloudlyRef: Cloudly;
// TaskBuffer integration
private taskBufferManager = new plugins.taskbuffer.TaskManager();
private taskRegistry = new Map<string, plugins.taskbuffer.Task>();
private taskInfo = new Map<string, ITaskInfo>();
private currentExecutions = new Map<string, TaskExecution>();
private cancellationRequests = new Set<string>();
// Database connection helper
get db() {
return this.cloudlyRef.mongodbConnector.smartdataDb;
}
// Set up TaskExecution document manager
public CTaskExecution = plugins.smartdata.setDefaultManagerForDoc(this, TaskExecution);
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
// Add router to main router
this.cloudlyRef.typedrouter.addTypedRouter(this.typedrouter);
// Set up API endpoints
this.setupApiEndpoints();
// Register predefined tasks
createPredefinedTasks(this);
}
/**
* Register a task with the manager
*/
public registerTask(
name: string,
task: plugins.taskbuffer.Task,
info: Omit<ITaskInfo, 'name' | 'lastRun'>
) {
this.taskRegistry.set(name, task);
this.taskInfo.set(name, {
name,
...info,
lastRun: undefined,
});
// Schedule if cron expression provided
if (info.schedule && info.enabled) {
this.scheduleTask(name, info.schedule);
}
logger.log('info', `Registered task: ${name}`);
}
/**
* Execute a task with tracking
*/
public async executeTask(
taskName: string,
triggeredBy: 'schedule' | 'manual' | 'system',
userId?: string
): Promise<TaskExecution | null> {
const task = this.taskRegistry.get(taskName);
const info = this.taskInfo.get(taskName);
if (!task) {
throw new Error(`Task ${taskName} not found`);
}
if (!info?.enabled && triggeredBy === 'schedule') {
logger.log('warn', `Skipping disabled scheduled task: ${taskName}`);
return null;
}
// Create execution record
const execution = await TaskExecution.createTaskExecution(taskName, triggeredBy, userId);
if (info?.description) {
execution.data.taskDescription = info.description;
}
if (info?.category) {
execution.data.category = info.category;
}
await execution.save();
// Store current execution for task to access
this.currentExecutions.set(taskName, execution);
try {
await execution.addLog(`Starting task: ${taskName}`, 'info');
// Execute the task
const result = await task.trigger();
// If a cancellation was requested during execution, don't mark as completed
if (execution.data.status === 'cancelled' || this.cancellationRequests.has(execution.id)) {
await execution.addLog('Task cancelled during execution', 'warning');
} else {
// Task completed successfully
await execution.complete(result);
await execution.addLog(`Task completed successfully`, 'success');
}
// Update last run time
if (info) {
info.lastRun = Date.now();
}
} catch (error) {
// If already cancelled, don't mark as failed
if (execution.data.status === 'cancelled' || this.cancellationRequests.has(execution.id)) {
await execution.addLog('Task was cancelled', 'warning');
} else {
// Task failed
await execution.fail(error as any);
await execution.addLog(`Task failed: ${(error as any).message}`, 'error');
logger.log('error', `Task ${taskName} failed: ${(error as any).message}`);
}
} finally {
// Clean up current execution
this.currentExecutions.delete(taskName);
this.cancellationRequests.delete(execution.id);
}
return execution;
}
/**
* Get current execution for a task (used by tasks to log)
*/
public getCurrentExecution(taskName: string): TaskExecution | undefined {
return this.currentExecutions.get(taskName);
}
/**
* Schedule a task with cron expression
*/
public scheduleTask(taskName: string, cronExpression: string) {
const task = this.taskRegistry.get(taskName);
if (!task) {
throw new Error(`Task ${taskName} not found`);
}
// Wrap task execution with tracking
const wrappedTask = new plugins.taskbuffer.Task({
name: `${taskName}-scheduled`,
taskFunction: async () => {
await this.executeTask(taskName, 'schedule');
},
});
this.taskBufferManager.addAndScheduleTask(wrappedTask, cronExpression);
logger.log('info', `Scheduled task ${taskName} with cron: ${cronExpression}`);
}
/**
* Cancel a running task
*/
public async cancelTask(executionId: string): Promise<boolean> {
const execution = await TaskExecution.getTaskExecutionById(executionId);
if (!execution || execution.data.status !== 'running') {
return false;
}
await execution.cancel();
await execution.addLog('Task cancelled by user', 'warning');
// mark cancellation request so running task functions can react cooperatively
this.cancellationRequests.add(execution.id);
return true;
}
/**
* Check if cancellation is requested for an execution
*/
public isCancellationRequested(executionId: string): boolean {
return this.cancellationRequests.has(executionId);
}
/**
* Get all registered tasks
*/
public getAllTasks(): ITaskInfo[] {
return Array.from(this.taskInfo.values());
}
/**
* Enable or disable a task
*/
public async setTaskEnabled(taskName: string, enabled: boolean) {
const info = this.taskInfo.get(taskName);
if (!info) {
throw new Error(`Task ${taskName} not found`);
}
info.enabled = enabled;
if (!enabled) {
// TODO: Remove from scheduler if disabled
logger.log('info', `Disabled task: ${taskName}`);
} else if (info.schedule) {
// Reschedule if enabled with schedule
this.scheduleTask(taskName, info.schedule);
}
}
/**
* Set up API endpoints
*/
private setupApiEndpoints() {
// Get all tasks
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.task.IRequest_Any_Cloudly_GetTasks>(
'getTasks',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const tasks = this.getAllTasks();
return {
tasks,
};
}
)
);
// Get task executions
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.task.IRequest_Any_Cloudly_GetTaskExecutions>(
'getTaskExecutions',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const executions = await TaskExecution.getTaskExecutions(reqArg.filter);
return {
executions: await Promise.all(
executions.map(e => e.createSavableObject())
),
};
}
)
);
// Get task execution by ID
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.task.IRequest_Any_Cloudly_GetTaskExecutionById>(
'getTaskExecutionById',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const execution = await TaskExecution.getTaskExecutionById(reqArg.executionId);
if (!execution) {
throw new Error('Task execution not found');
}
return {
execution: await execution.createSavableObject(),
};
}
)
);
// Trigger task manually
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.task.IRequest_Any_Cloudly_TriggerTask>(
'triggerTask',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const execution = await this.executeTask(
reqArg.taskName,
'manual',
reqArg.userId
);
if (!execution) {
throw new Error(`Task ${reqArg.taskName} did not start`);
}
return {
execution: await execution.createSavableObject(),
};
}
)
);
// Cancel task
this.typedrouter.addTypedHandler(
new plugins.typedrequest.TypedHandler<plugins.servezoneInterfaces.requests.task.IRequest_Any_Cloudly_CancelTask>(
'cancelTask',
async (reqArg) => {
await plugins.smartguard.passGuardsOrReject(reqArg, [
this.cloudlyRef.authManager.validIdentityGuard,
]);
const success = await this.cancelTask(reqArg.executionId);
return {
success,
};
}
)
);
}
/**
* Initialize the task manager
*/
public async init() {
logger.log('info', 'Task Manager initialized');
// Clean up old executions on startup
const deletedCount = await TaskExecution.cleanupOldExecutions(30);
if (deletedCount > 0) {
logger.log('info', `Cleaned up ${deletedCount} old task executions`);
}
await this.taskBufferManager.start();
}
/**
* Stop the task manager
*/
public async stop() {
// Stop all scheduled tasks
await this.taskBufferManager.stop();
logger.log('info', 'Task Manager stopped');
}
}
+574
View File
@@ -0,0 +1,574 @@
import * as plugins from '../plugins.js';
import { CloudlyTaskManager } from './classes.taskmanager.js';
import { logger } from '../logger.js';
const getErrorMessage = (errorArg: unknown) => errorArg instanceof Error ? errorArg.message : String(errorArg);
/**
* Create and register all predefined tasks
*/
export function createPredefinedTasks(taskManager: CloudlyTaskManager) {
// Cloudflare Domain Sync Task
const cfDomainSync = new plugins.taskbuffer.Task({
name: 'cloudflare-domain-sync',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('cloudflare-domain-sync');
try {
await execution?.addLog('Starting Cloudflare domain sync…', 'info');
const cf = taskManager.cloudlyRef.cloudflareConnector?.cloudflare;
if (!cf) {
await execution?.addLog('Cloudflare not configured; skipping sync.', 'warning');
return { created: 0, updated: 0, totalZones: 0 };
}
const zones = await cf.convenience.listZones();
await execution?.setMetric('totalZones', zones.length);
await execution?.addLog(`Fetched ${zones.length} zones from Cloudflare`, 'info');
let created = 0;
let updated = 0;
const now = Date.now();
for (const zone of zones) {
// zone fields from Cloudflare typings
const zoneName = (zone as any).name as string;
const zoneId = (zone as any).id as string;
const zoneStatus = ((zone as any).status || 'active') as 'active'|'pending'|'suspended'|'transferred'|'expired';
const nameServers: string[] = (zone as any).name_servers || [];
const existing = await taskManager.cloudlyRef.domainManager.CDomain.getDomainByName(zoneName);
if (existing) {
if (execution && (taskManager.isCancellationRequested(execution.id) || existing.data == null)) {
await execution?.addLog('Cancellation requested. Stopping CF sync…', 'warning');
break;
}
await execution?.addLog(`Updating domain: ${zoneName}`, 'info');
await taskManager.cloudlyRef.domainManager.CDomain.updateDomain(existing.id, {
status: zoneStatus as any,
nameservers: nameServers,
cloudflareZoneId: zoneId,
syncSource: 'cloudflare',
lastSyncAt: now,
activationState: existing.data.activationState || 'available',
});
updated++;
} else {
await execution?.addLog(`Creating domain: ${zoneName}`, 'info');
await taskManager.cloudlyRef.domainManager.CDomain.createDomain({
name: zoneName,
description: `Synced from Cloudflare zone ${zoneId}`,
status: zoneStatus as any,
verificationStatus: 'pending',
nameservers: nameServers,
autoRenew: true,
cloudflareZoneId: zoneId,
activationState: 'available',
syncSource: 'cloudflare',
lastSyncAt: now,
} as any);
created++;
}
}
await execution?.setMetric('created', created);
await execution?.setMetric('updated', updated);
await execution?.addLog(`Cloudflare sync done: ${created} created, ${updated} updated`, 'success');
return { created, updated, totalZones: zones.length };
} catch (error) {
await execution?.addLog(`Cloudflare sync error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('cloudflare-domain-sync', cfDomainSync, {
description: 'Import and update domains from Cloudflare zones',
category: 'system',
schedule: '0 3 * * *', // Daily at 3 AM
enabled: true,
});
// DNS Sync Task
const dnsSync = new plugins.taskbuffer.Task({
name: 'dns-sync',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('dns-sync');
const dnsManager = taskManager.cloudlyRef.dnsManager;
try {
await execution?.addLog('Starting DNS synchronization...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Aborting DNS sync...', 'warning');
return;
}
// Get all DNS entries marked as external
const dnsEntries = await dnsManager.CDnsEntry.getInstances({
'data.sourceType': 'external',
});
await execution?.addLog(`Found ${dnsEntries.length} external DNS entries to sync`, 'info');
await execution?.setMetric('totalEntries', dnsEntries.length);
let syncedCount = 0;
let failedCount = 0;
for (const entry of dnsEntries) {
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Stopping loop...', 'warning');
break;
}
try {
// TODO: Implement actual sync with external DNS provider
await execution?.addLog(`Syncing DNS entry: ${entry.data.name}.${entry.data.zone}`, 'info');
syncedCount++;
} catch (error) {
await execution?.addLog(`Failed to sync ${entry.data.name}: ${getErrorMessage(error)}`, 'warning');
failedCount++;
}
}
await execution?.setMetric('syncedCount', syncedCount);
await execution?.setMetric('failedCount', failedCount);
await execution?.addLog(`DNS sync completed: ${syncedCount} synced, ${failedCount} failed`, 'success');
return { synced: syncedCount, failed: failedCount };
} catch (error) {
await execution?.addLog(`DNS sync error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('dns-sync', dnsSync, {
description: 'Synchronize DNS entries with external providers',
category: 'system',
schedule: '0 */6 * * *', // Every 6 hours
enabled: true,
});
// Certificate Renewal Task
const certRenewal = new plugins.taskbuffer.Task({
name: 'cert-renewal',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('cert-renewal');
try {
await execution?.addLog('Checking certificates for renewal...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Aborting certificate renewal...', 'warning');
return;
}
// Get all domains (only activated ones are considered for renewal)
const domains = await taskManager.cloudlyRef.domainManager.CDomain.getInstances({
'data.activationState': 'activated',
} as any);
await execution?.setMetric('totalDomains', domains.length);
let renewedCount = 0;
let upToDateCount = 0;
for (const domain of domains) {
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Stopping loop...', 'warning');
break;
}
// TODO: Check certificate expiry and renew if needed
await execution?.addLog(`Checking certificate for ${domain.data.name}`, 'info');
// Placeholder logic
const needsRenewal = Math.random() > 0.8; // 20% chance for demo
if (needsRenewal) {
await execution?.addLog(`Renewing certificate for ${domain.data.name}`, 'info');
// TODO: Actual renewal logic
renewedCount++;
} else {
upToDateCount++;
}
}
await execution?.setMetric('renewedCount', renewedCount);
await execution?.setMetric('upToDateCount', upToDateCount);
await execution?.addLog(`Certificate check completed: ${renewedCount} renewed, ${upToDateCount} up to date`, 'success');
return { renewed: renewedCount, upToDate: upToDateCount };
} catch (error) {
await execution?.addLog(`Certificate renewal error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('cert-renewal', certRenewal, {
description: 'Check and renew SSL certificates',
category: 'security',
schedule: '0 2 * * *', // Daily at 2 AM
enabled: true,
});
// Cleanup Task
const cleanup = new plugins.taskbuffer.Task({
name: 'cleanup',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('cleanup');
try {
await execution?.addLog('Starting cleanup tasks...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Aborting cleanup...', 'warning');
return;
}
// Clean up old task executions
await execution?.addLog('Cleaning old task executions...', 'info');
const deletedExecutions = await taskManager.CTaskExecution.cleanupOldExecutions(30);
await execution?.setMetric('deletedExecutions', deletedExecutions);
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) return;
// TODO: Clean up old logs
await execution?.addLog('Cleaning old logs...', 'info');
// Placeholder
const deletedLogs = 0;
await execution?.setMetric('deletedLogs', deletedLogs);
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) return;
// TODO: Clean up Docker images
await execution?.addLog('Cleaning unused Docker images...', 'info');
// Placeholder
const deletedImages = 0;
await execution?.setMetric('deletedImages', deletedImages);
await execution?.addLog(`Cleanup completed: ${deletedExecutions} executions, ${deletedLogs} logs, ${deletedImages} images deleted`, 'success');
return {
executions: deletedExecutions,
logs: deletedLogs,
images: deletedImages,
};
} catch (error) {
await execution?.addLog(`Cleanup error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('cleanup', cleanup, {
description: 'Remove old logs, executions, and temporary files',
category: 'cleanup',
schedule: '0 3 * * *', // Daily at 3 AM
enabled: true,
});
// Health Check Task
const healthCheck = new plugins.taskbuffer.Task({
name: 'health-check',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('health-check');
try {
await execution?.addLog('Starting health checks...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Aborting health checks...', 'warning');
return;
}
// Check all deployments
const deployments = await taskManager.cloudlyRef.deploymentManager.getAllDeployments();
await execution?.setMetric('totalDeployments', deployments.length);
let healthyCount = 0;
let unhealthyCount = 0;
const issues: Array<{ deploymentId: string; serviceId: string; issue: string }> = [];
for (const deployment of deployments) {
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Stopping loop...', 'warning');
break;
}
if (deployment.status === 'running') {
// TODO: Actual health check logic
const isHealthy = Math.random() > 0.1; // 90% healthy for demo
if (isHealthy) {
healthyCount++;
} else {
unhealthyCount++;
issues.push({
deploymentId: deployment.id,
serviceId: deployment.serviceId,
issue: 'Health check failed',
});
await execution?.addLog(`Deployment ${deployment.id} is unhealthy`, 'warning');
}
}
}
await execution?.setMetric('healthyCount', healthyCount);
await execution?.setMetric('unhealthyCount', unhealthyCount);
await execution?.setMetric('issues', issues);
const severity = unhealthyCount > 0 ? 'warning' : 'success';
await execution?.addLog(
`Health check completed: ${healthyCount} healthy, ${unhealthyCount} unhealthy`,
severity as any
);
return { healthy: healthyCount, unhealthy: unhealthyCount, issues };
} catch (error) {
await execution?.addLog(`Health check error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('health-check', healthCheck, {
description: 'Monitor service health across deployments',
category: 'monitoring',
schedule: '*/15 * * * *', // Every 15 minutes
enabled: true,
});
// Resource Usage Report
const resourceReport = new plugins.taskbuffer.Task({
name: 'resource-report',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('resource-report');
try {
await execution?.addLog('Generating resource usage report...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Aborting report...', 'warning');
return;
}
// Get all nodes
const nodes = await taskManager.cloudlyRef.nodeManager.CClusterNode.getInstances({});
const report: {
timestamp: number;
nodes: Array<{ nodeId: string; nodeName: string; cpu: number; memory: number; disk: number }>;
totalCpu: number;
totalMemory: number;
totalDisk: number;
} = {
timestamp: Date.now(),
nodes: [],
totalCpu: 0,
totalMemory: 0,
totalDisk: 0,
};
for (const node of nodes) {
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Stopping loop...', 'warning');
break;
}
// TODO: Get actual resource usage
const nodeUsage = {
nodeId: node.id,
nodeName: node.data.swarmNodeId || node.id,
cpu: Math.random() * 100, // Placeholder
memory: Math.random() * 100, // Placeholder
disk: Math.random() * 100, // Placeholder
};
report.nodes.push(nodeUsage);
report.totalCpu += nodeUsage.cpu;
report.totalMemory += nodeUsage.memory;
report.totalDisk += nodeUsage.disk;
}
// Calculate averages
if (nodes.length > 0) {
report.totalCpu /= nodes.length;
report.totalMemory /= nodes.length;
report.totalDisk /= nodes.length;
}
await execution?.setMetric('report', report);
await execution?.addLog(
`Resource report generated: Avg CPU ${report.totalCpu.toFixed(1)}%, Memory ${report.totalMemory.toFixed(1)}%, Disk ${report.totalDisk.toFixed(1)}%`,
'success'
);
return report;
} catch (error) {
await execution?.addLog(`Resource report error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('resource-report', resourceReport, {
description: 'Generate resource usage reports',
category: 'monitoring',
schedule: '0 * * * *', // Every hour
enabled: true,
});
// Database Maintenance
const dbMaintenance = new plugins.taskbuffer.Task({
name: 'db-maintenance',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('db-maintenance');
try {
await execution?.addLog('Starting database maintenance...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Aborting DB maintenance...', 'warning');
return;
}
// TODO: Implement actual database maintenance
await execution?.addLog('Analyzing indexes...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) return;
await execution?.addLog('Compacting collections...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) return;
await execution?.addLog('Updating statistics...', 'info');
await execution?.setMetric('collectionsOptimized', 5); // Placeholder
await execution?.setMetric('indexesRebuilt', 3); // Placeholder
await execution?.addLog('Database maintenance completed', 'success');
return { success: true };
} catch (error) {
await execution?.addLog(`Database maintenance error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('db-maintenance', dbMaintenance, {
description: 'Optimize database performance',
category: 'maintenance',
schedule: '0 4 * * 0', // Weekly on Sunday at 4 AM
enabled: true,
});
// Security Scan
const securityScan = new plugins.taskbuffer.Task({
name: 'security-scan',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('security-scan');
try {
await execution?.addLog('Starting security scan...', 'info');
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Aborting security scan...', 'warning');
return;
}
const vulnerabilities: Array<{ type: string; severity: string; image: string; version: string }> = [];
// Check for exposed ports
await execution?.addLog('Checking for exposed ports...', 'info');
// TODO: Actual port scanning logic
// Check for outdated images
await execution?.addLog('Checking for outdated images...', 'info');
const images = await taskManager.cloudlyRef.imageManager.CImage.getInstances({});
for (const image of images) {
if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) {
await execution.addLog('Cancellation requested. Stopping loop...', 'warning');
break;
}
// TODO: Check if image is outdated
const isOutdated = Math.random() > 0.7; // 30% outdated for demo
if (isOutdated) {
vulnerabilities.push({
type: 'outdated-image',
severity: 'medium',
image: image.data.name,
version: image.data.versions[0]?.versionString || 'unknown',
});
}
}
// Check for weak passwords
await execution?.addLog('Checking for weak configurations...', 'info');
// TODO: Configuration checks
await execution?.setMetric('vulnerabilitiesFound', vulnerabilities.length);
await execution?.setMetric('vulnerabilities', vulnerabilities);
const severity = vulnerabilities.length > 0 ? 'warning' : 'success';
await execution?.addLog(
`Security scan completed: ${vulnerabilities.length} vulnerabilities found`,
severity as any
);
return { vulnerabilities };
} catch (error) {
await execution?.addLog(`Security scan error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('security-scan', securityScan, {
description: 'Run security checks on services',
category: 'security',
schedule: '0 1 * * *', // Daily at 1 AM
enabled: true,
});
// Docker Cleanup
const dockerCleanup = new plugins.taskbuffer.Task({
name: 'docker-cleanup',
taskFunction: async () => {
const execution = taskManager.getCurrentExecution('docker-cleanup');
try {
await execution?.addLog('Starting Docker cleanup...', 'info');
// TODO: Implement actual Docker cleanup
await execution?.addLog('Removing stopped containers...', 'info');
const removedContainers = 0; // Placeholder
await execution?.addLog('Removing unused images...', 'info');
const removedImages = 0; // Placeholder
await execution?.addLog('Removing unused volumes...', 'info');
const removedVolumes = 0; // Placeholder
await execution?.addLog('Removing unused networks...', 'info');
const removedNetworks = 0; // Placeholder
await execution?.setMetric('removedContainers', removedContainers);
await execution?.setMetric('removedImages', removedImages);
await execution?.setMetric('removedVolumes', removedVolumes);
await execution?.setMetric('removedNetworks', removedNetworks);
await execution?.addLog(
`Docker cleanup completed: ${removedContainers} containers, ${removedImages} images, ${removedVolumes} volumes, ${removedNetworks} networks removed`,
'success'
);
return {
containers: removedContainers,
images: removedImages,
volumes: removedVolumes,
networks: removedNetworks,
};
} catch (error) {
await execution?.addLog(`Docker cleanup error: ${getErrorMessage(error)}`, 'error');
throw error;
}
},
});
taskManager.registerTask('docker-cleanup', dockerCleanup, {
description: 'Remove unused Docker images and containers',
category: 'cleanup',
schedule: '0 5 * * *', // Daily at 5 AM
enabled: true,
});
logger.log('info', 'Predefined tasks registered successfully');
}
-35
View File
@@ -1,35 +0,0 @@
import * as plugins from '../plugins.js';
import { Cloudly } from '../classes.cloudly.js';
import { logger } from '../logger.js';
export class CloudlyTaskmanager {
public cloudlyRef: Cloudly;
constructor(cloudlyRefArg: Cloudly) {
this.cloudlyRef = cloudlyRefArg;
}
public everyMinuteTask = new plugins.taskbuffer.Task({
taskFunction: async () => {},
});
public everyHourTask = new plugins.taskbuffer.Task({
taskFunction: async () => {
logger.log('info', `Performing hourly maintenance check.`);
const configs = await this.cloudlyRef.clusterManager.getAllClusters();
logger.log('info', `Got ${configs.length} configs`);
configs.map((configArg) => {
console.log(configArg.name);
});
},
});
public everyDayTask = new plugins.taskbuffer.Task({
taskFunction: async () => {},
});
public everyWeekTask = new plugins.taskbuffer.Task({
taskFunction: async () => {},
});
}
+10 -3
View File
@@ -1,7 +1,10 @@
// node native
import * as path from 'path';
import * as crypto from 'node:crypto';
import * as stream from 'stream';
import * as fsPromises from 'node:fs/promises';
export { path };
export { path, crypto, stream, fsPromises };
// @apiglobal scope
import * as typedrequest from '@api.global/typedrequest';
@@ -23,7 +26,7 @@ import * as tsclass from '@tsclass/tsclass';
export { tsclass };
// @push.rocks scope
import * as npmextra from '@push.rocks/npmextra';
import * as smartconfig from '@push.rocks/smartconfig';
import * as projectinfo from '@push.rocks/projectinfo';
import * as qenv from '@push.rocks/qenv';
import * as smartacme from '@push.rocks/smartacme';
@@ -42,6 +45,8 @@ import * as smartlog from '@push.rocks/smartlog';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartregistry from '@push.rocks/smartregistry';
import * as smartsamba from '@push.rocks/smartsamba';
import * as smartssh from '@push.rocks/smartssh';
import * as smartstream from '@push.rocks/smartstream';
import * as smartstring from '@push.rocks/smartstring';
@@ -50,7 +55,7 @@ import * as taskbuffer from '@push.rocks/taskbuffer';
import * as typedserver from '@api.global/typedserver';
export {
npmextra,
smartconfig,
projectinfo,
qenv,
smartacme,
@@ -68,7 +73,9 @@ export {
smartlog,
smartpath,
smartpromise,
smartregistry,
smartrequest,
smartsamba,
smartssh,
smartstream,
smartstring,
-240
View File
@@ -1,240 +0,0 @@
import * as plugins from './plugins.js';
export type TClientType = 'api' | 'ci' | 'coreflow' | 'cli' | 'serverconfig';
import { Image } from './classes.image.js';
import { Service } from './classes.service.js';
import { Cluster } from './classes.cluster.js';
import { SecretBundle } from './classes.secretbundle.js';
import { SecretGroup } from './classes.secretgroup.js';
import { ExternalRegistry } from './classes.externalregistry.js';
export class CloudlyApiClient {
private cloudlyUrl: string;
private registerAs: string;
public typedrouter = new plugins.typedrequest.TypedRouter();
public typedsocketClient: plugins.typedsocket.TypedSocket;
// Subjects
public configUpdateSubject = new plugins.smartrx.rxjs.Subject<
plugins.servezoneInterfaces.requests.config.IRequest_Cloudly_Coreflow_PushClusterConfig['request']
>();
public serverActionSubject = new plugins.smartrx.rxjs.Subject<
plugins.servezoneInterfaces.requests.server.IRequest_TriggerServerAction['request']
>();
constructor(optionsArg: {
registerAs: TClientType;
cloudlyUrl?: string;
}) {
this.registerAs = optionsArg.registerAs;
this.cloudlyUrl =
optionsArg?.cloudlyUrl || process.env.CLOUDLY_URL || 'https://cloudly.layer.io:443';
console.log(
`creating LoleCloudlyClient: registering as ${this.registerAs} and target url ${this.cloudlyUrl}`
);
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.config.IRequest_Cloudly_Coreflow_PushClusterConfig>(
new plugins.typedrequest.TypedHandler('pushClusterConfig', async (dataArg) => {
this.configUpdateSubject.next(dataArg);
return {};
})
);
this.typedrouter.addTypedHandler<plugins.servezoneInterfaces.requests.server.IRequest_TriggerServerAction>(
new plugins.typedrequest.TypedHandler('triggerServerAction', async (dataArg) => {
this.serverActionSubject.next(dataArg);
return {
actionConfirmed: true,
};
})
);
}
public async start() {
this.typedsocketClient = await plugins.typedsocket.TypedSocket.createClient(
this.typedrouter,
this.cloudlyUrl
);
console.log(
`CloudlyClient connected to cloudly at ${this.cloudlyUrl}. Remember to get an identity.`
);
}
public async stop() {
await this.typedsocketClient.stop();
}
public identity: plugins.servezoneInterfaces.data.IIdentity;
public async getIdentityByToken(
token: string,
optionsArg?: {
tagConnection?: boolean;
statefullIdentity?: boolean;
}
): Promise<plugins.servezoneInterfaces.data.IIdentity> {
optionsArg = Object.assign({}, {
tagConnection: false,
statefullIdentity: true,
}, optionsArg);
const identityRequest =
this.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.identity.IRequest_Any_Cloudly_CoreflowManager_GetIdentityByToken>(
'getIdentityByToken'
);
console.log(`trying to get identity from cloudly with supplied jumpCodeArg: ${token}`);
const response = await identityRequest.fire({
token: token,
});
console.log('got identity response');
const identity = response.identity;
if (optionsArg.tagConnection) {
this.typedsocketClient.addTag('identity', identity);
}
if (optionsArg.statefullIdentity) {
this.identity = identity;
}
return identity;
}
/**
* will use statefull identity by default
*/
public async getClusterConfigFromCloudlyByIdentity(
identityArg: plugins.servezoneInterfaces.data.IIdentity = this.identity
): Promise<plugins.servezoneInterfaces.data.ICluster> {
const clusterConfigRequest =
this.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.config.IRequest_Any_Cloudly_GetClusterConfig>(
'getClusterConfig'
);
const response = await clusterConfigRequest.fire({
identity: identityArg,
});
return response.configData;
}
/**
* will use statefull identity by default
*/
public async getServerConfigFromCloudlyByIdentity(
identityArg: plugins.servezoneInterfaces.data.IIdentity = this.identity
): Promise<plugins.servezoneInterfaces.data.IServer> {
const serverConfigRequest =
this.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.config.IRequest_Any_Cloudly_GetServerConfig>(
'getServerConfig'
);
const response = await serverConfigRequest.fire({
identity: identityArg,
serverId: '', // TODO: get server id here
});
return response.configData;
}
/**
* gets a certificate for a domain used by a service
*/
public async getCertificateForDomain(optionsArg: {
domainName: string;
type: plugins.servezoneInterfaces.requests.certificate.IRequest_Any_Cloudly_GetCertificateForDomain['request']['type'];
identity?: plugins.servezoneInterfaces.data.IIdentity;
}): Promise<plugins.tsclass.network.ICert> {
optionsArg.identity = optionsArg.identity || this.identity;
if (!optionsArg.identity) {
throw new Error('identity is required. Either provide one or login first.');
}
const typedCertificateRequest =
this.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.certificate.IRequest_Any_Cloudly_GetCertificateForDomain>(
'getCertificateForDomain'
);
const typedResponse = await typedCertificateRequest.fire({
identity: this.identity, // do proper auth here
domainName: optionsArg.domainName,
type: optionsArg.type,
});
return typedResponse.certificate;
}
public externalRegistry = {
// ExternalRegistry
getRegistryById: async (registryNameArg: string) => {
return ExternalRegistry.getExternalRegistryById(this, registryNameArg);
},
getRegistries: async () => {
return ExternalRegistry.getExternalRegistries(this);
},
createRegistry: async (optionsArg: Parameters<typeof ExternalRegistry.createExternalRegistry>[1]) => {
return ExternalRegistry.createExternalRegistry(this, optionsArg);
}
}
public image = {
// Images
getImageById: async (imageIdArg: string) => {
return Image.getImageById(this, imageIdArg);
},
getImages: async () => {
return Image.getImages(this);
},
createImage: async (optionsArg: Parameters<typeof Image.createImage>[1]) => {
return Image.createImage(this, optionsArg);
}
}
public services = {
// Services
getServiceById: async (serviceIdArg: string) => {
return Service.getServiceById(this, serviceIdArg);
},
getServices: async () => {
return Service.getServices(this);
},
createService: async (optionsArg: Parameters<typeof Service.createService>[1]) => {
return Service.createService(this, optionsArg);
}
}
public cluster = {
// Clusters
getClusterById: async (clusterIdArg: string) => {
return Cluster.getClusterById(this, clusterIdArg);
},
getClusters: async () => {
return Cluster.getClusters(this);
},
createCluster: async (optionsArg: Parameters<typeof Cluster.createCluster>[1]) => {
return Cluster.createCluster(this, optionsArg);
}
}
public secretbundle = {
// SecretBundles
getSecretBundleById: async (secretBundleIdArg: string) => {
return SecretBundle.getSecretBundleById(this, secretBundleIdArg);
},
getSecretBundles: async () => {
return SecretBundle.getSecretBundles(this);
},
createSecretBundle: async (optionsArg: Parameters<typeof SecretBundle.createSecretBundle>[1]) => {
return SecretBundle.createSecretBundle(this, optionsArg);
}
}
public secretgroup = {
// SecretGroups
getSecretGroupById: async (secretGroupIdArg: string) => {
return SecretGroup.getSecretGroupById(this, secretGroupIdArg);
},
getSecretGroups: async () => {
return SecretGroup.getSecretGroups(this);
},
createSecretGroup: async (optionsArg: Parameters<typeof SecretGroup.createSecretGroup>[1]) => {
return SecretGroup.createSecretGroup(this, optionsArg);
}
}
}
-84
View File
@@ -1,84 +0,0 @@
import { CloudlyApiClient } from './classes.cloudlyapiclient.js';
import * as plugins from './plugins.js';
export class Cluster implements plugins.servezoneInterfaces.data.ICluster {
// STATIC
public static async getClusterById(cloudlyClientRef: CloudlyApiClient, clusterIdArg: string) {
const getClusterByIdTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.cluster.IReq_Any_Cloudly_GetClusterById>(
'getClusterById'
);
const response = await getClusterByIdTR.fire({
identity: cloudlyClientRef.identity,
clusterId: clusterIdArg,
});
const newCluster = new Cluster(cloudlyClientRef);
Object.assign(newCluster, response.cluster);
return newCluster;
}
public static async getClusters(cloudlyClientRef: CloudlyApiClient) {
const getClustersTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.cluster.IReq_Any_Cloudly_GetClusters>(
'getClusters'
);
const response = await getClustersTR.fire({
identity: cloudlyClientRef.identity,
});
const clusterConfigs: Cluster[] = [];
for (const clusterConfig of response.clusters) {
const newCluster = new Cluster(cloudlyClientRef);
Object.assign(newCluster, clusterConfig);
clusterConfigs.push(newCluster);
}
return clusterConfigs;
}
public static async createCluster(cloudlyClientRef: CloudlyApiClient, clusterNameArg: string) {
const createClusterTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.cluster.IRequest_CreateCluster>(
'createCluster'
);
const response = await createClusterTR.fire({
identity: cloudlyClientRef.identity,
clusterName: clusterNameArg,
});
const newCluster = new Cluster(cloudlyClientRef);
Object.assign(newCluster, response.cluster);
return newCluster;
}
// INSTANCE
public id: string;
public data: plugins.servezoneInterfaces.data.ICluster['data'];
public cloudlyClientRef: CloudlyApiClient;
constructor(cloudlyClientRef: CloudlyApiClient) {
this.cloudlyClientRef = cloudlyClientRef;
}
public async update() {
const updateClusterTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.cluster.IReq_Any_Cloudly_UpdateCluster>(
'updateCluster'
);
const response = await updateClusterTR.fire({
identity: this.cloudlyClientRef.identity,
clusterData: this.data,
});
const resultClusterData = response.resultCluster.data;
plugins.smartexpect.expect(resultClusterData).toEqual(this.data);
return this;
}
public async delete(cloudlyClientRef: CloudlyApiClient, clusterIdArg: string) {
const deleteClusterTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.cluster.IReq_Any_Cloudly_DeleteClusterById>(
'deleteClusterById'
);
const response = await deleteClusterTR.fire({
identity: cloudlyClientRef.identity,
clusterId: this.id,
});
plugins.smartexpect.expect(response.ok).toBeTrue();
return null;
}
}
-83
View File
@@ -1,83 +0,0 @@
import * as plugins from './plugins.js';
import type { CloudlyApiClient } from './classes.cloudlyapiclient.js';
export class ExternalRegistry implements plugins.servezoneInterfaces.data.IExternalRegistry {
// STATIC
public static async getExternalRegistryById(cloudlyClientRef: CloudlyApiClient, registryNameArg: string) {
const getRegistryByIdTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.externalRegistry.IReq_GetRegistryById>(
'getExternalRegistryById'
);
const response = await getRegistryByIdTR.fire({
identity: cloudlyClientRef.identity,
id: registryNameArg,
});
const newRegistry = new ExternalRegistry(cloudlyClientRef);
Object.assign(newRegistry, response.registry);
return newRegistry;
}
public static async getExternalRegistries(cloudlyClientRef: CloudlyApiClient) {
const getRegistriesTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.externalRegistry.IReq_GetRegistries>(
'getExternalRegistries'
);
const response = await getRegistriesTR.fire({
identity: cloudlyClientRef.identity,
});
const registryConfigs: ExternalRegistry[] = [];
for (const registryConfig of response.registries) {
const newRegistry = new ExternalRegistry(cloudlyClientRef);
Object.assign(newRegistry, registryConfig);
registryConfigs.push(newRegistry);
}
return registryConfigs;
}
public static async createExternalRegistry(cloudlyClientRef: CloudlyApiClient, registryDataArg: Partial<plugins.servezoneInterfaces.data.IExternalRegistry['data']>) {
const createRegistryTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.externalRegistry.IReq_CreateRegistry>(
'createExternalRegistry'
);
const response = await createRegistryTR.fire({
identity: cloudlyClientRef.identity,
registryData: registryDataArg as plugins.servezoneInterfaces.data.IExternalRegistry['data'],
});
const newRegistry = new ExternalRegistry(cloudlyClientRef);
Object.assign(newRegistry, response.registry);
return newRegistry;
}
// INSTANCE
public id: string;
public data: plugins.servezoneInterfaces.data.IExternalRegistry['data'];
public cloudlyClientRef: CloudlyApiClient;
constructor(cloudlyClientRef: CloudlyApiClient) {
this.cloudlyClientRef = cloudlyClientRef;
}
public async update() {
const updateRegistryTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.externalRegistry.IReq_UpdateRegistry>(
'updateExternalRegistry'
);
const response = await updateRegistryTR.fire({
identity: this.cloudlyClientRef.identity,
registryData: this.data,
});
const resultRegistryData = response.resultRegistry.data;
plugins.smartexpect.expect(resultRegistryData).toEqual(this.data);
return this;
}
public async delete(cloudlyClientRef: CloudlyApiClient, registryIdArg: string) {
const deleteRegistryTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.externalRegistry.IReq_DeleteRegistryById>(
'deleteExternalRegistryById'
);
const response = await deleteRegistryTR.fire({
identity: cloudlyClientRef.identity,
registryId: this.id,
});
plugins.smartexpect.expect(response.ok).toBeTrue();
return null;
}
}
-113
View File
@@ -1,113 +0,0 @@
import type { CloudlyApiClient } from './classes.cloudlyapiclient.js';
import * as plugins from './plugins.js';
export class Image implements plugins.servezoneInterfaces.data.IImage {
public static async getImages(cloudlyClientRef: CloudlyApiClient) {
const getAllImagesTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.image.IRequest_GetAllImages>(
'getAllImages'
);
const response = await getAllImagesTR.fire({
identity: cloudlyClientRef.identity,
});
const resultImages: Image[] = [];
for (const image of response.images) {
const newImage = new Image(cloudlyClientRef);
Object.assign(newImage, image);
resultImages.push(newImage);
}
return resultImages;
}
public static async getImageById(cloudlyClientRef: CloudlyApiClient, imageIdArg: string) {
const getImageByIdTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.image.IRequest_GetImage>(
'getImage'
);
const response = await getImageByIdTR.fire({
identity: cloudlyClientRef.identity,
imageId: imageIdArg,
});
const newImage = new Image(cloudlyClientRef);
Object.assign(newImage, response.image);
return newImage;
}
/**
* creates a new image
*/
public static async createImage(cloudlyClientRef: CloudlyApiClient, imageDataArg: Partial<plugins.servezoneInterfaces.data.IImage['data']>) {
const createImageTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.image.IRequest_CreateImage>(
'createImage'
);
const response = await createImageTR.fire({
identity: cloudlyClientRef.identity,
name: imageDataArg.name,
description: imageDataArg.description,
});
const newImage = new Image(cloudlyClientRef);
Object.assign(newImage, response.image);
return newImage;
}
// INSTANCE
cloudlyClientRef: CloudlyApiClient;
id: plugins.servezoneInterfaces.data.IImage['id'];
data: plugins.servezoneInterfaces.data.IImage['data'];
constructor(cloudlyClientRef: CloudlyApiClient) {
this.cloudlyClientRef = cloudlyClientRef;
}
/**
* updates the image data
*/
public async update() {
const getVersionsTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.image.IRequest_GetImage>(
'getImage'
);
const response = await getVersionsTR.fire({
identity: this.cloudlyClientRef.identity,
imageId: this.id,
});
Object.assign(this, response.image);
}
/**
* pushes a new version of the image
* @param imageVersion
* @param imageReadableArg
*/
public async pushImageVersion(imageVersion: string, imageReadableArg: ReadableStream<Uint8Array>): Promise<void> {
const done = plugins.smartpromise.defer();
const pushImageTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.image.IRequest_PushImageVersion>(
'pushImageVersion'
);
const virtualStream = new plugins.typedrequest.VirtualStream();
const response = await pushImageTR.fire({
identity: this.cloudlyClientRef.identity,
imageId: this.id,
versionString: '',
imageStream: virtualStream,
});
await virtualStream.readFromWebstream(imageReadableArg);
await this.update();
};
/**
* pulls a version of the image
*/
public async pullImageVersion(versionStringArg: string): Promise<ReadableStream<Uint8Array>> {
const pullImageTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.image.IRequest_PullImageVersion>(
'pullImageVersion'
);
const response = await pullImageTR.fire({
identity: this.cloudlyClientRef.identity,
imageId: this.id,
versionString: versionStringArg,
});
const imageStream = response.imageStream;
const webduplexStream = new plugins.webstream.WebDuplexStream({});
imageStream.writeToWebstream(webduplexStream.writable);
return webduplexStream.readable;
};
}
-135
View File
@@ -1,135 +0,0 @@
import * as plugins from './plugins.js';
import type { CloudlyApiClient } from './classes.cloudlyapiclient.js';
import { SecretGroup } from './classes.secretgroup.js';
export class SecretBundle implements plugins.servezoneInterfaces.data.ISecretBundle {
// STATIC
public static async getSecretBundleById(cloudlyClientRef: CloudlyApiClient, secretBundleIdArg: string) {
const getSecretBundleByIdTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretbundle.IReq_GetSecretBundleById>(
'getSecretBundleById'
);
const response = await getSecretBundleByIdTR.fire({
identity: cloudlyClientRef.identity,
secretBundleId: secretBundleIdArg,
});
const newSecretBundle = new SecretBundle(cloudlyClientRef);
Object.assign(newSecretBundle, response.secretBundle);
return newSecretBundle;
}
public static async getSecretBundleByAuthorization(cloudlyClientRef: CloudlyApiClient, secretBundleAuthorizationArg: plugins.servezoneInterfaces.data.ISecretBundleAuthorization) {
const getSecretBundleByAuthorizationTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretbundle.IReq_GetSecretBundleByAuthorization>(
'getSecretBundleByAuthorization'
);
const response = await getSecretBundleByAuthorizationTR.fire({
identity: cloudlyClientRef.identity,
secretBundleAuthorization: secretBundleAuthorizationArg,
});
const newSecretBundle = new SecretBundle(cloudlyClientRef);
Object.assign(newSecretBundle, response.secretBundle);
return newSecretBundle;
}
public static async getSecretBundles(cloudlyClientRef: CloudlyApiClient) {
const getSecretBundlesTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretbundle.IReq_GetSecretBundles>(
'getSecretBundles'
);
const response = await getSecretBundlesTR.fire({
identity: cloudlyClientRef.identity,
});
const secretBundles: SecretBundle[] = [];
for (const secretBundle of response.secretBundles) {
const newSecretBundle = new SecretBundle(cloudlyClientRef);
Object.assign(newSecretBundle, secretBundle);
secretBundles.push(newSecretBundle);
}
return secretBundles;
}
public static async createSecretBundle(cloudlyClientRef: CloudlyApiClient, secretBundleDataArg: Partial<plugins.servezoneInterfaces.data.ISecretBundle['data']>) {
const createSecretBundleTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretbundle.IReq_CreateSecretBundle>(
'createSecretBundle'
);
const response = await createSecretBundleTR.fire({
identity: cloudlyClientRef.identity,
secretBundle: {
id: null,
data: {
name: secretBundleDataArg.name,
description: secretBundleDataArg.description,
type: secretBundleDataArg.type,
authorizations: secretBundleDataArg.authorizations,
imageClaims: secretBundleDataArg.imageClaims,
includedSecretGroupIds: secretBundleDataArg.includedSecretGroupIds,
includedTags: secretBundleDataArg.includedTags,
},
},
});
const newSecretBundle = new SecretBundle(cloudlyClientRef);
Object.assign(newSecretBundle, response.resultSecretBundle);
return newSecretBundle;
}
// INSTANCE
public cloudlyClientRef: CloudlyApiClient;
public id: string;
public data: plugins.servezoneInterfaces.data.ISecretBundle['data'];
constructor(cloudlyClientRef: CloudlyApiClient) {
this.cloudlyClientRef = cloudlyClientRef;
}
public async update() {
const updateSecretBundleTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretbundle.IReq_UpdateSecretBundle>(
'updateSecretBundle'
);
const response = await updateSecretBundleTR.fire({
identity: this.cloudlyClientRef.identity,
secretBundle: {
id: this.id,
data: this.data,
},
});
const resultSecretBundleData = response.resultSecretBundle.data;
plugins.smartexpect.expect(resultSecretBundleData).toEqual(this.data);
return this;
}
public async delete(cloudlyClientRef: CloudlyApiClient, secretBundleIdArg: string) {
const deleteSecretBundleTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretbundle.IReq_DeleteSecretBundleById>(
'deleteSecretBundleById'
);
const response = await deleteSecretBundleTR.fire({
identity: cloudlyClientRef.identity,
secretBundleId: this.id,
});
plugins.smartexpect.expect(response.ok).toBeTrue();
return null;
}
public async getFlatKeyValueObjectForEnvironment(environmentArg: string = 'production') {
const bundleAuthorization = this.data.authorizations.find(authorization => {
return authorization.environment === environmentArg;
});
if (bundleAuthorization) {
throw new Error(`no matching environment >>${environmentArg} found in secret bundle`);
}
const getFlatKeyValueObjectTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretbundle.IReq_GetFlatKeyValueObject>(
'getFlatKeyValueObject'
);
const response = await getFlatKeyValueObjectTR.fire({
identity: this.cloudlyClientRef.identity,
seccretBundleId: this.id,
secretBundleAuthorization: bundleAuthorization,
});
const flatKeyValueObject: {[key: string]: string} = response.flatKeyValueObject;
return flatKeyValueObject;
}
}
-96
View File
@@ -1,96 +0,0 @@
import * as plugins from './plugins.js';
import type { CloudlyApiClient } from './classes.cloudlyapiclient.js';
export class SecretGroup implements plugins.servezoneInterfaces.data.ISecretGroup {
public cloudlyClientRef: CloudlyApiClient;
public id: string;
public data: plugins.servezoneInterfaces.data.ISecretGroup['data'];
constructor(cloudlyClientRef: CloudlyApiClient) {
this.cloudlyClientRef = cloudlyClientRef;
}
public static async getSecretGroupById(cloudlyClientRef: CloudlyApiClient, secretGroupIdArg: string) {
const getSecretGroupByIdTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretgroup.IReq_GetSecretGroupById>(
'getSecretGroupById'
);
const response = await getSecretGroupByIdTR.fire({
identity: cloudlyClientRef.identity,
secretGroupId: secretGroupIdArg,
});
const newSecretGroup = new SecretGroup(cloudlyClientRef);
Object.assign(newSecretGroup, response.secretGroup);
return newSecretGroup;
}
public static async getSecretGroups(cloudlyClientRef: CloudlyApiClient) {
const getSecretGroupsTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretgroup.IReq_GetSecretGroups>(
'getSecretGroups'
);
const response = await getSecretGroupsTR.fire({
identity: cloudlyClientRef.identity,
});
const secretGroups: SecretGroup[] = [];
for (const secretGroup of response.secretGroups) {
const newSecretGroup = new SecretGroup(cloudlyClientRef);
Object.assign(newSecretGroup, secretGroup);
secretGroups.push(newSecretGroup);
}
return secretGroups;
}
public static async createSecretGroup(cloudlyClientRef: CloudlyApiClient, secretGroupDataArg: Partial<plugins.servezoneInterfaces.data.ISecretGroup['data']>) {
const createSecretGroupTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretgroup.IReq_CreateSecretGroup>(
'createSecretGroup'
);
const response = await createSecretGroupTR.fire({
identity: cloudlyClientRef.identity,
secretGroup: {
id: null,
data: {
name: secretGroupDataArg.name,
description: secretGroupDataArg.description,
environments: secretGroupDataArg.environments,
key: secretGroupDataArg.key,
tags: secretGroupDataArg.tags,
priority: secretGroupDataArg.priority,
},
},
});
const newSecretGroup = new SecretGroup(cloudlyClientRef);
Object.assign(newSecretGroup, response.resultSecretGroup);
return newSecretGroup;
}
// INSTANCE
public async update() {
const updateSecretGroupTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretgroup.IReq_UpdateSecretGroup>(
'updateSecretGroup'
);
const response = await updateSecretGroupTR.fire({
identity: this.cloudlyClientRef.identity,
secretGroup: {
id: this.id,
data: this.data,
},
});
const resultSecretGroupData = response.resultSecretGroup.data;
plugins.smartexpect.expect(resultSecretGroupData).toEqual(this.data);
return this;
}
public async delete(cloudlyClientRef: CloudlyApiClient, secretGroupIdArg: string) {
const deleteSecretGroupTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.secretgroup.IReq_DeleteSecretGroupById>(
'deleteSecretGroupById'
);
const response = await deleteSecretGroupTR.fire({
identity: cloudlyClientRef.identity,
secretGroupId: this.id,
});
plugins.smartexpect.expect(response.ok).toBeTrue();
return null;
}
}
-7
View File
@@ -1,7 +0,0 @@
import * as plugins from './plugins.js';
export class Server {
public static getServers() {
}
}
-78
View File
@@ -1,78 +0,0 @@
import * as plugins from './plugins.js';
import type { CloudlyApiClient } from './classes.cloudlyapiclient.js';
export class Service implements plugins.servezoneInterfaces.data.IService {
public static async getServices(cloudlyClientRef: CloudlyApiClient) {
const getAllServicesTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_GetServices>(
'getServices'
);
const response = await getAllServicesTR.fire({
identity: cloudlyClientRef.identity,
});
const resultServices: Service[] = [];
for (const service of response.services) {
const newService = new Service(cloudlyClientRef);
Object.assign(newService, service);
resultServices.push(newService);
}
return resultServices;
}
public static async getServiceById(cloudlyClientRef: CloudlyApiClient, serviceIdArg: string) {
const getServiceByIdTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_GetServiceById>(
'getServiceById'
);
const response = await getServiceByIdTR.fire({
identity: cloudlyClientRef.identity,
serviceId: serviceIdArg,
});
const newService = new Service(cloudlyClientRef);
Object.assign(newService, response.service);
return newService;
}
/**
* creates a new service
*/
public static async createService(cloudlyClientRef: CloudlyApiClient, serviceDataArg: Partial<plugins.servezoneInterfaces.data.IService['data']>) {
const createServiceTR = cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_CreateService>(
'createService'
);
const response = await createServiceTR.fire({
identity: cloudlyClientRef.identity,
serviceData: serviceDataArg as plugins.servezoneInterfaces.data.IService['data'],
});
const newService = new Service(cloudlyClientRef);
Object.assign(newService, response.service);
return newService;
}
// INSTANCE
cloudlyClientRef: CloudlyApiClient;
public id: string;
public data: plugins.servezoneInterfaces.data.IService['data'];
constructor(cloudlyClientRef: CloudlyApiClient) {
this.cloudlyClientRef = cloudlyClientRef;
}
/**
* The service has a secret bundle.
* This function essentially returns the secret bundle as a flat object.
* In other words, it resolves secret groups and
*/
public async getSecretBundleAsFlatObject(environmentArg: string = 'production') {
const getServiceSecretBundlesAsFlatObjectTR = this.cloudlyClientRef.typedsocketClient.createTypedRequest<plugins.servezoneInterfaces.requests.service.IRequest_Any_Cloudly_GetServiceSecretBundlesAsFlatObject>(
'getServiceSecretBundlesAsFlatObject'
);
const response = await getServiceSecretBundlesAsFlatObjectTR.fire({
identity: this.cloudlyClientRef.identity,
serviceId: this.id,
environment: environmentArg,
});
const flatKeyValueObject: {[key: string]: string} = response.flatKeyValueObject;
return flatKeyValueObject;
}
}
-1
View File
@@ -1 +0,0 @@
export * from './classes.cloudlyapiclient.js';
-35
View File
@@ -1,35 +0,0 @@
// @serve.zone scope
import * as servezoneInterfaces from '@serve.zone/interfaces';
export {
servezoneInterfaces
}
// @push.rocks scope
import * as smartexpect from '@push.rocks/smartexpect';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrx from '@push.rocks/smartrx';
import * as webstream from '@push.rocks/smartstream/web';
export {
smartexpect,
smartpromise,
smartrx,
webstream,
}
// @api.global scope
import * as typedrequest from '@api.global/typedrequest';
import * as typedsocket from '@api.global/typedsocket';
export {
typedrequest,
typedsocket
}
// @tsclass scope
import * as tsclass from '@tsclass/tsclass';
export {
tsclass,
}
-306
View File
@@ -1,306 +0,0 @@
# @serve.zone/api 🔌
**The powerful API client for Cloudly.** Connect your applications to multi-cloud infrastructure with type-safe, real-time communication.
## 🎯 What is @serve.zone/api?
This is your programmatic gateway to the Cloudly platform. Built with TypeScript, it provides a robust, type-safe interface for managing cloud resources, orchestrating containers, and automating infrastructure operations.
## ✨ Features
- **🔒 Type-Safe** - Full TypeScript support with comprehensive interfaces
- **⚡ Real-Time** - WebSocket-based communication for instant updates
- **🔑 Secure Authentication** - Token-based identity management
- **📦 Resource Management** - Complete control over clusters, images, and services
- **🎭 Multi-Identity** - Support for service accounts and user authentication
- **🔄 Reactive Streams** - RxJS integration for event-driven programming
## 🚀 Installation
```bash
pnpm add @serve.zone/api
```
## 🎬 Quick Start
```typescript
import { CloudlyApiClient } from '@serve.zone/api';
// Initialize the client
const client = new CloudlyApiClient({
registerAs: 'api',
cloudlyUrl: 'https://cloudly.example.com:443'
});
// Start the connection
await client.start();
// Authenticate with a service token
const identity = await client.getIdentityByToken('your-service-token', {
tagConnection: true,
statefullIdentity: true
});
console.log('🎉 Connected as:', identity.name);
```
## 🔐 Authentication
### Service Token Authentication
```typescript
// Authenticate using a service token
const identity = await client.getIdentityByToken(serviceToken, {
tagConnection: true, // Tag this connection with the identity
statefullIdentity: true // Maintain state across reconnections
});
```
### Identity Management
```typescript
// Get current identity
const currentIdentity = client.identity;
// Check permissions
if (currentIdentity.permissions.includes('cluster:write')) {
// Perform cluster operations
}
```
## 📚 Core Operations
### 🐳 Image Management
```typescript
// Create an image entry
const image = await client.images.createImage({
name: 'my-app',
description: 'Production application image'
});
// Push a new version
const imageStream = fs.createReadStream('app.tar');
await image.pushImageVersion('2.0.0', imageStream);
// List all images
const images = await client.images.listImages();
```
### 🌐 Cluster Operations
```typescript
// Get cluster configuration
const clusterConfig = await client.getClusterConfigFromCloudlyByIdentity(identity);
// Deploy to cluster
await client.deployToCluster({
clusterName: 'production',
serviceName: 'api-service',
image: 'my-app:2.0.0',
replicas: 3
});
```
### 🔒 Certificate Management
```typescript
// Request SSL certificate
const certificate = await client.getCertificateForDomain({
domainName: 'api.example.com',
type: 'ssl',
identity: identity
});
// Use certificate in your application
console.log('Certificate:', certificate.cert);
console.log('Private Key:', certificate.key);
```
### 🔐 Secret Management
```typescript
// Create secret group
const secretGroup = await client.secrets.createSecretGroup({
name: 'api-secrets',
secrets: [
{ key: 'DATABASE_URL', value: 'postgres://...' },
{ key: 'REDIS_URL', value: 'redis://...' }
]
});
// Retrieve secrets
const secrets = await client.secrets.getSecretGroup('api-secrets');
```
## 🔄 Real-Time Updates
Subscribe to configuration changes and server actions using RxJS:
```typescript
// Listen for configuration updates
client.configUpdateSubject.subscribe({
next: (config) => {
console.log('📡 Configuration updated:', config);
// React to configuration changes
}
});
// Handle server action requests
client.serverActionSubject.subscribe({
next: (action) => {
console.log('⚡ Server action:', action.type);
// Process server-initiated actions
}
});
```
## 🎯 Advanced Usage
### Streaming Operations
```typescript
// Stream logs from a service
const logStream = await client.logs.streamLogs({
service: 'api-service',
follow: true
});
logStream.on('data', (log) => {
console.log(log.message);
});
```
### Batch Operations
```typescript
// Deploy multiple services
const deployments = await Promise.all([
client.deploy({ service: 'frontend', image: 'app:latest' }),
client.deploy({ service: 'backend', image: 'api:latest' }),
client.deploy({ service: 'worker', image: 'worker:latest' })
]);
```
### Error Handling
```typescript
try {
await client.start();
} catch (error) {
if (error.code === 'AUTH_FAILED') {
console.error('Authentication failed:', error.message);
} else if (error.code === 'CONNECTION_LOST') {
console.error('Connection lost, retrying...');
await client.reconnect();
}
}
```
## 🧹 Cleanup
Always gracefully disconnect when done:
```typescript
// Stop the client connection
await client.stop();
console.log('✅ Disconnected cleanly');
```
## 🔌 API Reference
### CloudlyApiClient
Main client class for interacting with Cloudly.
#### Constructor Options
```typescript
interface ICloudlyApiClientOptions {
registerAs: TClientType; // 'api' | 'cli' | 'web'
cloudlyUrl: string; // Full URL including protocol and port
}
```
#### Methods
- `start()` - Initialize connection
- `stop()` - Close connection
- `getIdentityByToken()` - Authenticate with token
- `getClusterConfigFromCloudlyByIdentity()` - Get cluster configuration
- `getCertificateForDomain()` - Request SSL certificate
- `images` - Image management namespace
- `secrets` - Secret management namespace
- `clusters` - Cluster management namespace
## 🎬 Complete Example
```typescript
import { CloudlyApiClient } from '@serve.zone/api';
async function main() {
// Initialize client
const client = new CloudlyApiClient({
registerAs: 'api',
cloudlyUrl: 'https://cloudly.example.com:443'
});
try {
// Connect and authenticate
await client.start();
const identity = await client.getIdentityByToken(process.env.SERVICE_TOKEN, {
tagConnection: true,
statefullIdentity: true
});
// Create and deploy an image
const image = await client.images.createImage({
name: 'my-service',
description: 'Microservice application'
});
// Push image version
const stream = getImageStream(); // Your image stream
await image.pushImageVersion('1.0.0', stream);
// Deploy to cluster
await client.deployToCluster({
clusterName: 'production',
serviceName: 'my-service',
image: 'my-service:1.0.0',
replicas: 3,
environment: {
NODE_ENV: 'production'
}
});
console.log('✅ Deployment successful!');
} catch (error) {
console.error('❌ Error:', error);
} finally {
await client.stop();
}
}
main();
```
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
### Trademarks
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
### Company Information
Task Venture Capital GmbH
Registered at District court Bremen HRB 35230 HB, Germany
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
-17
View File
@@ -1,17 +0,0 @@
{
"order": 1,
"name": "@serve.zone/api",
"dependencies": [
"@serve.zone/interfaces",
"@push.rocks/smartpromise",
"@push.rocks/smartrx",
"@push.rocks/smartstream",
"@api.global/typedrequest",
"@api.global/typedsocket",
"@tsclass/tsclass"
],
"registries": [
"registry.npmjs.org:public",
"verdaccio.lossless.digital:public"
]
}
+20 -3
View File
@@ -1,11 +1,28 @@
import * as plugins from './plugins.js';
import { CliClient } from "./classes.cliclient.js";
import { CliClient } from './classes.cliclient.js';
export const runCli = async () => {
const cliQenv = new plugins.qenv.Qenv();
const cloudlyUrl = await cliQenv.getEnvVarOnDemand('CLOUDLY_URL');
const token = process.env.CLOUDLY_TOKEN;
const username = process.env.CLOUDLY_USERNAME;
const password = process.env.CLOUDLY_PASSWORD;
const apiClient = new plugins.servezoneApi.CloudlyApiClient({
registerAs: 'cli',
cloudlyUrl: await cliQenv.getEnvVarOnDemand('CLOUDLY_URL'),
cloudlyUrl,
});
await apiClient.start();
if (token) {
await apiClient.getIdentityByToken(token, { tagConnection: true, statefullIdentity: true });
} else if (username && password) {
await apiClient.loginWithUsernameAndPassword(username, password);
} else {
console.log('No credentials provided. Set CLOUDLY_TOKEN or CLOUDLY_USERNAME/CLOUDLY_PASSWORD.');
}
const cliClient = new CliClient(apiClient);
};
// Default action example: list clusters when invoked without subcommands
await cliClient.getClusters();
};
+1 -1
View File
@@ -14,4 +14,4 @@ import * as qenv from '@push.rocks/qenv';
export {
projectinfo,
qenv,
}
}
+66 -319
View File
@@ -1,358 +1,105 @@
# @serve.zone/cli 🚀
# @serve.zone/cli
**Command-line interface for Cloudly.** Manage your multi-cloud infrastructure from the terminal with powerful, intuitive commands.
`@serve.zone/cli` is the published Cloudly CLI submodule. It provides the `servezone` binary and currently acts as a thin environment-driven client for connecting to a Cloudly control plane and listing clusters through `@serve.zone/api`.
## 🎯 What is @serve.zone/cli?
## Issue Reporting and Security
The Cloudly CLI brings the full power of the Cloudly platform to your terminal. Whether you're automating deployments, managing secrets, or monitoring services, the CLI provides a streamlined interface for all your cloud operations.
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
## ✨ Features
## Current Scope
- **⚡ Fast & Efficient** - Optimized for speed and minimal resource usage
- **🔐 Secure Authentication** - Token-based authentication with secure storage
- **📝 Intuitive Commands** - Clear, consistent command structure
- **🎨 Formatted Output** - Beautiful, readable output with color coding
- **🔄 Scriptable** - Perfect for CI/CD pipelines and automation
- **📊 Comprehensive** - Access to all Cloudly features from the terminal
This submodule is intentionally small in the current codebase:
## 🚀 Installation
- Reads `CLOUDLY_URL` through `@push.rocks/qenv`.
- Authenticates with either `CLOUDLY_TOKEN` or `CLOUDLY_USERNAME` plus `CLOUDLY_PASSWORD`.
- Starts a `CloudlyApiClient` registered as `cli`.
- Creates a `CliClient` wrapper around the API client.
- Calls `CliClient.getClusters()` and prints the result.
### Global Installation (Recommended)
It is not currently a full command tree for services, secrets, deployments, logs, profiles, or shell completion. Those flows should be implemented against `@serve.zone/api` before documenting them as CLI commands.
```bash
pnpm add -g @serve.zone/cli
## Installation
The package is published from `cloudly/ts_cliclient` via `tspublish.json` under the name `@serve.zone/cli` with the `servezone` binary.
```sh
pnpm add --global @serve.zone/cli
```
### Local Installation
For local development inside the Cloudly repository, build the parent package:
```bash
pnpm add @serve.zone/cli
```sh
pnpm install
pnpm build
```
## 🎬 Quick Start
## Usage
```bash
# Configure your Cloudly instance
servezone config --url https://cloudly.example.com
Authenticate with a machine token:
# Login with your service token
servezone login --token your-service-token
# List your clusters
servezone clusters list
# Deploy a service
servezone deploy --cluster production --image myapp:latest
```sh
CLOUDLY_URL=https://cloudly.example.com \
CLOUDLY_TOKEN=cluster-or-api-token \
servezone
```
## 🔑 Authentication
Authenticate with username and password:
### Initial Setup
```bash
# Set your Cloudly instance URL
servezone config --url https://cloudly.example.com
# Authenticate with a service token
servezone login --token YOUR_SERVICE_TOKEN
# Or use environment variables
export CLOUDLY_URL=https://cloudly.example.com
export CLOUDLY_TOKEN=YOUR_SERVICE_TOKEN
```sh
CLOUDLY_URL=https://cloudly.example.com \
CLOUDLY_USERNAME=admin \
CLOUDLY_PASSWORD=change-me \
servezone
```
### Managing Profiles
When `CLOUDLY_TOKEN` is present, the CLI requests a stateful identity and asks Cloudly to tag the WebSocket connection. When username/password are present instead, it uses Cloudly's admin login flow. If no credentials are provided, the CLI prints a warning before attempting the default cluster-list operation.
```bash
# Create a profile for different environments
servezone profile create production --url https://prod.cloudly.com
servezone profile create staging --url https://stage.cloudly.com
## Programmatic Use
# Switch between profiles
servezone profile use production
The published submodule exports the `runCli()` entry point. For automation, most callers should use `@serve.zone/api` directly; the internal `CliClient` currently wraps `CloudlyApiClient` only to run the default cluster-list action:
# List all profiles
servezone profile list
```ts
import { CloudlyApiClient } from '@serve.zone/api';
import { CliClient } from './classes.cliclient.js';
const apiClient = new CloudlyApiClient({
registerAs: 'cli',
cloudlyUrl: 'https://cloudly.example.com',
});
await apiClient.start();
await apiClient.loginWithUsernameAndPassword('admin', 'change-me');
const cli = new CliClient(apiClient);
await cli.getClusters();
```
## 📚 Core Commands
### 🌐 Cluster Management
```bash
# List all clusters
servezone clusters list
# Get cluster details
servezone clusters info production-cluster
# Create a new cluster
servezone clusters create \
--name production-cluster \
--region eu-central \
--nodes 3
# Scale a cluster
servezone clusters scale production-cluster --nodes 5
# Delete a cluster
servezone clusters delete staging-cluster
```
### 🐳 Service Deployment
```bash
# Deploy a service
servezone deploy \
--cluster production \
--name api-service \
--image myapp:2.0.0 \
--replicas 3 \
--port 80:3000
# Update a service
servezone service update api-service \
--image myapp:2.1.0 \
--replicas 5
# Scale a service
servezone service scale api-service --replicas 10
# Remove a service
servezone service remove api-service
```
### 🔐 Secret Management
```bash
# Create a secret
servezone secrets create \
--name database-url \
--value "postgres://user:pass@host/db"
# Create a secret group
servezone secrets create-group \
--name api-secrets \
--secret DATABASE_URL=postgres://... \
--secret REDIS_URL=redis://...
# List secrets
servezone secrets list
# Get secret value
servezone secrets get database-url
# Delete a secret
servezone secrets delete old-secret
```
### 📦 Image Management
```bash
# List images
servezone images list
# Push a new image
servezone images push \
--name myapp \
--version 2.0.0 \
--file ./myapp.tar
# Tag an image
servezone images tag myapp:2.0.0 myapp:latest
# Delete an image
servezone images delete myapp:1.0.0
```
### 📊 Monitoring & Logs
```bash
# View service logs
servezone logs api-service
# Follow logs in real-time
servezone logs api-service --follow
# Filter logs
servezone logs api-service --since 1h --grep ERROR
# Get service status
servezone service status api-service
# Monitor cluster health
servezone clusters health production-cluster
```
### 🔧 DNS Management
```bash
# List DNS records
servezone dns list --domain example.com
# Create a DNS record
servezone dns create \
--domain example.com \
--name api \
--type A \
--value 192.168.1.1
# Update a DNS record
servezone dns update api.example.com --value 192.168.1.2
# Delete a DNS record
servezone dns delete old.example.com
```
## 🎯 Advanced Usage
### Environment Variables
```bash
# Set environment variables for a service
servezone deploy \
--name api-service \
--env NODE_ENV=production \
--env PORT=3000 \
--env DATABASE_URL=@secret:database-url
```
### Configuration Files
Create a `cloudly.yaml` file:
```yaml
cluster: production
service:
name: api-service
image: myapp:latest
replicas: 3
ports:
- 80:3000
environment:
NODE_ENV: production
DATABASE_URL: "@secret:database-url"
```
Deploy using the config file:
```bash
servezone deploy --config cloudly.yaml
```
### Batch Operations
```bash
# Deploy multiple services
servezone deploy --config services/*.yaml
# Update all services in a namespace
servezone service update --namespace api --image-tag v2.0.0
# Delete all staging resources
servezone cleanup --environment staging
```
## 🔄 CI/CD Integration
### GitHub Actions
```yaml
- name: Deploy to Cloudly
run: |
servezone config --url ${{ secrets.CLOUDLY_URL }}
servezone login --token ${{ secrets.CLOUDLY_TOKEN }}
servezone deploy \
--cluster production \
--name api-service \
--image myapp:${{ github.sha }}
```
### GitLab CI
```yaml
deploy:
script:
- servezone config --url $CLOUDLY_URL
- servezone login --token $CLOUDLY_TOKEN
- servezone deploy --config cloudly.yaml
```
## 🎨 Output Formats
```bash
# JSON output for scripting
servezone clusters list --output json
# YAML output
servezone service info api-service --output yaml
# Table output (default)
servezone images list --output table
# Quiet mode (IDs only)
servezone clusters list --quiet
```
## 🛠️ Troubleshooting
```bash
# Enable debug output
servezone --debug clusters list
# Check CLI version
servezone version
# Test connection
servezone ping
# View configuration
servezone config show
# Clear cache and credentials
servezone logout --clear-cache
```
## 📝 Command Reference
```bash
servezone --help # Show all commands
servezone <command> --help # Show command-specific help
servezone clusters --help # Show cluster commands
servezone service --help # Show service commands
servezone secrets --help # Show secret commands
```
## 🔌 Shell Completion
Enable tab completion for your shell:
```bash
# Bash
servezone completion bash > /etc/bash_completion.d/servezone
# Zsh
servezone completion zsh > ~/.zsh/completions/_servezone
# Fish
servezone completion fish > ~/.config/fish/completions/servezone.fish
```
## Files
| Path | Purpose |
| --- | --- |
| `index.ts` | Runtime entry point for the published CLI. |
| `classes.cliclient.ts` | Minimal client wrapper; currently exposes `getClusters()`. |
| `plugins.ts` | Centralized imports for the submodule. |
| `tspublish.json` | Published package name, dependencies, registry targets, and `servezone` bin metadata. |
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [license](../license) file.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
### Trademarks
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
### Company Information
Task Venture Capital GmbH
Registered at District court Bremen HRB 35230 HB, Germany
Registered at District Court Bremen HRB 35230 HB, Germany
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
For any legal inquiries or further information, please contact us via email at hello@task.vc.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
+3 -2
View File
@@ -11,5 +11,6 @@
"registry.npmjs.org:public",
"verdaccio.lossless.digital:public"
],
"bin": ["servezone"]
}
"bin": ["servezone"],
"order": 10
}
-8
View File
@@ -1,8 +0,0 @@
/**
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: '@serve.zone/interfaces',
version: '1.1.2',
description: 'interfaces for working with containers'
}
-73
View File
@@ -1,73 +0,0 @@
import * as plugins from '../plugins.js';
export interface IBareMetal {
id: string;
data: {
hostname: string;
/**
* IPMI management IP address
*/
ipmiAddress?: string;
/**
* Encrypted IPMI credentials
*/
ipmiCredentials?: {
username: string;
passwordEncrypted: string;
};
/**
* Primary network IP address
*/
primaryIp: string;
/**
* Provider of the physical server
*/
provider: 'hetzner' | 'aws' | 'digitalocean' | 'onpremise';
/**
* Data center or location
*/
location: string;
/**
* Hardware specifications
*/
specs: {
cpuModel: string;
cpuCores: number;
memoryGB: number;
storageGB: number;
storageType: 'ssd' | 'hdd' | 'nvme';
};
/**
* Current power state
*/
powerState: 'on' | 'off' | 'unknown';
/**
* Operating system information
*/
osInfo: {
name: string;
version: string;
kernel?: string;
};
/**
* Array of ClusterNode IDs running on this hardware
*/
assignedNodeIds: string[];
/**
* Metadata for provider-specific information
*/
providerMetadata?: {
[key: string]: any;
};
};
}
-14
View File
@@ -1,14 +0,0 @@
import * as plugins from '../plugins.js';
export interface ICloudlyConfig {
environment?: 'production' | 'integration';
letsEncryptEmail?: string;
letsEncryptPrivateKey?: string;
jwtKeypair?: plugins.tsclass.network.IJwtKeypair;
mongoDescriptor?: plugins.tsclass.database.IMongoDescriptor;
s3Descriptor?: plugins.tsclass.storage.IS3Descriptor;
publicUrl?: string;
publicPort?: string;
sslMode?: 'none' | 'letsencrypt' | 'external';
servezoneAdminaccount?: string;
}
-41
View File
@@ -1,41 +0,0 @@
import * as plugins from '../plugins.js';
import { type IDockerRegistryInfo } from '../data/docker.js';
import type { IClusterNode } from './clusternode.js';
export interface ICluster {
id: string;
data: {
name: string;
/**
* a cluster has a machine user that governs access rights.
*/
userId: string;
/**
* how can the cluster reach cloudly
*/
cloudlyUrl?: string;
/**
* Cluster setup mode - manual by default, or auto-provision with cloud provider
*/
setupMode?: 'manual' | 'hetzner' | 'aws' | 'digitalocean';
/**
* Nodes that are part of the cluster
*/
nodes: IClusterNode[];
/**
* ACME info. This is used to get SSL certificates.
*/
acmeInfo: {
serverAddress: string;
serverSecret: string;
};
sshKeys: plugins.tsclass.network.ISshKey[];
};
}
-71
View File
@@ -1,71 +0,0 @@
import * as plugins from '../plugins.js';
export interface IClusterNodeMetrics {
cpuUsagePercent: number;
memoryUsedMB: number;
memoryAvailableMB: number;
diskUsedGB: number;
diskAvailableGB: number;
containerCount: number;
timestamp: number;
}
export interface IClusterNode {
id: string;
data: {
/**
* Reference to the cluster this node belongs to
*/
clusterId: string;
/**
* Reference to the physical server (if applicable)
*/
baremetalId?: string;
/**
* Type of node
*/
nodeType: 'baremetal' | 'vm' | 'container';
/**
* Current status of the node
*/
status: 'initializing' | 'online' | 'offline' | 'maintenance';
/**
* Role of the node in the cluster
*/
role: 'master' | 'worker';
/**
* Timestamp when node joined the cluster
*/
joinedAt: number;
/**
* Last health check timestamp
*/
lastHealthCheck: number;
/**
* Current metrics for the node
*/
metrics?: IClusterNodeMetrics;
/**
* Docker swarm node ID if part of swarm
*/
swarmNodeId?: string;
/**
* SSH keys deployed to this node
*/
sshKeys: plugins.tsclass.network.ISshKey[];
/**
* Debian packages installed on this node
*/
requiredDebianPackages: string[];
};
}
-1
View File
@@ -1 +0,0 @@
export type TConfigType = 'server' | 'cluster' | 'coreflow' | 'service';

Some files were not shown because too many files have changed in this diff Show More