Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fd868b101e | |||
| 347fe3bfbe | |||
| f4e5f02d0c | |||
| c3e5cabe3d | |||
| bcc18e3511 | |||
| 3b6cbe7ca8 | |||
| 41e1893d2d | |||
| 1ce31c52ec | |||
| 3bc9c0fc79 | |||
| bf838f938a | |||
| 32bf9bae0e | |||
| 038ceb976f | |||
| 0c96bbc281 | |||
| 14d9ade4d0 | |||
| 332281746e | |||
| 99d9a0a581 | |||
| ee0640bb80 |
@@ -0,0 +1,43 @@
|
|||||||
|
# Repository Guidance
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This repository is the product layer around `@push.rocks/smartstorage`.
|
||||||
|
|
||||||
|
Its job is to turn the storage engine into a usable product:
|
||||||
|
|
||||||
|
- a Docker image that starts cleanly and predictably
|
||||||
|
- a management UI and ops API
|
||||||
|
- product-facing config, auth, and persistence behavior
|
||||||
|
- tests, docs, and release hygiene
|
||||||
|
|
||||||
|
## Dependency Ownership
|
||||||
|
|
||||||
|
We control the dependencies used here, especially `@push.rocks/smartstorage` and the `@design.estate/*` UI stack.
|
||||||
|
|
||||||
|
- Do not keep product code full of workarounds for missing dependency capabilities.
|
||||||
|
- If a missing capability clearly belongs in a dependency, improve the dependency.
|
||||||
|
- If dependency work is not being done in this repo right now, write a focused implementation prompt into `./prompts/*.md`.
|
||||||
|
|
||||||
|
## Boundary
|
||||||
|
|
||||||
|
This repo should own:
|
||||||
|
|
||||||
|
- Docker and runtime packaging
|
||||||
|
- CLI, env var, and startup UX
|
||||||
|
- admin auth flow and management APIs
|
||||||
|
- product metadata persistence and migrations
|
||||||
|
- acceptance tests and docs
|
||||||
|
|
||||||
|
`@push.rocks/smartstorage` should own:
|
||||||
|
|
||||||
|
- storage-engine stats and health surfaces
|
||||||
|
- bucket and object metadata summaries that should be cheap at runtime
|
||||||
|
- cluster and drive introspection
|
||||||
|
- runtime auth/credential APIs when the engine must react to changes
|
||||||
|
|
||||||
|
## Planning Docs
|
||||||
|
|
||||||
|
- Put current repo improvement work in `readme.plan.md`.
|
||||||
|
- Keep durable findings in `readme.hints.md`.
|
||||||
|
- Put dependency implementation prompts in `prompts/*.md`.
|
||||||
+19
-15
@@ -6,6 +6,7 @@ FROM --platform=linux/amd64 node:22-alpine AS build
|
|||||||
RUN corepack enable && corepack prepare pnpm@latest --activate
|
RUN corepack enable && corepack prepare pnpm@latest --activate
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
ENV DENO_DIR=/deno-dir
|
||||||
|
|
||||||
# Use verdaccio registry (hosts private packages and proxies public ones)
|
# Use verdaccio registry (hosts private packages and proxies public ones)
|
||||||
RUN npm config set registry https://verdaccio.lossless.digital/
|
RUN npm config set registry https://verdaccio.lossless.digital/
|
||||||
@@ -15,7 +16,7 @@ COPY package.json pnpm-lock.yaml ./
|
|||||||
RUN pnpm install
|
RUN pnpm install
|
||||||
|
|
||||||
# Copy source and build
|
# Copy source and build
|
||||||
COPY npmextra.json ./
|
COPY .smartconfig.json ./
|
||||||
COPY html/ ./html/
|
COPY html/ ./html/
|
||||||
COPY ts_web/ ./ts_web/
|
COPY ts_web/ ./ts_web/
|
||||||
COPY ts_interfaces/ ./ts_interfaces/
|
COPY ts_interfaces/ ./ts_interfaces/
|
||||||
@@ -23,32 +24,35 @@ COPY ts_bundled/ ./ts_bundled/
|
|||||||
RUN pnpm run build
|
RUN pnpm run build
|
||||||
|
|
||||||
## STAGE 2 // production runtime with Deno
|
## STAGE 2 // production runtime with Deno
|
||||||
FROM alpine:edge AS final
|
FROM denoland/deno:debian AS final
|
||||||
|
ENV DENO_DIR=/deno-dir
|
||||||
|
|
||||||
# Install Deno and minimal runtime dependencies
|
# Install Deno and minimal runtime dependencies
|
||||||
RUN apk add --no-cache \
|
RUN apt-get update && \
|
||||||
deno \
|
apt-get install -y --no-install-recommends ca-certificates tini && \
|
||||||
ca-certificates \
|
rm -rf /var/lib/apt/lists/*
|
||||||
tini \
|
|
||||||
gcompat \
|
|
||||||
libstdc++
|
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Copy only what Deno needs at runtime
|
# Copy only what Deno needs at runtime
|
||||||
COPY deno.json ./
|
COPY deno.json ./
|
||||||
|
COPY deno.lock ./
|
||||||
COPY mod.ts ./
|
COPY mod.ts ./
|
||||||
COPY ts/ ./ts/
|
COPY ts/ ./ts/
|
||||||
COPY ts_interfaces/ ./ts_interfaces/
|
COPY ts_interfaces/ ./ts_interfaces/
|
||||||
COPY --from=build /app/ts_bundled/bundle.ts ./ts_bundled/bundle.ts
|
COPY --from=build /app/ts_bundled/bundle.ts ./ts_bundled/bundle.ts
|
||||||
|
|
||||||
# Pre-cache Deno dependencies
|
# Pre-cache Deno dependencies and prepare non-root runtime paths
|
||||||
RUN deno cache mod.ts
|
RUN deno cache mod.ts && \
|
||||||
|
groupadd --system objectstorage && \
|
||||||
|
useradd --system --gid objectstorage --home-dir /app objectstorage && \
|
||||||
|
mkdir -p /data /deno-dir && \
|
||||||
|
chown -R objectstorage:objectstorage /app /data /deno-dir
|
||||||
|
|
||||||
# Create storage directory
|
EXPOSE 9000 3000 4433
|
||||||
RUN mkdir -p /data
|
|
||||||
|
|
||||||
EXPOSE 9000 3000
|
|
||||||
VOLUME ["/data"]
|
VOLUME ["/data"]
|
||||||
ENTRYPOINT ["/sbin/tini", "--"]
|
USER objectstorage
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||||
|
CMD deno eval "const response = await fetch('http://127.0.0.1:3000/readyz'); Deno.exit(response.ok ? 0 : 1);"
|
||||||
|
ENTRYPOINT ["/usr/bin/tini", "--"]
|
||||||
CMD ["deno", "run", "--allow-all", "mod.ts", "server"]
|
CMD ["deno", "run", "--allow-all", "mod.ts", "server"]
|
||||||
|
|||||||
@@ -1,5 +1,55 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-04-30 - 1.9.0 - feat(opsserver)
|
||||||
|
add health, audit, cluster health, and durable credential management hardening
|
||||||
|
|
||||||
|
- persist managed access credentials to .objectstorage/admin-config.json and reload them on restart while allowing explicit environment credentials to override persisted values
|
||||||
|
- add management health endpoints (/livez, /readyz, /healthz, /metrics), append-only audit logging, logout token revocation, failed-login rate limiting, and a startup guard against default admin credentials on persistent /data storage
|
||||||
|
- surface smartstorage cluster health through the management API and config UI, and harden the Docker image with a non-root runtime user, DENO_DIR, ready healthcheck, updated build config copy, and smoke coverage
|
||||||
|
|
||||||
|
## 2026-03-24 - 1.8.1 - fix(build)
|
||||||
|
migrate build tool config to .smartconfig.json and bump tooling dependencies
|
||||||
|
|
||||||
|
- Move tsbundle, tsdocker, and tswatch configuration from npmextra.json to .smartconfig.json
|
||||||
|
- Update @git.zone/tsbundle, @git.zone/tsdocker, and @git.zone/tswatch devDependencies
|
||||||
|
- Bump @aws-sdk/client-s3 import to ^3.1016.0
|
||||||
|
|
||||||
|
## 2026-03-24 - 1.8.0 - feat(docs,web)
|
||||||
|
document cluster and erasure coding support and align UI storage provider naming
|
||||||
|
|
||||||
|
- expand the README with cluster mode, erasure coding, multi-drive configuration, and multi-node Docker Compose examples
|
||||||
|
- update web storage integration to use the renamed storage data provider interface and browser component
|
||||||
|
- bump runtime, UI, and tooling dependencies to newer compatible versions
|
||||||
|
|
||||||
|
## 2026-03-22 - 1.7.1 - fix(repo)
|
||||||
|
no changes to commit
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-03-22 - 1.7.0 - feat(cluster)
|
||||||
|
add cluster configuration support across server, CLI, and admin UI
|
||||||
|
|
||||||
|
- add environment variable and CLI support for cluster, QUIC, seed node, drive, erasure coding, and heartbeat settings
|
||||||
|
- pass cluster-aware configuration into smartstorage and expose the QUIC port in Docker
|
||||||
|
- extend config APIs and admin UI to display cluster, erasure coding, and storage drive configuration
|
||||||
|
- upgrade @push.rocks/smartstorage to ^6.3.1 to support the new cluster capabilities
|
||||||
|
|
||||||
|
## 2026-03-21 - 1.6.0 - feat(scripts)
|
||||||
|
add release script for committing and pushing docker images
|
||||||
|
|
||||||
|
- Adds a new npm release script to automate git commit and Docker image push steps.
|
||||||
|
|
||||||
|
## 2026-03-21 - 1.5.1 - fix(project)
|
||||||
|
no changes to commit
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-03-21 - 1.5.0 - feat(test)
|
||||||
|
add end-to-end test coverage for container lifecycle, auth, buckets, objects, policies, credentials, status, and S3 compatibility
|
||||||
|
|
||||||
|
- adds a reusable test helper for creating isolated ObjectStorageContainer instances and logging in as admin
|
||||||
|
- introduces a test script in package.json to run the new Deno test suite
|
||||||
|
- covers core management API flows including authentication, bucket operations, object operations, named policies, credentials, and server status/config
|
||||||
|
- verifies S3 SDK interoperability including bucket and object operations plus credential rejection cases
|
||||||
|
|
||||||
## 2026-03-15 - 1.4.2 - fix(license)
|
## 2026-03-15 - 1.4.2 - fix(license)
|
||||||
add missing license file
|
add missing license file
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lossless.zone/objectstorage",
|
"name": "@lossless.zone/objectstorage",
|
||||||
"version": "1.4.2",
|
"version": "1.9.0",
|
||||||
"exports": "./mod.ts",
|
"exports": "./mod.ts",
|
||||||
"nodeModulesDir": "auto",
|
"nodeModulesDir": "auto",
|
||||||
"tasks": {
|
"tasks": {
|
||||||
@@ -8,12 +8,12 @@
|
|||||||
"dev": "pnpm run watch"
|
"dev": "pnpm run watch"
|
||||||
},
|
},
|
||||||
"imports": {
|
"imports": {
|
||||||
"@push.rocks/smartstorage": "npm:@push.rocks/smartstorage@^6.0.1",
|
"@push.rocks/smartstorage": "npm:@push.rocks/smartstorage@^6.4.0",
|
||||||
"@push.rocks/smartbucket": "npm:@push.rocks/smartbucket@^4.3.0",
|
"@push.rocks/smartbucket": "npm:@push.rocks/smartbucket@^4.5.1",
|
||||||
"@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.937.0",
|
"@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.1016.0",
|
||||||
"@api.global/typedrequest-interfaces": "npm:@api.global/typedrequest-interfaces@^3.0.19",
|
"@api.global/typedrequest-interfaces": "npm:@api.global/typedrequest-interfaces@^3.0.19",
|
||||||
"@api.global/typedrequest": "npm:@api.global/typedrequest@^3.2.6",
|
"@api.global/typedrequest": "npm:@api.global/typedrequest@^3.3.0",
|
||||||
"@api.global/typedserver": "npm:@api.global/typedserver@^8.3.1",
|
"@api.global/typedserver": "npm:@api.global/typedserver@^8.4.6",
|
||||||
"@push.rocks/smartguard": "npm:@push.rocks/smartguard@^3.1.0",
|
"@push.rocks/smartguard": "npm:@push.rocks/smartguard@^3.1.0",
|
||||||
"@push.rocks/smartjwt": "npm:@push.rocks/smartjwt@^2.2.1"
|
"@push.rocks/smartjwt": "npm:@push.rocks/smartjwt@^2.2.1"
|
||||||
},
|
},
|
||||||
|
|||||||
+9
-6
@@ -1,27 +1,30 @@
|
|||||||
{
|
{
|
||||||
"name": "@lossless.zone/objectstorage",
|
"name": "@lossless.zone/objectstorage",
|
||||||
"version": "1.4.2",
|
"version": "1.9.0",
|
||||||
"description": "object storage server with management UI powered by smartstorage",
|
"description": "object storage server with management UI powered by smartstorage",
|
||||||
"main": "mod.ts",
|
"main": "mod.ts",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
"test": "deno task test",
|
||||||
|
"test:docker": "OBJST_RUN_DOCKER_SMOKE=1 deno test --allow-all test/test.docker-smoke.test.ts",
|
||||||
"watch": "tswatch",
|
"watch": "tswatch",
|
||||||
"build": "tsbundle",
|
"build": "tsbundle",
|
||||||
"bundle": "tsbundle",
|
"bundle": "tsbundle",
|
||||||
"build:docker": "tsdocker build --verbose",
|
"build:docker": "tsdocker build --verbose",
|
||||||
|
"release": "gitzone commit -yp && tsdocker push --verbose",
|
||||||
"start:docker": "docker stop objectstorage 2>/dev/null; docker rm objectstorage 2>/dev/null; docker build --load -t objectstorage:latest . && docker run --rm --name objectstorage -p 9000:9000 -p 3000:3000 -v objectstorage-data:/data objectstorage:latest"
|
"start:docker": "docker stop objectstorage 2>/dev/null; docker rm objectstorage 2>/dev/null; docker build --load -t objectstorage:latest . && docker run --rm --name objectstorage -p 9000:9000 -p 3000:3000 -v objectstorage-data:/data objectstorage:latest"
|
||||||
},
|
},
|
||||||
"author": "Lossless GmbH",
|
"author": "Lossless GmbH",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@api.global/typedrequest-interfaces": "^3.0.19",
|
"@api.global/typedrequest-interfaces": "^3.0.19",
|
||||||
"@design.estate/dees-catalog": "^3.48.0",
|
"@design.estate/dees-catalog": "^3.49.0",
|
||||||
"@design.estate/dees-element": "^2.2.2"
|
"@design.estate/dees-element": "^2.2.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@git.zone/tsbundle": "^2.9.0",
|
"@git.zone/tsbundle": "^2.10.0",
|
||||||
"@git.zone/tsdocker": "^2.0.0",
|
"@git.zone/tsdocker": "^2.2.4",
|
||||||
"@git.zone/tswatch": "^3.2.0"
|
"@git.zone/tswatch": "^3.3.2"
|
||||||
},
|
},
|
||||||
"pnpm": {
|
"pnpm": {
|
||||||
"onlyBuiltDependencies": [
|
"onlyBuiltDependencies": [
|
||||||
|
|||||||
Generated
+560
-265
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
|||||||
|
# Project Hints
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
- Deno-based backend with `deno.json` for imports and tasks
|
||||||
|
- Frontend bundled with `@git.zone/tsbundle` (esbuild, base64ts output mode)
|
||||||
|
- Config in `.smartconfig.json` (renamed from npmextra.json as of 2026-03-24)
|
||||||
|
- Runtime-managed credentials persist in `${storageDirectory}/.objectstorage/admin-config.json`
|
||||||
|
- Admin audit entries append to `${storageDirectory}/.objectstorage/audit.log`
|
||||||
|
- Management health endpoints: `/livez`, `/readyz`, `/healthz`, `/metrics`
|
||||||
|
- Persistent `/data` deployments reject default `admin/admin` credentials unless `OBJST_ALLOW_INSECURE_DEFAULTS=true`
|
||||||
|
- Tests run with `deno task test` (not tstest)
|
||||||
|
- Docker image built with `@git.zone/tsdocker`
|
||||||
|
|
||||||
|
## Build Tools Config
|
||||||
|
|
||||||
|
- `.smartconfig.json` contains config for `@git.zone/tsbundle`, `@git.zone/tswatch`, and `@git.zone/tsdocker`
|
||||||
|
- tsbundle uses base64ts output mode for Deno compile embedding
|
||||||
|
- tswatch runs backend watcher with `deno run --allow-all mod.ts server --ephemeral`
|
||||||
|
- Docker smoke coverage is opt-in via `pnpm run test:docker`
|
||||||
|
- Docker runtime has a `/readyz` healthcheck and runs as the `objectstorage` user
|
||||||
|
|
||||||
|
## Dependencies (as of 2026-03-24)
|
||||||
|
|
||||||
|
- devDependencies: tsbundle@2.10.0, tsdocker@2.2.4, tswatch@3.3.2
|
||||||
|
- No tsconfig.json — uses compilerOptions in deno.json
|
||||||
@@ -1,10 +1,10 @@
|
|||||||
# @lossless.zone/objectstorage
|
# @lossless.zone/objectstorage
|
||||||
|
|
||||||
> 🚀 S3-compatible object storage server with a slick management UI — powered by [`smartstorage`](https://code.foss.global/push.rocks/smartstorage).
|
> 🚀 S3-compatible object storage server with clustering, erasure coding, and a slick management UI — powered by [`smartstorage`](https://code.foss.global/push.rocks/smartstorage).
|
||||||
|
|
||||||
**objectstorage** gives you a fully featured, self-hosted S3-compatible storage server with a beautiful web-based management interface — all in a single Docker image. No Java, no bloat, no fuss.
|
**objectstorage** gives you a fully featured, self-hosted S3-compatible storage server with a beautiful web-based management interface — all in a single Docker image. No Java, no bloat, no fuss.
|
||||||
|
|
||||||
Built on Deno for the backend and [`@design.estate/dees-catalog`](https://code.foss.global/design.estate/dees-catalog) for a polished UI, it speaks the S3 protocol out of the box while adding powerful management features on top.
|
Built on Deno for the backend and [`@design.estate/dees-catalog`](https://code.foss.global/design.estate/dees-catalog) for a polished UI, it speaks the S3 protocol out of the box while adding powerful management features on top. Scale from a single node to a distributed cluster with erasure coding and multi-drive support.
|
||||||
|
|
||||||
## Issue Reporting and Security
|
## Issue Reporting and Security
|
||||||
|
|
||||||
@@ -13,14 +13,18 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community
|
|||||||
## ✨ Features
|
## ✨ Features
|
||||||
|
|
||||||
- **Full S3 API compatibility** — Works with any S3 client, SDK, or tool (AWS CLI, boto3, etc.)
|
- **Full S3 API compatibility** — Works with any S3 client, SDK, or tool (AWS CLI, boto3, etc.)
|
||||||
- **Management UI** — Web dashboard for buckets, objects, policies, credentials, and server config
|
- **🔗 Cluster mode** — Distribute storage across multiple nodes with QUIC transport, automatic discovery, and quorum writes/reads
|
||||||
|
- **🛡️ Erasure coding** — Reed-Solomon erasure coding (default 4+2) for data durability with minimal overhead
|
||||||
|
- **💾 Multi-drive support** — Stripe data across multiple disks per node with per-drive health monitoring
|
||||||
|
- **🔄 Self-healing** — Background scanner detects and reconstructs missing or corrupt shards automatically
|
||||||
|
- **Management UI** — Web dashboard for buckets, objects, policies, credentials, cluster config, and storage drives
|
||||||
- **Finder-style object browser** — Column view with file preview, drag-and-drop upload, move/rename, context menus
|
- **Finder-style object browser** — Column view with file preview, drag-and-drop upload, move/rename, context menus
|
||||||
- **Inline code editing** — Built-in Monaco editor with syntax highlighting and save-back-to-storage
|
- **Inline code editing** — Built-in Monaco editor with syntax highlighting and save-back-to-storage
|
||||||
- **PDF viewer** — Render PDFs inline with page navigation, zoom, and thumbnails
|
- **PDF viewer** — Render PDFs inline with page navigation, zoom, and thumbnails
|
||||||
- **Named policy management** — Create reusable IAM-style policies, attach them to multiple buckets
|
- **Named policy management** — Create reusable IAM-style policies, attach them to multiple buckets
|
||||||
- **Credential management** — Add/remove access keys through the UI with live-reload
|
- **Credential management** — Add/remove access keys through the UI with live-reload
|
||||||
- **Single Docker image** — Multi-arch (`amd64` + `arm64`), tiny Alpine-based image
|
- **Single Docker image** — Multi-arch (`amd64` + `arm64`), tiny Alpine-based image
|
||||||
- **Fast** — Rust-powered storage engine via `smartstorage`, Deno runtime for management layer
|
- **Fast** — Rust-powered storage engine via `smartstorage`, streaming I/O with zero-copy and backpressure
|
||||||
- **Secure by default** — JWT-based admin auth, S3 SigV4 authentication, bucket policies
|
- **Secure by default** — JWT-based admin auth, S3 SigV4 authentication, bucket policies
|
||||||
- **🌙 Dark theme** — Automatic dark mode following your system preference
|
- **🌙 Dark theme** — Automatic dark mode following your system preference
|
||||||
|
|
||||||
@@ -59,7 +63,7 @@ deno run --allow-all mod.ts server --ephemeral
|
|||||||
|
|
||||||
objectstorage is configured through environment variables, CLI flags, or programmatic config. **Environment variables take precedence** over CLI flags.
|
objectstorage is configured through environment variables, CLI flags, or programmatic config. **Environment variables take precedence** over CLI flags.
|
||||||
|
|
||||||
### Environment Variables
|
### Server Environment Variables
|
||||||
|
|
||||||
| Variable | Description | Default |
|
| Variable | Description | Default |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
@@ -71,18 +75,86 @@ objectstorage is configured through environment variables, CLI flags, or program
|
|||||||
| `OBJST_ADMIN_PASSWORD` | Admin UI password | `admin` |
|
| `OBJST_ADMIN_PASSWORD` | Admin UI password | `admin` |
|
||||||
| `OBJST_REGION` | Storage region identifier | `us-east-1` |
|
| `OBJST_REGION` | Storage region identifier | `us-east-1` |
|
||||||
|
|
||||||
|
### Cluster Environment Variables
|
||||||
|
|
||||||
|
| Variable | Description | Default |
|
||||||
|
|---|---|---|
|
||||||
|
| `OBJST_CLUSTER_ENABLED` | Enable cluster mode (`true`/`false`) | `false` |
|
||||||
|
| `OBJST_CLUSTER_NODE_ID` | Unique node identifier | auto-generated |
|
||||||
|
| `OBJST_CLUSTER_QUIC_PORT` | QUIC transport port | `4433` |
|
||||||
|
| `OBJST_CLUSTER_SEED_NODES` | Comma-separated seed node addresses | _(empty)_ |
|
||||||
|
| `OBJST_DRIVE_PATHS` | Comma-separated drive mount paths | storage dir |
|
||||||
|
| `OBJST_ERASURE_DATA_SHARDS` | Erasure coding data shards | `4` |
|
||||||
|
| `OBJST_ERASURE_PARITY_SHARDS` | Erasure coding parity shards | `2` |
|
||||||
|
| `OBJST_ERASURE_CHUNK_SIZE` | Erasure chunk size in bytes | `4194304` (4 MB) |
|
||||||
|
| `OBJST_HEARTBEAT_INTERVAL_MS` | Cluster heartbeat interval | `5000` |
|
||||||
|
| `OBJST_HEARTBEAT_TIMEOUT_MS` | Cluster heartbeat timeout | `30000` |
|
||||||
|
|
||||||
### CLI Flags
|
### CLI Flags
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
deno run --allow-all mod.ts server [options]
|
deno run --allow-all mod.ts server [options]
|
||||||
|
|
||||||
Options:
|
Server Options:
|
||||||
--storage-port <port> Storage API port (default: 9000)
|
--storage-port <port> Storage API port (default: 9000)
|
||||||
--ui-port <port> Management UI port (default: 3000)
|
--ui-port <port> Management UI port (default: 3000)
|
||||||
--storage-dir <path> Storage directory (default: /data)
|
--storage-dir <path> Storage directory (default: /data)
|
||||||
--ephemeral Use ./.nogit/objstdata for storage (dev mode)
|
--ephemeral Use ./.nogit/objstdata for storage (dev mode)
|
||||||
|
|
||||||
|
Clustering Options:
|
||||||
|
--cluster-enabled Enable cluster mode
|
||||||
|
--cluster-node-id <id> Unique node identifier
|
||||||
|
--cluster-quic-port <port> QUIC transport port (default: 4433)
|
||||||
|
--cluster-seed-nodes <list> Comma-separated seed node addresses
|
||||||
|
--drive-paths <list> Comma-separated drive mount paths
|
||||||
|
--erasure-data-shards <n> Erasure coding data shards (default: 4)
|
||||||
|
--erasure-parity-shards <n> Erasure coding parity shards (default: 2)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## 🔗 Cluster Mode
|
||||||
|
|
||||||
|
objectstorage supports distributed storage across multiple nodes with automatic failover and data redundancy.
|
||||||
|
|
||||||
|
### How it works
|
||||||
|
|
||||||
|
1. **Enable clustering** on each node with `OBJST_CLUSTER_ENABLED=true`
|
||||||
|
2. **Point nodes at each other** using `OBJST_CLUSTER_SEED_NODES` — nodes discover the full cluster from any seed
|
||||||
|
3. **Configure drives** per node with `OBJST_DRIVE_PATHS` — each drive is independently managed
|
||||||
|
4. **Erasure coding** splits objects into data + parity shards across drives and nodes
|
||||||
|
|
||||||
|
### Example: 3-node cluster
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Node 1
|
||||||
|
docker run -d --name objst-node1 \
|
||||||
|
-p 9000:9000 -p 3000:3000 -p 4433:4433/udp \
|
||||||
|
-v /mnt/disk1:/drive1 -v /mnt/disk2:/drive2 \
|
||||||
|
-e OBJST_CLUSTER_ENABLED=true \
|
||||||
|
-e OBJST_CLUSTER_NODE_ID=node-1 \
|
||||||
|
-e OBJST_CLUSTER_SEED_NODES=node2:4433,node3:4433 \
|
||||||
|
-e OBJST_DRIVE_PATHS=/drive1,/drive2 \
|
||||||
|
-e OBJST_ACCESS_KEY=myadminkey \
|
||||||
|
-e OBJST_SECRET_KEY=mysupersecret \
|
||||||
|
code.foss.global/lossless.zone/objectstorage:latest
|
||||||
|
|
||||||
|
# Node 2 and Node 3 — same pattern, different node IDs and seed nodes
|
||||||
|
```
|
||||||
|
|
||||||
|
### Erasure coding presets
|
||||||
|
|
||||||
|
| Config | Data Shards | Parity Shards | Overhead | Fault Tolerance |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| Default | 4 | 2 | 50% | 2 failures |
|
||||||
|
| High durability | 6 | 3 | 50% | 3 failures |
|
||||||
|
| Minimal | 2 | 1 | 50% | 1 failure |
|
||||||
|
|
||||||
|
### Inter-node transport
|
||||||
|
|
||||||
|
Cluster communication uses **QUIC** (UDP port 4433 by default) with:
|
||||||
|
- Auto-generated TLS certificates
|
||||||
|
- Multiplexed streams with flow-control backpressure
|
||||||
|
- Heartbeat-based failure detection (default: 5s interval, 30s timeout)
|
||||||
|
|
||||||
## 🖥️ Management UI
|
## 🖥️ Management UI
|
||||||
|
|
||||||
The web-based management UI is served on the UI port (default: `3000`). Log in with username `admin` and the configured admin password.
|
The web-based management UI is served on the UI port (default: `3000`). Log in with username `admin` and the configured admin password.
|
||||||
@@ -147,7 +219,7 @@ Click "Add Key" to create new access credentials. They're immediately available
|
|||||||
|
|
||||||
### Configuration
|
### Configuration
|
||||||
|
|
||||||
View your server's current configuration at a glance — ports, region, storage directory, and auth/CORS status.
|
View your server's current configuration at a glance — ports, region, storage directory, auth/CORS status, cluster configuration, erasure coding settings, and storage drive paths. The config view also includes an environment variable reference guide for cluster setup.
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
@@ -279,7 +351,7 @@ pnpm run build:docker
|
|||||||
pnpm run start:docker
|
pnpm run start:docker
|
||||||
```
|
```
|
||||||
|
|
||||||
### Docker Compose
|
### Docker Compose (standalone)
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
@@ -299,45 +371,117 @@ volumes:
|
|||||||
objstdata:
|
objstdata:
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Docker Compose (3-node cluster)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
node1:
|
||||||
|
image: code.foss.global/lossless.zone/objectstorage:latest
|
||||||
|
ports:
|
||||||
|
- "9001:9000"
|
||||||
|
- "3001:3000"
|
||||||
|
- "4433:4433/udp"
|
||||||
|
volumes:
|
||||||
|
- node1-drive1:/drive1
|
||||||
|
- node1-drive2:/drive2
|
||||||
|
environment:
|
||||||
|
OBJST_CLUSTER_ENABLED: "true"
|
||||||
|
OBJST_CLUSTER_NODE_ID: node-1
|
||||||
|
OBJST_CLUSTER_QUIC_PORT: "4433"
|
||||||
|
OBJST_CLUSTER_SEED_NODES: node2:4433,node3:4433
|
||||||
|
OBJST_DRIVE_PATHS: /drive1,/drive2
|
||||||
|
OBJST_ACCESS_KEY: myadminkey
|
||||||
|
OBJST_SECRET_KEY: mysupersecret
|
||||||
|
|
||||||
|
node2:
|
||||||
|
image: code.foss.global/lossless.zone/objectstorage:latest
|
||||||
|
ports:
|
||||||
|
- "9002:9000"
|
||||||
|
- "3002:3000"
|
||||||
|
- "4434:4433/udp"
|
||||||
|
volumes:
|
||||||
|
- node2-drive1:/drive1
|
||||||
|
- node2-drive2:/drive2
|
||||||
|
environment:
|
||||||
|
OBJST_CLUSTER_ENABLED: "true"
|
||||||
|
OBJST_CLUSTER_NODE_ID: node-2
|
||||||
|
OBJST_CLUSTER_QUIC_PORT: "4433"
|
||||||
|
OBJST_CLUSTER_SEED_NODES: node1:4433,node3:4433
|
||||||
|
OBJST_DRIVE_PATHS: /drive1,/drive2
|
||||||
|
OBJST_ACCESS_KEY: myadminkey
|
||||||
|
OBJST_SECRET_KEY: mysupersecret
|
||||||
|
|
||||||
|
node3:
|
||||||
|
image: code.foss.global/lossless.zone/objectstorage:latest
|
||||||
|
ports:
|
||||||
|
- "9003:9000"
|
||||||
|
- "3003:3000"
|
||||||
|
- "4435:4433/udp"
|
||||||
|
volumes:
|
||||||
|
- node3-drive1:/drive1
|
||||||
|
- node3-drive2:/drive2
|
||||||
|
environment:
|
||||||
|
OBJST_CLUSTER_ENABLED: "true"
|
||||||
|
OBJST_CLUSTER_NODE_ID: node-3
|
||||||
|
OBJST_CLUSTER_QUIC_PORT: "4433"
|
||||||
|
OBJST_CLUSTER_SEED_NODES: node1:4433,node2:4433
|
||||||
|
OBJST_DRIVE_PATHS: /drive1,/drive2
|
||||||
|
OBJST_ACCESS_KEY: myadminkey
|
||||||
|
OBJST_SECRET_KEY: mysupersecret
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
node1-drive1:
|
||||||
|
node1-drive2:
|
||||||
|
node2-drive1:
|
||||||
|
node2-drive2:
|
||||||
|
node3-drive1:
|
||||||
|
node3-drive2:
|
||||||
|
```
|
||||||
|
|
||||||
### Image Details
|
### Image Details
|
||||||
|
|
||||||
- **Base**: `alpine:edge` with Deno runtime
|
- **Base**: `alpine:edge` with Deno runtime
|
||||||
- **Architectures**: `linux/amd64`, `linux/arm64`
|
- **Architectures**: `linux/amd64`, `linux/arm64`
|
||||||
- **Size**: ~150 MB compressed
|
- **Size**: ~150 MB compressed
|
||||||
- **Init system**: `tini` for proper signal handling
|
- **Init system**: `tini` for proper signal handling
|
||||||
- **Exposed ports**: `9000` (S3), `3000` (UI)
|
- **Exposed ports**: `9000` (S3), `3000` (UI), `4433` (QUIC cluster transport)
|
||||||
- **Volume**: `/data` — all bucket data and config persisted here
|
- **Volume**: `/data` — all bucket data and config persisted here
|
||||||
|
|
||||||
## 🏗️ Architecture
|
## 🏗️ Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
┌─────────────────────────────────────────────────┐
|
┌──────────────────────────────────────────────────────────────┐
|
||||||
│ objectstorage │
|
│ objectstorage │
|
||||||
│ │
|
│ │
|
||||||
│ ┌──────────────┐ ┌───────────────────────┐ │
|
│ ┌──────────────┐ ┌────────────────────────────────────┐ │
|
||||||
│ │ Management │ │ Storage Engine │ │
|
│ │ Management │ │ Storage Engine (smartstorage) │ │
|
||||||
│ │ UI (port │ │ (smartstorage/ │ │
|
│ │ UI (port │ │ Rust binary via ruststorage │ │
|
||||||
│ │ 3000) │ │ ruststorage) │ │
|
│ │ 3000) │ │ (port 9000) │ │
|
||||||
│ │ │ │ (port 9000) │ │
|
│ │ │ │ │ │
|
||||||
│ │ dees-catalog │ │ • S3 API compat │ │
|
│ │ dees-catalog │ │ • S3 API (path-style routing) │ │
|
||||||
│ │ SPA bundle │ │ • SigV4 auth │ │
|
│ │ SPA bundle │ │ • SigV4 authentication │ │
|
||||||
│ └──────┬───────┘ │ • Bucket policies │ │
|
│ └──────┬───────┘ │ • Bucket policies │ │
|
||||||
│ │ │ • Rust binary engine │ │
|
│ │ │ • Streaming I/O (zero-copy) │ │
|
||||||
│ ┌──────▼───────┐ └───────────────────────┘ │
|
│ ┌──────▼───────┐ │ • Multipart upload support │ │
|
||||||
│ │ OpsServer │ │
|
│ │ OpsServer │ └─────────────┬──────────────────────┘ │
|
||||||
│ │ (TypedReq │──── AWS SDK S3 Client ────────│
|
│ │ (TypedReq │ │ │
|
||||||
│ │ handlers) │ (manages own storage) │
|
│ │ handlers) │── S3 Client ──────┘ │
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│ │ • Admin auth │ │
|
│ │ • Admin auth │ ┌────────────────────────────────────┐ │
|
||||||
│ │ • CRUD APIs │ │
|
│ │ • CRUD APIs │ │ Cluster Layer (optional) │ │
|
||||||
│ │ • Policy mgr │ │
|
│ │ • Policy mgr │ │ │ │
|
||||||
│ └──────────────┘ │
|
│ └──────────────┘ │ • QUIC transport (port 4433) │ │
|
||||||
|
│ │ • Reed-Solomon erasure coding │ │
|
||||||
|
│ │ • Quorum writes / reads │ │
|
||||||
|
│ │ • Heartbeat failure detection │ │
|
||||||
|
│ │ • Self-healing shard repair │ │
|
||||||
|
│ └────────────────────────────────────┘ │
|
||||||
│ │
|
│ │
|
||||||
│ ┌──────────────────────────────────────────────┐│
|
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||||
│ │ /data (persistent volume) ││
|
│ │ Storage Drives │ │
|
||||||
│ │ buckets/ .policies/ .objectstorage/ ││
|
│ │ /drive1 /drive2 /drive3 ... (or single /data) │ │
|
||||||
│ └──────────────────────────────────────────────┘│
|
│ └──────────────────────────────────────────────────────────┘ │
|
||||||
└─────────────────────────────────────────────────┘
|
└──────────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
### Tech Stack
|
### Tech Stack
|
||||||
@@ -345,6 +489,8 @@ volumes:
|
|||||||
| Layer | Technology |
|
| Layer | Technology |
|
||||||
|---|---|
|
|---|---|
|
||||||
| **Storage Engine** | [`@push.rocks/smartstorage`](https://code.foss.global/push.rocks/smartstorage) (Rust binary via `ruststorage`) |
|
| **Storage Engine** | [`@push.rocks/smartstorage`](https://code.foss.global/push.rocks/smartstorage) (Rust binary via `ruststorage`) |
|
||||||
|
| **Cluster Transport** | QUIC via `quinn` (auto-TLS, multiplexed streams, backpressure) |
|
||||||
|
| **Erasure Coding** | Reed-Solomon (configurable data + parity shards) |
|
||||||
| **Runtime** | Deno |
|
| **Runtime** | Deno |
|
||||||
| **Management API** | [`@api.global/typedrequest`](https://code.foss.global/api.global/typedrequest) + [`@api.global/typedserver`](https://code.foss.global/api.global/typedserver) |
|
| **Management API** | [`@api.global/typedrequest`](https://code.foss.global/api.global/typedrequest) + [`@api.global/typedserver`](https://code.foss.global/api.global/typedserver) |
|
||||||
| **Auth** | JWT via [`@push.rocks/smartjwt`](https://code.foss.global/push.rocks/smartjwt), S3 SigV4 |
|
| **Auth** | JWT via [`@push.rocks/smartjwt`](https://code.foss.global/push.rocks/smartjwt), S3 SigV4 |
|
||||||
@@ -367,13 +513,16 @@ pnpm run build
|
|||||||
# Type check backend
|
# Type check backend
|
||||||
deno check mod.ts
|
deno check mod.ts
|
||||||
|
|
||||||
|
# Run tests
|
||||||
|
pnpm test
|
||||||
|
|
||||||
# Run in development mode
|
# Run in development mode
|
||||||
deno run --allow-all mod.ts server --ephemeral
|
deno run --allow-all mod.ts server --ephemeral
|
||||||
```
|
```
|
||||||
|
|
||||||
## License and Legal Information
|
## License and Legal Information
|
||||||
|
|
||||||
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file.
|
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [license](./license) file.
|
||||||
|
|
||||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
|||||||
+125
@@ -0,0 +1,125 @@
|
|||||||
|
# Improvement Plan
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Make this repository a solid product wrapper for `@push.rocks/smartstorage`: reliable Docker image, predictable runtime behavior, durable admin configuration, and a management surface that scales past toy datasets.
|
||||||
|
|
||||||
|
## Current Position
|
||||||
|
|
||||||
|
The repository already has a clear structure and decent coverage for auth, buckets, objects, policies, credentials, status/config, lifecycle, and S3 compatibility.
|
||||||
|
|
||||||
|
The main gaps are product hardening and dependency boundaries:
|
||||||
|
|
||||||
|
- the Docker build still references `npmextra.json` even though the repo migrated to `.smartconfig.json`
|
||||||
|
- some admin/runtime behavior is still in-memory only
|
||||||
|
- status and bucket summaries are computed by rescanning storage through S3 calls
|
||||||
|
- the product surface does not yet expose real cluster and drive health
|
||||||
|
- the UI error model is still mostly `console.error()` + stale state retention
|
||||||
|
|
||||||
|
## Repo-Owned Work
|
||||||
|
|
||||||
|
### 1. Fix Packaging Regressions First
|
||||||
|
|
||||||
|
- Update `Dockerfile` to stop copying `npmextra.json` and use the current build config layout.
|
||||||
|
- Add a Docker smoke test that proves the image builds and the container starts with both ports exposed.
|
||||||
|
|
||||||
|
Reason:
|
||||||
|
|
||||||
|
- This repo is explicitly about shipping a usable container. A broken or stale Docker build path is a product failure, not a nice-to-have cleanup.
|
||||||
|
|
||||||
|
### 2. Make Admin Configuration Durable
|
||||||
|
|
||||||
|
- Decide which runtime-managed settings must survive restart.
|
||||||
|
- Credentials are the biggest gap today: add/remove currently mutates runtime config only.
|
||||||
|
- Align credential durability with the existing named-policy persistence model.
|
||||||
|
- Add restart-persistence tests.
|
||||||
|
|
||||||
|
Reason:
|
||||||
|
|
||||||
|
- A product container should not pretend to manage credentials if those changes disappear on restart.
|
||||||
|
|
||||||
|
### 3. Tighten Management API Behavior
|
||||||
|
|
||||||
|
- Fix bucket/policy cleanup ordering so deleting a bucket does not trigger follow-up work against a missing bucket.
|
||||||
|
- Return cleaner typed errors from handlers where failure is expected.
|
||||||
|
- Surface actionable errors in the UI instead of only logging and keeping stale state.
|
||||||
|
|
||||||
|
Reason:
|
||||||
|
|
||||||
|
- The current tests pass, but post-test output already shows avoidable `NoSuchBucket` noise during policy cleanup.
|
||||||
|
|
||||||
|
### 4. Reduce Large-Object and Large-Dataset Pain
|
||||||
|
|
||||||
|
- Keep inline editing and preview for small objects.
|
||||||
|
- Add explicit size thresholds for inline/base64 flows.
|
||||||
|
- Prefer direct object URLs or streaming paths for large-object download/preview paths.
|
||||||
|
- Stop treating expensive full-dataset scans as normal refresh behavior.
|
||||||
|
|
||||||
|
Reason:
|
||||||
|
|
||||||
|
- The product should stay usable once users have real buckets and non-trivial object counts.
|
||||||
|
|
||||||
|
### 5. Add Product Acceptance Coverage
|
||||||
|
|
||||||
|
- Docker build/start smoke test.
|
||||||
|
- Restart-persistence test for credentials and policies.
|
||||||
|
- Cluster-config smoke test.
|
||||||
|
- One browser-level smoke test for login, bucket browse, and config view.
|
||||||
|
|
||||||
|
Reason:
|
||||||
|
|
||||||
|
- The repo is productization code. Acceptance coverage matters more here than unit-test density.
|
||||||
|
|
||||||
|
## Dependency-Owned Work Completed
|
||||||
|
|
||||||
|
The dependency boundary work landed in `@push.rocks/smartstorage` `v6.4.0` and is now consumed here through the Deno import map.
|
||||||
|
|
||||||
|
- Runtime storage stats and bucket summaries come from `smartstorage` instead of product-side S3 scans.
|
||||||
|
- Runtime credential listing/replacement uses supported `smartstorage` APIs instead of mutating engine internals.
|
||||||
|
- Cluster and drive health are exposed through `smartstorage` and surfaced through the management API/UI.
|
||||||
|
|
||||||
|
## Suggested Execution Order
|
||||||
|
|
||||||
|
1. Fix Dockerfile and add a container smoke test.
|
||||||
|
2. Make credential changes durable and test restart behavior.
|
||||||
|
3. Clean up bucket/policy deletion behavior and UI error reporting.
|
||||||
|
4. Keep product code on supported `smartstorage` runtime APIs for stats, credentials, and cluster health.
|
||||||
|
5. Add browser-level smoke coverage for login, bucket browse, and config views.
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
- Docker image builds from a clean checkout.
|
||||||
|
- Runtime-managed credentials survive restart or are explicitly documented as ephemeral until fixed.
|
||||||
|
- Status and bucket views no longer require full object scans for routine refreshes.
|
||||||
|
- Bucket deletion and policy cleanup complete without noisy missing-bucket follow-up errors.
|
||||||
|
- Cluster mode exposes live health, not just configured values.
|
||||||
|
|
||||||
|
## Enterprise Readiness Plan
|
||||||
|
|
||||||
|
### Acceptance Criteria Implemented In-Repo
|
||||||
|
|
||||||
|
- Runtime health: expose unauthenticated `/livez`, `/readyz`, `/healthz`, and `/metrics` on the management server.
|
||||||
|
- Container health: Docker `HEALTHCHECK` must use `/readyz` and the image must run as a non-root user.
|
||||||
|
- Admin auditability: login, logout, bucket mutation, object mutation, credential mutation, and named-policy mutation must emit append-only audit events under `.objectstorage/audit.log`.
|
||||||
|
- Audit access: admins can query recent audit entries through a typed management API.
|
||||||
|
- Session security: JWT role is validated from verified claims, logout revokes the active token, and repeated failed logins are rate-limited.
|
||||||
|
- Secret persistence: admin metadata files are written with restrictive file mode where the OS supports it.
|
||||||
|
- Default-secret guardrail: persistent `/data` deployments refuse `admin/admin` defaults unless explicitly allowed for disposable development.
|
||||||
|
- Frontend session storage: admin identity is no longer stored in persistent browser state.
|
||||||
|
|
||||||
|
### Dependency Work Implemented In `@push.rocks/smartstorage`
|
||||||
|
|
||||||
|
- Cluster identity and topology snapshots persist under `.smartstorage/cluster/`.
|
||||||
|
- Cluster startup refuses unsafe seed-node fallback instead of silently forming a split-brain cluster.
|
||||||
|
- Heartbeats probe all known peers, including suspect/offline peers, using configured timeout.
|
||||||
|
- Operational S3-side endpoints expose `/-/live`, `/-/ready`, `/-/health`, and `/-/metrics`.
|
||||||
|
- Runtime credential listing returns metadata only; secrets are write-only.
|
||||||
|
- Multi-node tests cover topology convergence, remote drive routing, and restart/rejoin identity durability.
|
||||||
|
|
||||||
|
### Production Requirements Outside This Repo
|
||||||
|
|
||||||
|
- Run the management UI behind TLS or add native typedserver TLS configuration for the deployment.
|
||||||
|
- Configure real admin and S3 credentials through secrets management, not image defaults.
|
||||||
|
- Decide cluster transport policy: pinned CA or mTLS with operational certificate rotation. Do not operate QUIC insecure-dev transport on untrusted networks.
|
||||||
|
- Define backup/restore procedures for `/data`, `.objectstorage`, `.smartstorage/cluster`, bucket manifests, and policies.
|
||||||
|
- Add load, soak, and failure-injection runs to release qualification for large datasets and network partitions.
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import { ObjectStorageContainer } from '../../ts/index.ts';
|
||||||
|
import type { IObjectStorageConfig } from '../../ts/types.ts';
|
||||||
|
import type * as interfaces from '../../ts_interfaces/index.ts';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import type { IReq_AdminLoginWithUsernameAndPassword } from '../../ts_interfaces/requests/admin.ts';
|
||||||
|
|
||||||
|
export const TEST_ADMIN_PASSWORD = 'testpassword';
|
||||||
|
export const TEST_ACCESS_KEY = 'testkey';
|
||||||
|
export const TEST_SECRET_KEY = 'testsecret';
|
||||||
|
|
||||||
|
export function getTestPorts(index: number): { objstPort: number; uiPort: number } {
|
||||||
|
return {
|
||||||
|
objstPort: 19000 + index * 10,
|
||||||
|
uiPort: 19001 + index * 10,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createTestContainer(
|
||||||
|
index: number,
|
||||||
|
extraConfig?: Partial<IObjectStorageConfig>,
|
||||||
|
): ObjectStorageContainer {
|
||||||
|
const ports = getTestPorts(index);
|
||||||
|
return new ObjectStorageContainer({
|
||||||
|
objstPort: ports.objstPort,
|
||||||
|
uiPort: ports.uiPort,
|
||||||
|
storageDirectory: `.nogit/testdata-${index}`,
|
||||||
|
accessCredentials: [{ accessKeyId: TEST_ACCESS_KEY, secretAccessKey: TEST_SECRET_KEY }],
|
||||||
|
adminPassword: TEST_ADMIN_PASSWORD,
|
||||||
|
region: 'us-east-1',
|
||||||
|
...extraConfig,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loginAndGetIdentity(
|
||||||
|
uiPort: number,
|
||||||
|
): Promise<interfaces.data.IIdentity> {
|
||||||
|
const req = new TypedRequest<IReq_AdminLoginWithUsernameAndPassword>(
|
||||||
|
`http://localhost:${uiPort}/typedrequest`,
|
||||||
|
'adminLoginWithUsernameAndPassword',
|
||||||
|
);
|
||||||
|
const response = await req.fire({
|
||||||
|
username: 'admin',
|
||||||
|
password: TEST_ADMIN_PASSWORD,
|
||||||
|
});
|
||||||
|
if (!response.identity) {
|
||||||
|
throw new Error('Login failed: no identity returned');
|
||||||
|
}
|
||||||
|
return response.identity;
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
import { assertEquals, assertExists } from 'jsr:@std/assert';
|
||||||
|
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import { createTestContainer, getTestPorts, loginAndGetIdentity, TEST_ADMIN_PASSWORD } from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type * as interfaces from '../ts_interfaces/index.ts';
|
||||||
|
import type { IReq_AdminLoginWithUsernameAndPassword } from '../ts_interfaces/requests/admin.ts';
|
||||||
|
import type { IReq_VerifyIdentity } from '../ts_interfaces/requests/admin.ts';
|
||||||
|
import type { IReq_AdminLogout } from '../ts_interfaces/requests/admin.ts';
|
||||||
|
import type { IReq_GetServerStatus } from '../ts_interfaces/requests/status.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 1;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
|
||||||
|
describe('Authentication', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
let container: ObjectStorageContainer;
|
||||||
|
let identity: interfaces.data.IIdentity;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
container = createTestContainer(PORT_INDEX);
|
||||||
|
await container.start();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await container.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should login with valid credentials', async () => {
|
||||||
|
identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
assertExists(identity.jwt);
|
||||||
|
assertEquals(identity.userId, 'admin');
|
||||||
|
assertEquals(identity.username, 'admin');
|
||||||
|
assertEquals(identity.role, 'admin');
|
||||||
|
assertEquals(identity.expiresAt > Date.now(), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reject login with wrong password', async () => {
|
||||||
|
const req = new TypedRequest<IReq_AdminLoginWithUsernameAndPassword>(
|
||||||
|
url,
|
||||||
|
'adminLoginWithUsernameAndPassword',
|
||||||
|
);
|
||||||
|
let threw = false;
|
||||||
|
try {
|
||||||
|
await req.fire({ username: 'admin', password: 'wrongpassword' });
|
||||||
|
} catch {
|
||||||
|
threw = true;
|
||||||
|
}
|
||||||
|
assertEquals(threw, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reject login with wrong username', async () => {
|
||||||
|
const req = new TypedRequest<IReq_AdminLoginWithUsernameAndPassword>(
|
||||||
|
url,
|
||||||
|
'adminLoginWithUsernameAndPassword',
|
||||||
|
);
|
||||||
|
let threw = false;
|
||||||
|
try {
|
||||||
|
await req.fire({ username: 'notadmin', password: TEST_ADMIN_PASSWORD });
|
||||||
|
} catch {
|
||||||
|
threw = true;
|
||||||
|
}
|
||||||
|
assertEquals(threw, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify a valid identity', async () => {
|
||||||
|
const req = new TypedRequest<IReq_VerifyIdentity>(url, 'verifyIdentity');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.valid, true);
|
||||||
|
assertExists(response.identity);
|
||||||
|
assertEquals(response.identity!.userId, 'admin');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reject verification with tampered JWT', async () => {
|
||||||
|
const req = new TypedRequest<IReq_VerifyIdentity>(url, 'verifyIdentity');
|
||||||
|
const tamperedIdentity = { ...identity, jwt: identity.jwt + 'tampered' };
|
||||||
|
const response = await req.fire({ identity: tamperedIdentity });
|
||||||
|
assertEquals(response.valid, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reject verification with missing identity', async () => {
|
||||||
|
const req = new TypedRequest<IReq_VerifyIdentity>(url, 'verifyIdentity');
|
||||||
|
const response = await req.fire({ identity: null as any });
|
||||||
|
assertEquals(response.valid, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should logout successfully', async () => {
|
||||||
|
const req = new TypedRequest<IReq_AdminLogout>(url, 'adminLogout');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reject protected endpoint without identity', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetServerStatus>(url, 'getServerStatus');
|
||||||
|
let threw = false;
|
||||||
|
try {
|
||||||
|
await req.fire({ identity: null as any });
|
||||||
|
} catch {
|
||||||
|
threw = true;
|
||||||
|
}
|
||||||
|
assertEquals(threw, true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
import { assertEquals, assertExists } from 'jsr:@std/assert';
|
||||||
|
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type * as interfaces from '../ts_interfaces/index.ts';
|
||||||
|
import type {
|
||||||
|
IReq_ListBuckets,
|
||||||
|
IReq_CreateBucket,
|
||||||
|
IReq_DeleteBucket,
|
||||||
|
IReq_GetBucketPolicy,
|
||||||
|
IReq_PutBucketPolicy,
|
||||||
|
IReq_DeleteBucketPolicy,
|
||||||
|
} from '../ts_interfaces/requests/buckets.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 2;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
|
||||||
|
describe('Bucket management', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
let container: ObjectStorageContainer;
|
||||||
|
let identity: interfaces.data.IIdentity;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
container = createTestContainer(PORT_INDEX);
|
||||||
|
await container.start();
|
||||||
|
identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await container.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list buckets (initially empty)', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.buckets.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should create a bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should create a second bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-2' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list both buckets', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.buckets.length, 2);
|
||||||
|
const names = response.buckets.map((b) => b.name).sort();
|
||||||
|
assertEquals(names, ['test-bucket-1', 'test-bucket-2']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should have valid bucket metadata', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
for (const bucket of response.buckets) {
|
||||||
|
assertEquals(bucket.objectCount, 0);
|
||||||
|
assertEquals(bucket.totalSizeBytes, 0);
|
||||||
|
assertEquals(bucket.creationDate > 0, true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should delete a bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-2' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list one remaining bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.buckets.length, 1);
|
||||||
|
assertEquals(response.buckets[0].name, 'test-bucket-1');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get bucket policy (initially null)', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
|
||||||
|
assertEquals(response.policy, null);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should put a bucket policy', async () => {
|
||||||
|
const policy = JSON.stringify({
|
||||||
|
Version: '2012-10-17',
|
||||||
|
Statement: [
|
||||||
|
{
|
||||||
|
Sid: 'PublicRead',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: 'arn:aws:s3:::test-bucket-1/*',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const req = new TypedRequest<IReq_PutBucketPolicy>(url, 'putBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-1', policy });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get the bucket policy back', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
|
||||||
|
assertExists(response.policy);
|
||||||
|
const parsed = JSON.parse(response.policy!);
|
||||||
|
assertExists(parsed.Statement);
|
||||||
|
assertEquals(parsed.Statement[0].Sid, 'PublicRead');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should delete bucket policy', async () => {
|
||||||
|
const req = new TypedRequest<IReq_DeleteBucketPolicy>(url, 'deleteBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should confirm policy is deleted', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
|
||||||
|
assertEquals(response.policy, null);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('cleanup: delete remaining bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
import { assertEquals } from 'jsr:@std/assert';
|
||||||
|
import { describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type { IReq_GetClusterHealth } from '../ts_interfaces/requests/status.ts';
|
||||||
|
import type * as interfaces from '../ts_interfaces/index.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 9;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
const storageDirectory = `.nogit/testdata-${PORT_INDEX}`;
|
||||||
|
|
||||||
|
const cleanupStorageDirectory = async () => {
|
||||||
|
try {
|
||||||
|
await Deno.remove(storageDirectory, { recursive: true });
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Deno.errors.NotFound)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('Cluster health', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
it('exposes smartstorage cluster health through the management API', async () => {
|
||||||
|
await cleanupStorageDirectory();
|
||||||
|
|
||||||
|
let container: ObjectStorageContainer | null = null;
|
||||||
|
try {
|
||||||
|
const drivePaths = Array.from({ length: 6 }, (_value, index) => {
|
||||||
|
return `${storageDirectory}/drive-${index + 1}`;
|
||||||
|
});
|
||||||
|
|
||||||
|
container = createTestContainer(PORT_INDEX, {
|
||||||
|
storageDirectory,
|
||||||
|
clusterEnabled: true,
|
||||||
|
clusterNodeId: 'objectstorage-test-node',
|
||||||
|
clusterQuicPort: 19433,
|
||||||
|
drivePaths,
|
||||||
|
erasureDataShards: 4,
|
||||||
|
erasureParityShards: 2,
|
||||||
|
erasureChunkSizeBytes: 1024 * 1024,
|
||||||
|
});
|
||||||
|
await container.start();
|
||||||
|
|
||||||
|
const identity: interfaces.data.IIdentity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
const req = new TypedRequest<IReq_GetClusterHealth>(url, 'getClusterHealth');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
const health = response.clusterHealth;
|
||||||
|
|
||||||
|
assertEquals(health.enabled, true);
|
||||||
|
assertEquals(health.nodeId, 'objectstorage-test-node');
|
||||||
|
assertEquals(health.quorumHealthy, true);
|
||||||
|
assertEquals(health.majorityHealthy, true);
|
||||||
|
assertEquals(health.drives?.length, 6);
|
||||||
|
assertEquals(health.drives?.every((drive) => drive.status === 'online'), true);
|
||||||
|
assertEquals(health.erasure?.dataShards, 4);
|
||||||
|
assertEquals(health.erasure?.parityShards, 2);
|
||||||
|
assertEquals(health.erasure?.totalShards, 6);
|
||||||
|
} finally {
|
||||||
|
if (container) {
|
||||||
|
await container.stop();
|
||||||
|
}
|
||||||
|
await cleanupStorageDirectory();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import { assertEquals, assertExists } from 'jsr:@std/assert';
|
||||||
|
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { createTestContainer, getTestPorts } from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import { defaultConfig } from '../ts/types.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 0;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
|
||||||
|
describe('ObjectStorageContainer lifecycle', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
let container: ObjectStorageContainer;
|
||||||
|
|
||||||
|
it('should create container with default config values', () => {
|
||||||
|
const c = new ObjectStorageContainer();
|
||||||
|
assertEquals(c.config.objstPort, defaultConfig.objstPort);
|
||||||
|
assertEquals(c.config.uiPort, defaultConfig.uiPort);
|
||||||
|
assertEquals(c.config.region, defaultConfig.region);
|
||||||
|
assertEquals(c.config.adminPassword, defaultConfig.adminPassword);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should create container with custom config overrides', () => {
|
||||||
|
container = createTestContainer(PORT_INDEX);
|
||||||
|
assertEquals(container.config.objstPort, ports.objstPort);
|
||||||
|
assertEquals(container.config.uiPort, ports.uiPort);
|
||||||
|
assertEquals(container.config.adminPassword, 'testpassword');
|
||||||
|
assertEquals(container.config.accessCredentials[0].accessKeyId, 'testkey');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should start successfully', async () => {
|
||||||
|
await container.start();
|
||||||
|
assertEquals(container.startedAt > 0, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should have s3Client initialized after start', () => {
|
||||||
|
assertExists(container.s3Client);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should have smartstorageInstance initialized after start', () => {
|
||||||
|
assertExists(container.smartstorageInstance);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should have policyManager with empty policies', () => {
|
||||||
|
assertExists(container.policyManager);
|
||||||
|
const policies = container.policyManager.listPolicies();
|
||||||
|
assertEquals(policies.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should have opsServer started', () => {
|
||||||
|
assertExists(container.opsServer);
|
||||||
|
assertExists(container.opsServer.server);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should stop cleanly', async () => {
|
||||||
|
await container.stop();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
import { assertEquals, assertRejects } from 'jsr:@std/assert';
|
||||||
|
import { describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import {
|
||||||
|
createTestContainer,
|
||||||
|
getTestPorts,
|
||||||
|
loginAndGetIdentity,
|
||||||
|
TEST_ACCESS_KEY,
|
||||||
|
} from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type { IReq_CreateBucket, IReq_ListBuckets } from '../ts_interfaces/requests/buckets.ts';
|
||||||
|
import type {
|
||||||
|
IReq_AddCredential,
|
||||||
|
IReq_GetCredentials,
|
||||||
|
IReq_RemoveCredential,
|
||||||
|
} from '../ts_interfaces/requests/credentials.ts';
|
||||||
|
import type * as interfaces from '../ts_interfaces/index.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 8;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
const storageDirectory = `.nogit/testdata-${PORT_INDEX}`;
|
||||||
|
|
||||||
|
const cleanupStorageDirectory = async () => {
|
||||||
|
try {
|
||||||
|
await Deno.remove(storageDirectory, { recursive: true });
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Deno.errors.NotFound)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('Credential persistence', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
it('persists managed credentials across restart and refreshes the internal client', async () => {
|
||||||
|
await cleanupStorageDirectory();
|
||||||
|
|
||||||
|
let activeContainer: ObjectStorageContainer | null = null;
|
||||||
|
|
||||||
|
const stopContainer = async () => {
|
||||||
|
if (!activeContainer) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await activeContainer.stop();
|
||||||
|
} finally {
|
||||||
|
activeContainer = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
activeContainer = createTestContainer(PORT_INDEX);
|
||||||
|
await activeContainer.start();
|
||||||
|
|
||||||
|
let identity: interfaces.data.IIdentity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
|
||||||
|
const addCredential = new TypedRequest<IReq_AddCredential>(url, 'addCredential');
|
||||||
|
await addCredential.fire({
|
||||||
|
identity,
|
||||||
|
accessKeyId: 'persisted-key',
|
||||||
|
secretAccessKey: 'persisted-secret',
|
||||||
|
});
|
||||||
|
|
||||||
|
const removeCredential = new TypedRequest<IReq_RemoveCredential>(url, 'removeCredential');
|
||||||
|
await removeCredential.fire({ identity, accessKeyId: TEST_ACCESS_KEY });
|
||||||
|
|
||||||
|
const getCredentials = new TypedRequest<IReq_GetCredentials>(url, 'getCredentials');
|
||||||
|
const credentialsBeforeRestart = await getCredentials.fire({ identity });
|
||||||
|
assertEquals(credentialsBeforeRestart.credentials.length, 1);
|
||||||
|
assertEquals(credentialsBeforeRestart.credentials[0].accessKeyId, 'persisted-key');
|
||||||
|
|
||||||
|
const listBuckets = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
|
||||||
|
const bucketsBeforeRestart = await listBuckets.fire({ identity });
|
||||||
|
assertEquals(Array.isArray(bucketsBeforeRestart.buckets), true);
|
||||||
|
|
||||||
|
await stopContainer();
|
||||||
|
|
||||||
|
activeContainer = createTestContainer(PORT_INDEX);
|
||||||
|
await activeContainer.start();
|
||||||
|
|
||||||
|
identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
|
||||||
|
const credentialsAfterRestart = await getCredentials.fire({ identity });
|
||||||
|
assertEquals(credentialsAfterRestart.credentials.length, 1);
|
||||||
|
assertEquals(credentialsAfterRestart.credentials[0].accessKeyId, 'persisted-key');
|
||||||
|
|
||||||
|
const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
|
||||||
|
await createBucket.fire({ identity, bucketName: 'persisted-creds-bucket' });
|
||||||
|
|
||||||
|
const bucketsAfterRestart = await listBuckets.fire({ identity });
|
||||||
|
assertEquals(
|
||||||
|
bucketsAfterRestart.buckets.some((bucket) => bucket.name === 'persisted-creds-bucket'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await stopContainer();
|
||||||
|
await cleanupStorageDirectory();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('lets explicit environment credentials override persisted managed credentials', async () => {
|
||||||
|
const portIndex = 10;
|
||||||
|
const envPorts = getTestPorts(portIndex);
|
||||||
|
const envUrl = `http://localhost:${envPorts.uiPort}/typedrequest`;
|
||||||
|
const envStorageDirectory = `.nogit/testdata-${portIndex}`;
|
||||||
|
const previousAccessKey = Deno.env.get('OBJST_ACCESS_KEY');
|
||||||
|
const previousSecretKey = Deno.env.get('OBJST_SECRET_KEY');
|
||||||
|
let container: ObjectStorageContainer | null = null;
|
||||||
|
|
||||||
|
const cleanupEnvStorageDirectory = async () => {
|
||||||
|
try {
|
||||||
|
await Deno.remove(envStorageDirectory, { recursive: true });
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Deno.errors.NotFound)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await cleanupEnvStorageDirectory();
|
||||||
|
await Deno.mkdir(`${envStorageDirectory}/.objectstorage`, { recursive: true });
|
||||||
|
await Deno.writeTextFile(
|
||||||
|
`${envStorageDirectory}/.objectstorage/admin-config.json`,
|
||||||
|
JSON.stringify({
|
||||||
|
accessCredentials: [{ accessKeyId: 'persisted-key', secretAccessKey: 'persisted-secret' }],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
Deno.env.set('OBJST_ACCESS_KEY', 'env-key');
|
||||||
|
Deno.env.set('OBJST_SECRET_KEY', 'env-secret');
|
||||||
|
|
||||||
|
container = createTestContainer(portIndex, { storageDirectory: envStorageDirectory });
|
||||||
|
await container.start();
|
||||||
|
|
||||||
|
const identity = await loginAndGetIdentity(envPorts.uiPort);
|
||||||
|
const getCredentials = new TypedRequest<IReq_GetCredentials>(envUrl, 'getCredentials');
|
||||||
|
const response = await getCredentials.fire({ identity });
|
||||||
|
|
||||||
|
assertEquals(response.credentials.length, 1);
|
||||||
|
assertEquals(response.credentials[0].accessKeyId, 'env-key');
|
||||||
|
} finally {
|
||||||
|
if (container) {
|
||||||
|
await container.stop();
|
||||||
|
}
|
||||||
|
if (previousAccessKey === undefined) {
|
||||||
|
Deno.env.delete('OBJST_ACCESS_KEY');
|
||||||
|
} else {
|
||||||
|
Deno.env.set('OBJST_ACCESS_KEY', previousAccessKey);
|
||||||
|
}
|
||||||
|
if (previousSecretKey === undefined) {
|
||||||
|
Deno.env.delete('OBJST_SECRET_KEY');
|
||||||
|
} else {
|
||||||
|
Deno.env.set('OBJST_SECRET_KEY', previousSecretKey);
|
||||||
|
}
|
||||||
|
await cleanupEnvStorageDirectory();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not persist rejected credential replacements', async () => {
|
||||||
|
const portIndex = 11;
|
||||||
|
const rejectPorts = getTestPorts(portIndex);
|
||||||
|
const rejectUrl = `http://localhost:${rejectPorts.uiPort}/typedrequest`;
|
||||||
|
const rejectStorageDirectory = `.nogit/testdata-${portIndex}`;
|
||||||
|
let container: ObjectStorageContainer | null = null;
|
||||||
|
|
||||||
|
const cleanupRejectStorageDirectory = async () => {
|
||||||
|
try {
|
||||||
|
await Deno.remove(rejectStorageDirectory, { recursive: true });
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Deno.errors.NotFound)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await cleanupRejectStorageDirectory();
|
||||||
|
container = createTestContainer(portIndex, { storageDirectory: rejectStorageDirectory });
|
||||||
|
await container.start();
|
||||||
|
|
||||||
|
await assertRejects(() =>
|
||||||
|
container!.replaceAccessCredentials([
|
||||||
|
{ accessKeyId: 'duplicate-key', secretAccessKey: 'secret-a' },
|
||||||
|
{ accessKeyId: 'duplicate-key', secretAccessKey: 'secret-b' },
|
||||||
|
])
|
||||||
|
);
|
||||||
|
|
||||||
|
const identity = await loginAndGetIdentity(rejectPorts.uiPort);
|
||||||
|
const getCredentials = new TypedRequest<IReq_GetCredentials>(rejectUrl, 'getCredentials');
|
||||||
|
const response = await getCredentials.fire({ identity });
|
||||||
|
|
||||||
|
assertEquals(response.credentials.length, 1);
|
||||||
|
assertEquals(response.credentials[0].accessKeyId, TEST_ACCESS_KEY);
|
||||||
|
await assertRejects(() =>
|
||||||
|
Deno.readTextFile(`${rejectStorageDirectory}/.objectstorage/admin-config.json`)
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
if (container) {
|
||||||
|
await container.stop();
|
||||||
|
}
|
||||||
|
await cleanupRejectStorageDirectory();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
import { assertEquals } from 'jsr:@std/assert';
|
||||||
|
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import {
|
||||||
|
createTestContainer,
|
||||||
|
getTestPorts,
|
||||||
|
loginAndGetIdentity,
|
||||||
|
TEST_ACCESS_KEY,
|
||||||
|
} from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type * as interfaces from '../ts_interfaces/index.ts';
|
||||||
|
import type {
|
||||||
|
IReq_GetCredentials,
|
||||||
|
IReq_AddCredential,
|
||||||
|
IReq_RemoveCredential,
|
||||||
|
} from '../ts_interfaces/requests/credentials.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 6;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
|
||||||
|
describe('Credential management', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
let container: ObjectStorageContainer;
|
||||||
|
let identity: interfaces.data.IIdentity;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
container = createTestContainer(PORT_INDEX);
|
||||||
|
await container.start();
|
||||||
|
identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await container.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get credentials with masked secrets', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetCredentials>(url, 'getCredentials');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.credentials.length, 1);
|
||||||
|
assertEquals(response.credentials[0].accessKeyId, TEST_ACCESS_KEY);
|
||||||
|
// Secret should be masked: first 4 chars + ****
|
||||||
|
assertEquals(response.credentials[0].secretAccessKey.includes('****'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should add a new credential', async () => {
|
||||||
|
const req = new TypedRequest<IReq_AddCredential>(url, 'addCredential');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
accessKeyId: 'newkey',
|
||||||
|
secretAccessKey: 'newsecret',
|
||||||
|
});
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list credentials showing both', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetCredentials>(url, 'getCredentials');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.credentials.length, 2);
|
||||||
|
const keys = response.credentials.map((c) => c.accessKeyId);
|
||||||
|
assertEquals(keys.includes(TEST_ACCESS_KEY), true);
|
||||||
|
assertEquals(keys.includes('newkey'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify new credential is stored in config', () => {
|
||||||
|
const creds = container.config.accessCredentials;
|
||||||
|
const newCred = creds.find((c) => c.accessKeyId === 'newkey');
|
||||||
|
assertEquals(newCred?.secretAccessKey, 'newsecret');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should remove a credential', async () => {
|
||||||
|
const req = new TypedRequest<IReq_RemoveCredential>(url, 'removeCredential');
|
||||||
|
const response = await req.fire({ identity, accessKeyId: 'newkey' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list credentials showing one remaining', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetCredentials>(url, 'getCredentials');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.credentials.length, 1);
|
||||||
|
assertEquals(response.credentials[0].accessKeyId, TEST_ACCESS_KEY);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reject removing the last credential', async () => {
|
||||||
|
const req = new TypedRequest<IReq_RemoveCredential>(url, 'removeCredential');
|
||||||
|
let threw = false;
|
||||||
|
try {
|
||||||
|
await req.fire({ identity, accessKeyId: TEST_ACCESS_KEY });
|
||||||
|
} catch {
|
||||||
|
threw = true;
|
||||||
|
}
|
||||||
|
assertEquals(threw, true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import { assertEquals } from 'jsr:@std/assert';
|
||||||
|
|
||||||
|
const shouldRunDockerSmoke = Deno.env.get('OBJST_RUN_DOCKER_SMOKE') === '1';
|
||||||
|
|
||||||
|
interface ICommandResult {
|
||||||
|
code: number;
|
||||||
|
stdout: string;
|
||||||
|
stderr: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runCommand(
|
||||||
|
command: string[],
|
||||||
|
options: { cwd?: string; check?: boolean } = {},
|
||||||
|
): Promise<ICommandResult> {
|
||||||
|
const output = await new Deno.Command(command[0], {
|
||||||
|
args: command.slice(1),
|
||||||
|
cwd: options.cwd,
|
||||||
|
stdout: 'piped',
|
||||||
|
stderr: 'piped',
|
||||||
|
}).output();
|
||||||
|
|
||||||
|
const result = {
|
||||||
|
code: output.code,
|
||||||
|
stdout: new TextDecoder().decode(output.stdout).trim(),
|
||||||
|
stderr: new TextDecoder().decode(output.stderr).trim(),
|
||||||
|
};
|
||||||
|
|
||||||
|
if (options.check !== false && result.code !== 0) {
|
||||||
|
throw new Error(`Command failed: ${command.join(' ')}\n${result.stderr}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForOk(url: string, timeoutMs: number): Promise<void> {
|
||||||
|
const deadline = Date.now() + timeoutMs;
|
||||||
|
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
try {
|
||||||
|
const response = await fetch(url);
|
||||||
|
if (response.ok) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Container may still be starting.
|
||||||
|
}
|
||||||
|
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Timed out waiting for ${url}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
Deno.test({
|
||||||
|
name: 'Docker image builds and serves the management UI',
|
||||||
|
ignore: !shouldRunDockerSmoke,
|
||||||
|
sanitizeOps: false,
|
||||||
|
sanitizeResources: false,
|
||||||
|
fn: async () => {
|
||||||
|
await runCommand(['docker', '--version']);
|
||||||
|
|
||||||
|
const imageTag = `objectstorage-smoke:${crypto.randomUUID().slice(0, 8)}`;
|
||||||
|
const containerName = `objectstorage-smoke-${crypto.randomUUID().slice(0, 8)}`;
|
||||||
|
const storagePort = 19190;
|
||||||
|
const uiPort = 19191;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await runCommand(['docker', 'build', '-t', imageTag, '.']);
|
||||||
|
|
||||||
|
const runResult = await runCommand([
|
||||||
|
'docker',
|
||||||
|
'run',
|
||||||
|
'-d',
|
||||||
|
'--name',
|
||||||
|
containerName,
|
||||||
|
'-p',
|
||||||
|
`${storagePort}:9000`,
|
||||||
|
'-p',
|
||||||
|
`${uiPort}:3000`,
|
||||||
|
'-e',
|
||||||
|
'OBJST_ADMIN_PASSWORD=docker-smoke-admin',
|
||||||
|
'-e',
|
||||||
|
'OBJST_ACCESS_KEY=docker-smoke-key',
|
||||||
|
'-e',
|
||||||
|
'OBJST_SECRET_KEY=docker-smoke-secret',
|
||||||
|
imageTag,
|
||||||
|
]);
|
||||||
|
|
||||||
|
assertEquals(runResult.stdout.length > 0, true);
|
||||||
|
await waitForOk(`http://127.0.0.1:${uiPort}/readyz`, 30000);
|
||||||
|
await waitForOk(`http://127.0.0.1:${storagePort}/-/ready`, 30000);
|
||||||
|
} finally {
|
||||||
|
await runCommand(['docker', 'rm', '-f', containerName], { check: false });
|
||||||
|
await runCommand(['docker', 'rmi', '-f', imageTag], { check: false });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import { assertEquals, assertRejects } from 'jsr:@std/assert';
|
||||||
|
import { describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import {
|
||||||
|
createTestContainer,
|
||||||
|
getTestPorts,
|
||||||
|
loginAndGetIdentity,
|
||||||
|
} from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type { IReq_AdminLogout, IReq_VerifyIdentity } from '../ts_interfaces/requests/admin.ts';
|
||||||
|
import type { IReq_CreateBucket } from '../ts_interfaces/requests/buckets.ts';
|
||||||
|
import type { IReq_AddCredential } from '../ts_interfaces/requests/credentials.ts';
|
||||||
|
import type { IReq_ListAuditEntries } from '../ts_interfaces/requests/audit.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 12;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
const storageDirectory = `.nogit/testdata-${PORT_INDEX}`;
|
||||||
|
|
||||||
|
const cleanupStorageDirectory = async () => {
|
||||||
|
try {
|
||||||
|
await Deno.remove(storageDirectory, { recursive: true });
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Deno.errors.NotFound)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
describe('Enterprise hardening', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
it('refuses default admin credentials on persistent production storage', async () => {
|
||||||
|
const previousAllowInsecureDefaults = Deno.env.get('OBJST_ALLOW_INSECURE_DEFAULTS');
|
||||||
|
const previousAccessKey = Deno.env.get('OBJST_ACCESS_KEY');
|
||||||
|
const previousSecretKey = Deno.env.get('OBJST_SECRET_KEY');
|
||||||
|
const previousAdminPassword = Deno.env.get('OBJST_ADMIN_PASSWORD');
|
||||||
|
const previousStorageDir = Deno.env.get('OBJST_STORAGE_DIR');
|
||||||
|
try {
|
||||||
|
Deno.env.delete('OBJST_ALLOW_INSECURE_DEFAULTS');
|
||||||
|
Deno.env.delete('OBJST_ACCESS_KEY');
|
||||||
|
Deno.env.delete('OBJST_SECRET_KEY');
|
||||||
|
Deno.env.delete('OBJST_ADMIN_PASSWORD');
|
||||||
|
Deno.env.delete('OBJST_STORAGE_DIR');
|
||||||
|
const container = new ObjectStorageContainer();
|
||||||
|
await assertRejects(() => container.start(), Error, 'Refusing to start with default admin credentials');
|
||||||
|
} finally {
|
||||||
|
if (previousAllowInsecureDefaults === undefined) {
|
||||||
|
Deno.env.delete('OBJST_ALLOW_INSECURE_DEFAULTS');
|
||||||
|
} else {
|
||||||
|
Deno.env.set('OBJST_ALLOW_INSECURE_DEFAULTS', previousAllowInsecureDefaults);
|
||||||
|
}
|
||||||
|
if (previousAccessKey === undefined) Deno.env.delete('OBJST_ACCESS_KEY');
|
||||||
|
else Deno.env.set('OBJST_ACCESS_KEY', previousAccessKey);
|
||||||
|
if (previousSecretKey === undefined) Deno.env.delete('OBJST_SECRET_KEY');
|
||||||
|
else Deno.env.set('OBJST_SECRET_KEY', previousSecretKey);
|
||||||
|
if (previousAdminPassword === undefined) Deno.env.delete('OBJST_ADMIN_PASSWORD');
|
||||||
|
else Deno.env.set('OBJST_ADMIN_PASSWORD', previousAdminPassword);
|
||||||
|
if (previousStorageDir === undefined) Deno.env.delete('OBJST_STORAGE_DIR');
|
||||||
|
else Deno.env.set('OBJST_STORAGE_DIR', previousStorageDir);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('exposes health endpoints and writes audit entries for privileged actions', async () => {
|
||||||
|
await cleanupStorageDirectory();
|
||||||
|
let container: ObjectStorageContainer | null = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
container = createTestContainer(PORT_INDEX, { storageDirectory });
|
||||||
|
await container.start();
|
||||||
|
|
||||||
|
const live = await fetch(`http://localhost:${ports.uiPort}/livez`);
|
||||||
|
assertEquals(live.status, 200);
|
||||||
|
assertEquals((await live.json()).status, 'alive');
|
||||||
|
|
||||||
|
const ready = await fetch(`http://localhost:${ports.uiPort}/readyz`);
|
||||||
|
assertEquals(ready.status, 200);
|
||||||
|
assertEquals((await ready.json()).status, 'ready');
|
||||||
|
|
||||||
|
const health = await fetch(`http://localhost:${ports.uiPort}/healthz`);
|
||||||
|
assertEquals(health.status, 200);
|
||||||
|
assertEquals((await health.json()).ok, true);
|
||||||
|
|
||||||
|
const metrics = await fetch(`http://localhost:${ports.uiPort}/metrics`);
|
||||||
|
assertEquals(metrics.status, 200);
|
||||||
|
assertEquals((await metrics.text()).includes('objectstorage_ready 1'), true);
|
||||||
|
|
||||||
|
const identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
|
||||||
|
await createBucket.fire({ identity, bucketName: 'enterprise-audit-bucket' });
|
||||||
|
|
||||||
|
const addCredential = new TypedRequest<IReq_AddCredential>(url, 'addCredential');
|
||||||
|
await addCredential.fire({
|
||||||
|
identity,
|
||||||
|
accessKeyId: 'enterprise-key',
|
||||||
|
secretAccessKey: 'enterprise-secret',
|
||||||
|
});
|
||||||
|
|
||||||
|
const adminConfigInfo = await Deno.stat(
|
||||||
|
`${storageDirectory}/.objectstorage/admin-config.json`,
|
||||||
|
);
|
||||||
|
if (adminConfigInfo.mode !== null) {
|
||||||
|
assertEquals(adminConfigInfo.mode & 0o777, 0o600);
|
||||||
|
}
|
||||||
|
|
||||||
|
const listAuditEntries = new TypedRequest<IReq_ListAuditEntries>(url, 'listAuditEntries');
|
||||||
|
const auditResponse = await listAuditEntries.fire({ identity, limit: 10 });
|
||||||
|
const actions = auditResponse.entries.map((entry) => entry.action);
|
||||||
|
|
||||||
|
assertEquals(actions.includes('admin.login'), true);
|
||||||
|
assertEquals(actions.includes('bucket.create'), true);
|
||||||
|
assertEquals(actions.includes('credential.add'), true);
|
||||||
|
} finally {
|
||||||
|
if (container) {
|
||||||
|
await container.stop();
|
||||||
|
}
|
||||||
|
await cleanupStorageDirectory();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('revokes admin identities on logout', async () => {
|
||||||
|
await cleanupStorageDirectory();
|
||||||
|
let container: ObjectStorageContainer | null = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
container = createTestContainer(PORT_INDEX, { storageDirectory });
|
||||||
|
await container.start();
|
||||||
|
|
||||||
|
const identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
const logout = new TypedRequest<IReq_AdminLogout>(url, 'adminLogout');
|
||||||
|
await logout.fire({ identity });
|
||||||
|
|
||||||
|
const verifyIdentity = new TypedRequest<IReq_VerifyIdentity>(url, 'verifyIdentity');
|
||||||
|
const verification = await verifyIdentity.fire({ identity });
|
||||||
|
assertEquals(verification.valid, false);
|
||||||
|
|
||||||
|
const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
|
||||||
|
await assertRejects(() =>
|
||||||
|
createBucket.fire({ identity, bucketName: 'revoked-token-bucket' })
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
if (container) {
|
||||||
|
await container.stop();
|
||||||
|
}
|
||||||
|
await cleanupStorageDirectory();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,215 @@
|
|||||||
|
import { assertEquals, assertExists } from 'jsr:@std/assert';
|
||||||
|
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type * as interfaces from '../ts_interfaces/index.ts';
|
||||||
|
import type { IReq_CreateBucket, IReq_DeleteBucket } from '../ts_interfaces/requests/buckets.ts';
|
||||||
|
import type {
|
||||||
|
IReq_ListObjects,
|
||||||
|
IReq_PutObject,
|
||||||
|
IReq_GetObject,
|
||||||
|
IReq_DeleteObject,
|
||||||
|
IReq_DeletePrefix,
|
||||||
|
IReq_GetObjectUrl,
|
||||||
|
IReq_MoveObject,
|
||||||
|
IReq_MovePrefix,
|
||||||
|
} from '../ts_interfaces/requests/objects.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 3;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
const BUCKET = 'obj-test-bucket';
|
||||||
|
|
||||||
|
function toBase64(text: string): string {
|
||||||
|
return btoa(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
function fromBase64(b64: string): string {
|
||||||
|
return atob(b64);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('Object operations', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
let container: ObjectStorageContainer;
|
||||||
|
let identity: interfaces.data.IIdentity;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
container = createTestContainer(PORT_INDEX);
|
||||||
|
await container.start();
|
||||||
|
identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
|
||||||
|
// Create test bucket
|
||||||
|
const req = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
|
||||||
|
await req.fire({ identity, bucketName: BUCKET });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
// Delete test bucket
|
||||||
|
try {
|
||||||
|
const req = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
|
||||||
|
await req.fire({ identity, bucketName: BUCKET });
|
||||||
|
} catch { /* bucket may already be empty/deleted */ }
|
||||||
|
await container.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list objects (initially empty)', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET });
|
||||||
|
assertEquals(response.result.objects.length, 0);
|
||||||
|
assertEquals(response.result.commonPrefixes.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should put a text object', async () => {
|
||||||
|
const req = new TypedRequest<IReq_PutObject>(url, 'putObject');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
key: 'hello.txt',
|
||||||
|
base64Content: toBase64('Hello World'),
|
||||||
|
contentType: 'text/plain',
|
||||||
|
});
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should put nested objects', async () => {
|
||||||
|
const req = new TypedRequest<IReq_PutObject>(url, 'putObject');
|
||||||
|
await req.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
key: 'folder/nested.txt',
|
||||||
|
base64Content: toBase64('Nested content'),
|
||||||
|
contentType: 'text/plain',
|
||||||
|
});
|
||||||
|
await req.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
key: 'folder/a.txt',
|
||||||
|
base64Content: toBase64('File A'),
|
||||||
|
contentType: 'text/plain',
|
||||||
|
});
|
||||||
|
await req.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
key: 'folder/b.txt',
|
||||||
|
base64Content: toBase64('File B'),
|
||||||
|
contentType: 'text/plain',
|
||||||
|
});
|
||||||
|
await req.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
key: 'folder/sub/c.txt',
|
||||||
|
base64Content: toBase64('File C'),
|
||||||
|
contentType: 'text/plain',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list objects at root with delimiter', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, delimiter: '/' });
|
||||||
|
// Root should have hello.txt as direct object
|
||||||
|
const rootKeys = response.result.objects.map((o) => o.key);
|
||||||
|
assertEquals(rootKeys.includes('hello.txt'), true);
|
||||||
|
// folder/ should appear as a common prefix
|
||||||
|
assertEquals(response.result.commonPrefixes.includes('folder/'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list objects with prefix', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
prefix: 'folder/',
|
||||||
|
delimiter: '/',
|
||||||
|
});
|
||||||
|
const keys = response.result.objects.map((o) => o.key);
|
||||||
|
assertEquals(keys.includes('folder/nested.txt'), true);
|
||||||
|
assertEquals(keys.includes('folder/a.txt'), true);
|
||||||
|
assertEquals(keys.includes('folder/b.txt'), true);
|
||||||
|
// sub/ should be a common prefix
|
||||||
|
assertEquals(response.result.commonPrefixes.includes('folder/sub/'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get a text object', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetObject>(url, 'getObject');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, key: 'hello.txt' });
|
||||||
|
assertEquals(fromBase64(response.content), 'Hello World');
|
||||||
|
assertEquals(response.size > 0, true);
|
||||||
|
assertExists(response.lastModified);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get object URL', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetObjectUrl>(url, 'getObjectUrl');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, key: 'hello.txt' });
|
||||||
|
assertExists(response.url);
|
||||||
|
assertEquals(response.url.includes(BUCKET), true);
|
||||||
|
assertEquals(response.url.includes('hello.txt'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should move an object', async () => {
|
||||||
|
const req = new TypedRequest<IReq_MoveObject>(url, 'moveObject');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
sourceKey: 'hello.txt',
|
||||||
|
destKey: 'moved-hello.txt',
|
||||||
|
});
|
||||||
|
assertEquals(response.success, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify moved object exists at new key', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetObject>(url, 'getObject');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, key: 'moved-hello.txt' });
|
||||||
|
assertEquals(fromBase64(response.content), 'Hello World');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify source key no longer exists after move', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, delimiter: '/' });
|
||||||
|
const rootKeys = response.result.objects.map((o) => o.key);
|
||||||
|
assertEquals(rootKeys.includes('hello.txt'), false);
|
||||||
|
assertEquals(rootKeys.includes('moved-hello.txt'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should move a prefix', async () => {
|
||||||
|
const req = new TypedRequest<IReq_MovePrefix>(url, 'movePrefix');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
sourcePrefix: 'folder/',
|
||||||
|
destPrefix: 'renamed/',
|
||||||
|
});
|
||||||
|
assertEquals(response.success, true);
|
||||||
|
assertEquals((response.movedCount ?? 0) >= 4, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify moved prefix contents', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, prefix: 'renamed/' });
|
||||||
|
assertEquals(response.result.objects.length >= 1, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify old prefix is empty', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, prefix: 'folder/' });
|
||||||
|
assertEquals(response.result.objects.length, 0);
|
||||||
|
assertEquals(response.result.commonPrefixes.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should delete a single object', async () => {
|
||||||
|
const req = new TypedRequest<IReq_DeleteObject>(url, 'deleteObject');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, key: 'moved-hello.txt' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should delete a prefix', async () => {
|
||||||
|
const req = new TypedRequest<IReq_DeletePrefix>(url, 'deletePrefix');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET, prefix: 'renamed/' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify bucket is empty after cleanup', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
|
||||||
|
const response = await req.fire({ identity, bucketName: BUCKET });
|
||||||
|
assertEquals(response.result.objects.length, 0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
import { assertEquals, assertExists } from 'jsr:@std/assert';
|
||||||
|
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type * as interfaces from '../ts_interfaces/index.ts';
|
||||||
|
import type { IReq_CreateBucket, IReq_DeleteBucket, IReq_GetBucketPolicy } from '../ts_interfaces/requests/buckets.ts';
|
||||||
|
import type {
|
||||||
|
IReq_ListNamedPolicies,
|
||||||
|
IReq_CreateNamedPolicy,
|
||||||
|
IReq_UpdateNamedPolicy,
|
||||||
|
IReq_DeleteNamedPolicy,
|
||||||
|
IReq_GetBucketNamedPolicies,
|
||||||
|
IReq_AttachPolicyToBucket,
|
||||||
|
IReq_DetachPolicyFromBucket,
|
||||||
|
IReq_GetPolicyBuckets,
|
||||||
|
IReq_SetPolicyBuckets,
|
||||||
|
} from '../ts_interfaces/requests/policies.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 5;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
|
||||||
|
describe('Named policy management', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
let container: ObjectStorageContainer;
|
||||||
|
let identity: interfaces.data.IIdentity;
|
||||||
|
let policy1Id: string;
|
||||||
|
let policy2Id: string;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
container = createTestContainer(PORT_INDEX);
|
||||||
|
await container.start();
|
||||||
|
identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
|
||||||
|
// Create test buckets
|
||||||
|
const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
|
||||||
|
await createBucket.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
await createBucket.fire({ identity, bucketName: 'pol-bucket-2' });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
// Cleanup
|
||||||
|
try {
|
||||||
|
const del = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
|
||||||
|
await del.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
} catch { /* may already be deleted */ }
|
||||||
|
try {
|
||||||
|
const del = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
|
||||||
|
await del.fire({ identity, bucketName: 'pol-bucket-2' });
|
||||||
|
} catch { /* may already be deleted */ }
|
||||||
|
await container.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list policies (initially empty)', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListNamedPolicies>(url, 'listNamedPolicies');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.policies.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should create a named policy', async () => {
|
||||||
|
const req = new TypedRequest<IReq_CreateNamedPolicy>(url, 'createNamedPolicy');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
name: 'Public Read',
|
||||||
|
description: 'Allows public read access',
|
||||||
|
statements: [
|
||||||
|
{
|
||||||
|
Sid: 'PublicRead',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: 'arn:aws:s3:::${bucket}/*',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
assertExists(response.policy.id);
|
||||||
|
assertEquals(response.policy.name, 'Public Read');
|
||||||
|
assertEquals(response.policy.statements.length, 1);
|
||||||
|
assertEquals(response.policy.createdAt > 0, true);
|
||||||
|
policy1Id = response.policy.id;
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should create a second policy', async () => {
|
||||||
|
const req = new TypedRequest<IReq_CreateNamedPolicy>(url, 'createNamedPolicy');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
name: 'Deny Delete',
|
||||||
|
description: 'Denies delete actions',
|
||||||
|
statements: [
|
||||||
|
{
|
||||||
|
Sid: 'DenyDelete',
|
||||||
|
Effect: 'Deny',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:DeleteObject',
|
||||||
|
Resource: 'arn:aws:s3:::${bucket}/*',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
policy2Id = response.policy.id;
|
||||||
|
assertExists(policy2Id);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list both policies', async () => {
|
||||||
|
const req = new TypedRequest<IReq_ListNamedPolicies>(url, 'listNamedPolicies');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.policies.length, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get bucket named policies (none attached)', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketNamedPolicies>(url, 'getBucketNamedPolicies');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
assertEquals(response.attachedPolicies.length, 0);
|
||||||
|
assertEquals(response.availablePolicies.length, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should attach policy to bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_AttachPolicyToBucket>(url, 'attachPolicyToBucket');
|
||||||
|
const response = await req.fire({ identity, policyId: policy1Id, bucketName: 'pol-bucket-1' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify bucket policy was applied with placeholder replaced', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
assertExists(response.policy);
|
||||||
|
const parsed = JSON.parse(response.policy!);
|
||||||
|
const resource = parsed.Statement[0].Resource;
|
||||||
|
// Resource may be a string or array depending on the S3 engine
|
||||||
|
const resourceStr = Array.isArray(resource) ? resource.join(' ') : resource;
|
||||||
|
// ${bucket} should be replaced with actual bucket name
|
||||||
|
assertEquals(resourceStr.includes('pol-bucket-1'), true);
|
||||||
|
assertEquals(resourceStr.includes('${bucket}'), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get bucket named policies (one attached)', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketNamedPolicies>(url, 'getBucketNamedPolicies');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
assertEquals(response.attachedPolicies.length, 1);
|
||||||
|
assertEquals(response.availablePolicies.length, 1);
|
||||||
|
assertEquals(response.attachedPolicies[0].id, policy1Id);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should attach second policy to same bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_AttachPolicyToBucket>(url, 'attachPolicyToBucket');
|
||||||
|
const response = await req.fire({ identity, policyId: policy2Id, bucketName: 'pol-bucket-1' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify merged policy has statements from both', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
assertExists(response.policy);
|
||||||
|
const parsed = JSON.parse(response.policy!);
|
||||||
|
assertEquals(parsed.Statement.length >= 2, true);
|
||||||
|
const sids = parsed.Statement.map((s: any) => s.Sid);
|
||||||
|
assertEquals(sids.includes('PublicRead'), true);
|
||||||
|
assertEquals(sids.includes('DenyDelete'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get policy buckets', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetPolicyBuckets>(url, 'getPolicyBuckets');
|
||||||
|
const response = await req.fire({ identity, policyId: policy1Id });
|
||||||
|
assertEquals(response.attachedBuckets.includes('pol-bucket-1'), true);
|
||||||
|
assertEquals(response.availableBuckets.includes('pol-bucket-2'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should set policy buckets (batch)', async () => {
|
||||||
|
const req = new TypedRequest<IReq_SetPolicyBuckets>(url, 'setPolicyBuckets');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
policyId: policy1Id,
|
||||||
|
bucketNames: ['pol-bucket-1', 'pol-bucket-2'],
|
||||||
|
});
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify policy applied to second bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'pol-bucket-2' });
|
||||||
|
assertExists(response.policy);
|
||||||
|
const parsed = JSON.parse(response.policy!);
|
||||||
|
const resource = parsed.Statement[0].Resource;
|
||||||
|
const resourceStr = Array.isArray(resource) ? resource.join(' ') : resource;
|
||||||
|
assertEquals(resourceStr.includes('pol-bucket-2'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should update a policy', async () => {
|
||||||
|
const req = new TypedRequest<IReq_UpdateNamedPolicy>(url, 'updateNamedPolicy');
|
||||||
|
const response = await req.fire({
|
||||||
|
identity,
|
||||||
|
policyId: policy1Id,
|
||||||
|
name: 'Public Read Updated',
|
||||||
|
description: 'Updated policy',
|
||||||
|
statements: [
|
||||||
|
{
|
||||||
|
Sid: 'PublicReadUpdated',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:GetObject', 's3:ListBucket'],
|
||||||
|
Resource: 'arn:aws:s3:::${bucket}/*',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
assertEquals(response.policy.name, 'Public Read Updated');
|
||||||
|
assertEquals(response.policy.updatedAt >= response.policy.createdAt, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify updated policy cascaded to attached buckets', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
assertExists(response.policy);
|
||||||
|
const parsed = JSON.parse(response.policy!);
|
||||||
|
const sids = parsed.Statement.map((s: any) => s.Sid);
|
||||||
|
assertEquals(sids.includes('PublicReadUpdated'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should detach policy from bucket', async () => {
|
||||||
|
const req = new TypedRequest<IReq_DetachPolicyFromBucket>(url, 'detachPolicyFromBucket');
|
||||||
|
const response = await req.fire({ identity, policyId: policy2Id, bucketName: 'pol-bucket-1' });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should verify bucket policy updated after detach', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
|
||||||
|
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
assertExists(response.policy);
|
||||||
|
const parsed = JSON.parse(response.policy!);
|
||||||
|
const sids = parsed.Statement.map((s: any) => s.Sid);
|
||||||
|
assertEquals(sids.includes('DenyDelete'), false);
|
||||||
|
assertEquals(sids.includes('PublicReadUpdated'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should delete a named policy', async () => {
|
||||||
|
const req = new TypedRequest<IReq_DeleteNamedPolicy>(url, 'deleteNamedPolicy');
|
||||||
|
const response = await req.fire({ identity, policyId: policy2Id });
|
||||||
|
assertEquals(response.ok, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle bucket deletion cleaning up attachments', async () => {
|
||||||
|
// Delete pol-bucket-1 which has policy1 attached
|
||||||
|
const delBucket = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
|
||||||
|
await delBucket.fire({ identity, bucketName: 'pol-bucket-1' });
|
||||||
|
|
||||||
|
// Verify policy1 no longer lists pol-bucket-1
|
||||||
|
const req = new TypedRequest<IReq_GetPolicyBuckets>(url, 'getPolicyBuckets');
|
||||||
|
const response = await req.fire({ identity, policyId: policy1Id });
|
||||||
|
assertEquals(response.attachedBuckets.includes('pol-bucket-1'), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('cleanup: delete remaining policy and bucket', async () => {
|
||||||
|
const delPolicy = new TypedRequest<IReq_DeleteNamedPolicy>(url, 'deleteNamedPolicy');
|
||||||
|
await delPolicy.fire({ identity, policyId: policy1Id });
|
||||||
|
|
||||||
|
const listPolicies = new TypedRequest<IReq_ListNamedPolicies>(url, 'listNamedPolicies');
|
||||||
|
const response = await listPolicies.fire({ identity });
|
||||||
|
assertEquals(response.policies.length, 0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
import { assertEquals, assertExists } from 'jsr:@std/assert';
|
||||||
|
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import {
|
||||||
|
S3Client,
|
||||||
|
ListBucketsCommand,
|
||||||
|
CreateBucketCommand,
|
||||||
|
DeleteBucketCommand,
|
||||||
|
PutObjectCommand,
|
||||||
|
GetObjectCommand,
|
||||||
|
ListObjectsV2Command,
|
||||||
|
CopyObjectCommand,
|
||||||
|
DeleteObjectCommand,
|
||||||
|
} from '@aws-sdk/client-s3';
|
||||||
|
import { createTestContainer, getTestPorts, TEST_ACCESS_KEY, TEST_SECRET_KEY } from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 4;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const BUCKET = 's3-test-bucket';
|
||||||
|
|
||||||
|
describe('S3 SDK compatibility', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
let container: ObjectStorageContainer;
|
||||||
|
let s3: S3Client;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
container = createTestContainer(PORT_INDEX);
|
||||||
|
await container.start();
|
||||||
|
s3 = new S3Client({
|
||||||
|
endpoint: `http://localhost:${ports.objstPort}`,
|
||||||
|
region: 'us-east-1',
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: TEST_ACCESS_KEY,
|
||||||
|
secretAccessKey: TEST_SECRET_KEY,
|
||||||
|
},
|
||||||
|
forcePathStyle: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
s3.destroy();
|
||||||
|
await container.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list buckets (empty)', async () => {
|
||||||
|
const response = await s3.send(new ListBucketsCommand({}));
|
||||||
|
assertEquals((response.Buckets || []).length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should create bucket via S3', async () => {
|
||||||
|
await s3.send(new CreateBucketCommand({ Bucket: BUCKET }));
|
||||||
|
const response = await s3.send(new ListBucketsCommand({}));
|
||||||
|
const names = (response.Buckets || []).map((b) => b.Name);
|
||||||
|
assertEquals(names.includes(BUCKET), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should put object via S3', async () => {
|
||||||
|
await s3.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Key: 'test.txt',
|
||||||
|
Body: new TextEncoder().encode('Hello from S3 SDK'),
|
||||||
|
ContentType: 'text/plain',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get object via S3', async () => {
|
||||||
|
const response = await s3.send(
|
||||||
|
new GetObjectCommand({ Bucket: BUCKET, Key: 'test.txt' }),
|
||||||
|
);
|
||||||
|
const body = await response.Body!.transformToString();
|
||||||
|
assertEquals(body, 'Hello from S3 SDK');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should list objects via S3', async () => {
|
||||||
|
const response = await s3.send(
|
||||||
|
new ListObjectsV2Command({ Bucket: BUCKET }),
|
||||||
|
);
|
||||||
|
const keys = (response.Contents || []).map((o) => o.Key);
|
||||||
|
assertEquals(keys.includes('test.txt'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should copy object via S3', async () => {
|
||||||
|
await s3.send(
|
||||||
|
new CopyObjectCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
CopySource: `${BUCKET}/test.txt`,
|
||||||
|
Key: 'copy.txt',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const response = await s3.send(
|
||||||
|
new GetObjectCommand({ Bucket: BUCKET, Key: 'copy.txt' }),
|
||||||
|
);
|
||||||
|
const body = await response.Body!.transformToString();
|
||||||
|
assertEquals(body, 'Hello from S3 SDK');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should delete object via S3', async () => {
|
||||||
|
await s3.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'test.txt' }));
|
||||||
|
await s3.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'copy.txt' }));
|
||||||
|
const response = await s3.send(
|
||||||
|
new ListObjectsV2Command({ Bucket: BUCKET }),
|
||||||
|
);
|
||||||
|
assertEquals((response.Contents || []).length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reject requests with wrong credentials', async () => {
|
||||||
|
const badS3 = new S3Client({
|
||||||
|
endpoint: `http://localhost:${ports.objstPort}`,
|
||||||
|
region: 'us-east-1',
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: 'wrongkey',
|
||||||
|
secretAccessKey: 'wrongsecret',
|
||||||
|
},
|
||||||
|
forcePathStyle: true,
|
||||||
|
});
|
||||||
|
let threw = false;
|
||||||
|
try {
|
||||||
|
await badS3.send(new ListBucketsCommand({}));
|
||||||
|
} catch {
|
||||||
|
threw = true;
|
||||||
|
}
|
||||||
|
badS3.destroy();
|
||||||
|
assertEquals(threw, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('cleanup: delete bucket', async () => {
|
||||||
|
await s3.send(new DeleteBucketCommand({ Bucket: BUCKET }));
|
||||||
|
const response = await s3.send(new ListBucketsCommand({}));
|
||||||
|
assertEquals((response.Buckets || []).length, 0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
import { assertEquals, assertExists } from 'jsr:@std/assert';
|
||||||
|
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
|
||||||
|
import { TypedRequest } from '@api.global/typedrequest';
|
||||||
|
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
|
||||||
|
import { ObjectStorageContainer } from '../ts/index.ts';
|
||||||
|
import type * as interfaces from '../ts_interfaces/index.ts';
|
||||||
|
import type { IReq_CreateBucket, IReq_DeleteBucket } from '../ts_interfaces/requests/buckets.ts';
|
||||||
|
import type { IReq_DeletePrefix, IReq_PutObject } from '../ts_interfaces/requests/objects.ts';
|
||||||
|
import type {
|
||||||
|
IReq_GetClusterHealth,
|
||||||
|
IReq_GetServerStatus,
|
||||||
|
} from '../ts_interfaces/requests/status.ts';
|
||||||
|
import type { IReq_GetServerConfig } from '../ts_interfaces/requests/config.ts';
|
||||||
|
|
||||||
|
const PORT_INDEX = 7;
|
||||||
|
const ports = getTestPorts(PORT_INDEX);
|
||||||
|
const url = `http://localhost:${ports.uiPort}/typedrequest`;
|
||||||
|
const BUCKET = 'stats-bucket';
|
||||||
|
|
||||||
|
describe('Status and config', { sanitizeResources: false, sanitizeOps: false }, () => {
|
||||||
|
let container: ObjectStorageContainer;
|
||||||
|
let identity: interfaces.data.IIdentity;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
container = createTestContainer(PORT_INDEX);
|
||||||
|
await container.start();
|
||||||
|
identity = await loginAndGetIdentity(ports.uiPort);
|
||||||
|
|
||||||
|
// Create bucket with objects for stats testing
|
||||||
|
const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
|
||||||
|
await createBucket.fire({ identity, bucketName: BUCKET });
|
||||||
|
|
||||||
|
const putObj = new TypedRequest<IReq_PutObject>(url, 'putObject');
|
||||||
|
await putObj.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
key: 'file1.txt',
|
||||||
|
base64Content: btoa('Content of file 1'),
|
||||||
|
contentType: 'text/plain',
|
||||||
|
});
|
||||||
|
await putObj.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
key: 'file2.txt',
|
||||||
|
base64Content: btoa('Content of file 2'),
|
||||||
|
contentType: 'text/plain',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
try {
|
||||||
|
const delPrefix = new TypedRequest<IReq_DeletePrefix>(url, 'deletePrefix');
|
||||||
|
await delPrefix.fire({ identity, bucketName: BUCKET, prefix: '' });
|
||||||
|
const delBucket = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
|
||||||
|
await delBucket.fire({ identity, bucketName: BUCKET });
|
||||||
|
} catch { /* cleanup best-effort */ }
|
||||||
|
await container.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get server status', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetServerStatus>(url, 'getServerStatus');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
const status = response.status;
|
||||||
|
|
||||||
|
assertEquals(status.running, true);
|
||||||
|
assertEquals(status.objstPort, ports.objstPort);
|
||||||
|
assertEquals(status.uiPort, ports.uiPort);
|
||||||
|
assertEquals(status.bucketCount, 1);
|
||||||
|
assertEquals(status.totalObjectCount, 2);
|
||||||
|
assertEquals(status.totalStorageBytes > 0, true);
|
||||||
|
assertEquals(status.uptime >= 0, true);
|
||||||
|
assertEquals(status.startedAt > 0, true);
|
||||||
|
assertEquals(status.region, 'us-east-1');
|
||||||
|
assertEquals(status.authEnabled, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get connection info from status', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetServerStatus>(url, 'getServerStatus');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
const info = response.connectionInfo;
|
||||||
|
|
||||||
|
assertExists(info.endpoint);
|
||||||
|
assertEquals(info.port, ports.objstPort);
|
||||||
|
assertExists(info.accessKey);
|
||||||
|
assertEquals(info.region, 'us-east-1');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should get server config', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetServerConfig>(url, 'getServerConfig');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
const config = response.config;
|
||||||
|
|
||||||
|
assertEquals(config.objstPort, ports.objstPort);
|
||||||
|
assertEquals(config.uiPort, ports.uiPort);
|
||||||
|
assertEquals(config.region, 'us-east-1');
|
||||||
|
assertEquals(config.authEnabled, true);
|
||||||
|
assertEquals(config.corsEnabled, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should report standalone cluster health', async () => {
|
||||||
|
const req = new TypedRequest<IReq_GetClusterHealth>(url, 'getClusterHealth');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
|
||||||
|
assertEquals(response.clusterHealth.enabled, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should reflect correct stats after adding objects', async () => {
|
||||||
|
// Add a third object
|
||||||
|
const putObj = new TypedRequest<IReq_PutObject>(url, 'putObject');
|
||||||
|
await putObj.fire({
|
||||||
|
identity,
|
||||||
|
bucketName: BUCKET,
|
||||||
|
key: 'file3.txt',
|
||||||
|
base64Content: btoa('Content of file 3'),
|
||||||
|
contentType: 'text/plain',
|
||||||
|
});
|
||||||
|
|
||||||
|
const req = new TypedRequest<IReq_GetServerStatus>(url, 'getServerStatus');
|
||||||
|
const response = await req.fire({ identity });
|
||||||
|
assertEquals(response.status.totalObjectCount, 3);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@lossless.zone/objectstorage',
|
name: '@lossless.zone/objectstorage',
|
||||||
version: '1.4.2',
|
version: '1.9.0',
|
||||||
description: 'object storage server with management UI powered by smartstorage'
|
description: 'object storage server with management UI powered by smartstorage'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
import type * as interfaces from '../../ts_interfaces/index.ts';
|
||||||
|
|
||||||
|
export interface IAuditLogEntry {
|
||||||
|
timestamp: number;
|
||||||
|
actorUserId: string;
|
||||||
|
action: string;
|
||||||
|
targetType: string;
|
||||||
|
targetId?: string;
|
||||||
|
success: boolean;
|
||||||
|
message?: string;
|
||||||
|
metadata?: Record<string, string | number | boolean>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AuditLogger {
|
||||||
|
constructor(private storageDirectory: string) {}
|
||||||
|
|
||||||
|
public get auditLogPath(): string {
|
||||||
|
return `${this.storageDirectory}/.objectstorage/audit.log`;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async log(entry: Omit<IAuditLogEntry, 'timestamp'>): Promise<void> {
|
||||||
|
const logEntry: IAuditLogEntry = {
|
||||||
|
timestamp: Date.now(),
|
||||||
|
...entry,
|
||||||
|
};
|
||||||
|
const dirPath = this.auditLogPath.substring(0, this.auditLogPath.lastIndexOf('/'));
|
||||||
|
await Deno.mkdir(dirPath, { recursive: true });
|
||||||
|
await Deno.writeTextFile(this.auditLogPath, `${JSON.stringify(logEntry)}\n`, {
|
||||||
|
append: true,
|
||||||
|
create: true,
|
||||||
|
mode: 0o600,
|
||||||
|
});
|
||||||
|
await this.restrictAuditLogPermissions();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async listRecent(limit = 100): Promise<interfaces.data.IAuditEntry[]> {
|
||||||
|
let content = '';
|
||||||
|
try {
|
||||||
|
content = await Deno.readTextFile(this.auditLogPath);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof Deno.errors.NotFound) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return content
|
||||||
|
.trim()
|
||||||
|
.split('\n')
|
||||||
|
.filter(Boolean)
|
||||||
|
.slice(-limit)
|
||||||
|
.map((line) => JSON.parse(line) as interfaces.data.IAuditEntry)
|
||||||
|
.reverse();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async restrictAuditLogPermissions(): Promise<void> {
|
||||||
|
try {
|
||||||
|
await Deno.chmod(this.auditLogPath, 0o600);
|
||||||
|
} catch {
|
||||||
|
// chmod is not available on every platform Deno supports.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,13 @@
|
|||||||
import * as plugins from '../plugins.ts';
|
import * as plugins from '../plugins.ts';
|
||||||
import { type IObjectStorageConfig, defaultConfig } from '../types.ts';
|
import { defaultConfig, type IObjectStorageConfig } from '../types.ts';
|
||||||
import type * as interfaces from '../../ts_interfaces/index.ts';
|
import type * as interfaces from '../../ts_interfaces/index.ts';
|
||||||
import { OpsServer } from '../opsserver/index.ts';
|
import { OpsServer } from '../opsserver/index.ts';
|
||||||
import { PolicyManager } from './policymanager.ts';
|
import { PolicyManager } from './policymanager.ts';
|
||||||
|
import { AuditLogger } from './auditlogger.ts';
|
||||||
|
|
||||||
|
interface IPersistedAdminConfig {
|
||||||
|
accessCredentials?: Array<{ accessKeyId: string; secretAccessKey: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
export class ObjectStorageContainer {
|
export class ObjectStorageContainer {
|
||||||
public config: IObjectStorageConfig;
|
public config: IObjectStorageConfig;
|
||||||
@@ -10,7 +15,9 @@ export class ObjectStorageContainer {
|
|||||||
public s3Client!: plugins.S3Client;
|
public s3Client!: plugins.S3Client;
|
||||||
public opsServer: OpsServer;
|
public opsServer: OpsServer;
|
||||||
public policyManager: PolicyManager;
|
public policyManager: PolicyManager;
|
||||||
|
public auditLogger: AuditLogger;
|
||||||
public startedAt: number = 0;
|
public startedAt: number = 0;
|
||||||
|
private envAccessCredentialsProvided = false;
|
||||||
|
|
||||||
constructor(configArg?: Partial<IObjectStorageConfig>) {
|
constructor(configArg?: Partial<IObjectStorageConfig>) {
|
||||||
this.config = { ...defaultConfig, ...configArg };
|
this.config = { ...defaultConfig, ...configArg };
|
||||||
@@ -28,6 +35,7 @@ export class ObjectStorageContainer {
|
|||||||
const envAccessKey = Deno.env.get('OBJST_ACCESS_KEY');
|
const envAccessKey = Deno.env.get('OBJST_ACCESS_KEY');
|
||||||
const envSecretKey = Deno.env.get('OBJST_SECRET_KEY');
|
const envSecretKey = Deno.env.get('OBJST_SECRET_KEY');
|
||||||
if (envAccessKey && envSecretKey) {
|
if (envAccessKey && envSecretKey) {
|
||||||
|
this.envAccessCredentialsProvided = true;
|
||||||
this.config.accessCredentials = [
|
this.config.accessCredentials = [
|
||||||
{ accessKeyId: envAccessKey, secretAccessKey: envSecretKey },
|
{ accessKeyId: envAccessKey, secretAccessKey: envSecretKey },
|
||||||
];
|
];
|
||||||
@@ -39,47 +47,88 @@ export class ObjectStorageContainer {
|
|||||||
const envRegion = Deno.env.get('OBJST_REGION');
|
const envRegion = Deno.env.get('OBJST_REGION');
|
||||||
if (envRegion) this.config.region = envRegion;
|
if (envRegion) this.config.region = envRegion;
|
||||||
|
|
||||||
|
// Cluster environment variables
|
||||||
|
const envClusterEnabled = Deno.env.get('OBJST_CLUSTER_ENABLED');
|
||||||
|
if (envClusterEnabled) {
|
||||||
|
this.config.clusterEnabled = envClusterEnabled === 'true' || envClusterEnabled === '1';
|
||||||
|
}
|
||||||
|
|
||||||
|
const envClusterNodeId = Deno.env.get('OBJST_CLUSTER_NODE_ID');
|
||||||
|
if (envClusterNodeId) this.config.clusterNodeId = envClusterNodeId;
|
||||||
|
|
||||||
|
const envClusterQuicPort = Deno.env.get('OBJST_CLUSTER_QUIC_PORT');
|
||||||
|
if (envClusterQuicPort) this.config.clusterQuicPort = parseInt(envClusterQuicPort, 10);
|
||||||
|
|
||||||
|
const envClusterSeedNodes = Deno.env.get('OBJST_CLUSTER_SEED_NODES');
|
||||||
|
if (envClusterSeedNodes) {
|
||||||
|
this.config.clusterSeedNodes = envClusterSeedNodes.split(',').map((s) => s.trim()).filter(
|
||||||
|
Boolean,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const envDrivePaths = Deno.env.get('OBJST_DRIVE_PATHS');
|
||||||
|
if (envDrivePaths) {
|
||||||
|
this.config.drivePaths = envDrivePaths.split(',').map((s) => s.trim()).filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
const envErasureDataShards = Deno.env.get('OBJST_ERASURE_DATA_SHARDS');
|
||||||
|
if (envErasureDataShards) this.config.erasureDataShards = parseInt(envErasureDataShards, 10);
|
||||||
|
|
||||||
|
const envErasureParityShards = Deno.env.get('OBJST_ERASURE_PARITY_SHARDS');
|
||||||
|
if (envErasureParityShards) {
|
||||||
|
this.config.erasureParityShards = parseInt(envErasureParityShards, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
const envErasureChunkSize = Deno.env.get('OBJST_ERASURE_CHUNK_SIZE');
|
||||||
|
if (envErasureChunkSize) this.config.erasureChunkSizeBytes = parseInt(envErasureChunkSize, 10);
|
||||||
|
|
||||||
|
const envHeartbeatInterval = Deno.env.get('OBJST_HEARTBEAT_INTERVAL_MS');
|
||||||
|
if (envHeartbeatInterval) {
|
||||||
|
this.config.clusterHeartbeatIntervalMs = parseInt(envHeartbeatInterval, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
const envHeartbeatTimeout = Deno.env.get('OBJST_HEARTBEAT_TIMEOUT_MS');
|
||||||
|
if (envHeartbeatTimeout) {
|
||||||
|
this.config.clusterHeartbeatTimeoutMs = parseInt(envHeartbeatTimeout, 10);
|
||||||
|
}
|
||||||
|
|
||||||
this.opsServer = new OpsServer(this);
|
this.opsServer = new OpsServer(this);
|
||||||
this.policyManager = new PolicyManager(this);
|
this.policyManager = new PolicyManager(this);
|
||||||
|
this.auditLogger = new AuditLogger(this.config.storageDirectory);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async start(): Promise<void> {
|
public async start(): Promise<void> {
|
||||||
|
this.assertSecureStartupConfig();
|
||||||
|
await this.loadPersistedAdminConfig();
|
||||||
|
|
||||||
console.log(`Starting ObjectStorage...`);
|
console.log(`Starting ObjectStorage...`);
|
||||||
console.log(` Storage port: ${this.config.objstPort}`);
|
console.log(` Storage port: ${this.config.objstPort}`);
|
||||||
console.log(` UI port: ${this.config.uiPort}`);
|
console.log(` UI port: ${this.config.uiPort}`);
|
||||||
console.log(` Storage: ${this.config.storageDirectory}`);
|
console.log(` Storage: ${this.config.storageDirectory}`);
|
||||||
console.log(` Region: ${this.config.region}`);
|
console.log(` Region: ${this.config.region}`);
|
||||||
|
console.log(` Cluster: ${this.config.clusterEnabled ? 'enabled' : 'disabled'}`);
|
||||||
|
if (this.config.clusterEnabled) {
|
||||||
|
console.log(` Node ID: ${this.config.clusterNodeId || '(auto-generated)'}`);
|
||||||
|
console.log(` QUIC Port: ${this.config.clusterQuicPort}`);
|
||||||
|
console.log(` Seed Nodes: ${this.config.clusterSeedNodes.join(', ') || '(none)'}`);
|
||||||
|
console.log(
|
||||||
|
` Drives: ${
|
||||||
|
this.config.drivePaths.length > 0
|
||||||
|
? this.config.drivePaths.join(', ')
|
||||||
|
: this.config.storageDirectory
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
console.log(` Erasure: ${this.config.erasureDataShards}+${this.config.erasureParityShards}`);
|
||||||
|
}
|
||||||
|
|
||||||
// Start smartstorage
|
this.smartstorageInstance = await plugins.smartstorage.SmartStorage.createAndStart(
|
||||||
this.smartstorageInstance = await plugins.smartstorage.SmartStorage.createAndStart({
|
this.buildSmartstorageConfig(),
|
||||||
server: {
|
);
|
||||||
port: this.config.objstPort,
|
|
||||||
address: '0.0.0.0',
|
|
||||||
region: this.config.region,
|
|
||||||
},
|
|
||||||
storage: {
|
|
||||||
directory: this.config.storageDirectory,
|
|
||||||
},
|
|
||||||
auth: {
|
|
||||||
enabled: true,
|
|
||||||
credentials: this.config.accessCredentials,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
this.startedAt = Date.now();
|
this.startedAt = Date.now();
|
||||||
console.log(`Storage server started on port ${this.config.objstPort}`);
|
console.log(`Storage server started on port ${this.config.objstPort}`);
|
||||||
|
|
||||||
// Create S3 client for management operations
|
await this.refreshManagementClient();
|
||||||
const descriptor = await this.smartstorageInstance.getStorageDescriptor();
|
|
||||||
this.s3Client = new plugins.S3Client({
|
|
||||||
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
|
|
||||||
region: this.config.region,
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: descriptor.accessKey,
|
|
||||||
secretAccessKey: descriptor.accessSecret,
|
|
||||||
},
|
|
||||||
forcePathStyle: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Load named policies
|
// Load named policies
|
||||||
await this.policyManager.load();
|
await this.policyManager.load();
|
||||||
@@ -96,41 +145,54 @@ export class ObjectStorageContainer {
|
|||||||
console.log('ObjectStorage stopped.');
|
console.log('ObjectStorage stopped.');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async replaceAccessCredentials(
|
||||||
|
credentials: Array<{ accessKeyId: string; secretAccessKey: string }>,
|
||||||
|
): Promise<void> {
|
||||||
|
const nextCredentials = credentials.map((credential) => ({ ...credential }));
|
||||||
|
const previousCredentials = this.config.accessCredentials.map((credential) => ({
|
||||||
|
...credential,
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (this.smartstorageInstance) {
|
||||||
|
await this.smartstorageInstance.replaceCredentials(nextCredentials);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.config.accessCredentials = nextCredentials;
|
||||||
|
try {
|
||||||
|
await this.savePersistedAdminConfig();
|
||||||
|
} catch (error) {
|
||||||
|
this.config.accessCredentials = previousCredentials;
|
||||||
|
if (this.smartstorageInstance) {
|
||||||
|
await this.smartstorageInstance.replaceCredentials(previousCredentials);
|
||||||
|
await this.refreshManagementClient();
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.smartstorageInstance) {
|
||||||
|
await this.refreshManagementClient();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async listAccessCredentials(): Promise<Array<{ accessKeyId: string }>> {
|
||||||
|
if (!this.smartstorageInstance) {
|
||||||
|
return this.config.accessCredentials.map((credential) => ({
|
||||||
|
accessKeyId: credential.accessKeyId,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
return await this.smartstorageInstance.listCredentials();
|
||||||
|
}
|
||||||
|
|
||||||
// ── Management methods ──
|
// ── Management methods ──
|
||||||
|
|
||||||
public async listBuckets(): Promise<interfaces.data.IBucketInfo[]> {
|
public async listBuckets(): Promise<interfaces.data.IBucketInfo[]> {
|
||||||
const response = await this.s3Client.send(new plugins.ListBucketsCommand({}));
|
const summaries = await this.smartstorageInstance.listBucketSummaries();
|
||||||
const buckets: interfaces.data.IBucketInfo[] = [];
|
return summaries.map((bucket) => ({
|
||||||
|
name: bucket.name,
|
||||||
for (const bucket of response.Buckets || []) {
|
creationDate: bucket.creationDate || 0,
|
||||||
const name = bucket.Name || '';
|
objectCount: bucket.objectCount,
|
||||||
const creationDate = bucket.CreationDate?.getTime() || 0;
|
totalSizeBytes: bucket.totalSizeBytes,
|
||||||
|
}));
|
||||||
// Get object count and size for each bucket
|
|
||||||
let objectCount = 0;
|
|
||||||
let totalSizeBytes = 0;
|
|
||||||
let continuationToken: string | undefined;
|
|
||||||
|
|
||||||
do {
|
|
||||||
const listResp = await this.s3Client.send(
|
|
||||||
new plugins.ListObjectsV2Command({
|
|
||||||
Bucket: name,
|
|
||||||
ContinuationToken: continuationToken,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
for (const obj of listResp.Contents || []) {
|
|
||||||
objectCount++;
|
|
||||||
totalSizeBytes += obj.Size || 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
continuationToken = listResp.IsTruncated ? listResp.NextContinuationToken : undefined;
|
|
||||||
} while (continuationToken);
|
|
||||||
|
|
||||||
buckets.push({ name, creationDate, objectCount, totalSizeBytes });
|
|
||||||
}
|
|
||||||
|
|
||||||
return buckets;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async createBucket(bucketName: string): Promise<void> {
|
public async createBucket(bucketName: string): Promise<void> {
|
||||||
@@ -314,13 +376,7 @@ export class ObjectStorageContainer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getServerStats(): Promise<interfaces.data.IServerStatus> {
|
public async getServerStats(): Promise<interfaces.data.IServerStatus> {
|
||||||
const buckets = await this.listBuckets();
|
const stats = await this.smartstorageInstance.getStorageStats();
|
||||||
let totalObjectCount = 0;
|
|
||||||
let totalStorageBytes = 0;
|
|
||||||
for (const b of buckets) {
|
|
||||||
totalObjectCount += b.objectCount;
|
|
||||||
totalStorageBytes += b.totalSizeBytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
running: true,
|
running: true,
|
||||||
@@ -328,15 +384,19 @@ export class ObjectStorageContainer {
|
|||||||
uiPort: this.config.uiPort,
|
uiPort: this.config.uiPort,
|
||||||
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
|
||||||
startedAt: this.startedAt,
|
startedAt: this.startedAt,
|
||||||
bucketCount: buckets.length,
|
bucketCount: stats.bucketCount,
|
||||||
totalObjectCount,
|
totalObjectCount: stats.totalObjectCount,
|
||||||
totalStorageBytes,
|
totalStorageBytes: stats.totalStorageBytes,
|
||||||
storageDirectory: this.config.storageDirectory,
|
storageDirectory: stats.storageDirectory,
|
||||||
region: this.config.region,
|
region: this.config.region,
|
||||||
authEnabled: true,
|
authEnabled: true,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async getClusterHealth(): Promise<interfaces.data.IClusterHealth> {
|
||||||
|
return await this.smartstorageInstance.getClusterHealth();
|
||||||
|
}
|
||||||
|
|
||||||
public async getBucketPolicy(bucketName: string): Promise<string | null> {
|
public async getBucketPolicy(bucketName: string): Promise<string | null> {
|
||||||
try {
|
try {
|
||||||
const response = await this.s3Client.send(
|
const response = await this.s3Client.send(
|
||||||
@@ -373,4 +433,186 @@ export class ObjectStorageContainer {
|
|||||||
region: this.config.region,
|
region: this.config.region,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public isReady(): boolean {
|
||||||
|
return Boolean(this.smartstorageInstance && this.s3Client && this.startedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getOperationalHealth(): Promise<Record<string, unknown>> {
|
||||||
|
const clusterHealth = this.smartstorageInstance
|
||||||
|
? await this.getClusterHealth()
|
||||||
|
: { enabled: false };
|
||||||
|
const stats = this.smartstorageInstance
|
||||||
|
? await this.getServerStats()
|
||||||
|
: null;
|
||||||
|
const ready = this.isReady();
|
||||||
|
return {
|
||||||
|
ok: ready,
|
||||||
|
status: ready ? 'healthy' : 'starting',
|
||||||
|
startedAt: this.startedAt || null,
|
||||||
|
uptimeSeconds: this.startedAt ? Math.floor((Date.now() - this.startedAt) / 1000) : 0,
|
||||||
|
storageDirectory: this.config.storageDirectory,
|
||||||
|
stats,
|
||||||
|
cluster: clusterHealth,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getOperationalMetrics(): Promise<string> {
|
||||||
|
const stats = this.smartstorageInstance
|
||||||
|
? await this.smartstorageInstance.getStorageStats()
|
||||||
|
: null;
|
||||||
|
const clusterHealth = this.smartstorageInstance
|
||||||
|
? await this.smartstorageInstance.getClusterHealth()
|
||||||
|
: { enabled: false };
|
||||||
|
return [
|
||||||
|
'# HELP objectstorage_ready ObjectStorage readiness state.',
|
||||||
|
'# TYPE objectstorage_ready gauge',
|
||||||
|
`objectstorage_ready ${this.isReady() ? 1 : 0}`,
|
||||||
|
'# HELP objectstorage_buckets_total Runtime bucket count.',
|
||||||
|
'# TYPE objectstorage_buckets_total gauge',
|
||||||
|
`objectstorage_buckets_total ${stats?.bucketCount ?? 0}`,
|
||||||
|
'# HELP objectstorage_objects_total Runtime object count.',
|
||||||
|
'# TYPE objectstorage_objects_total gauge',
|
||||||
|
`objectstorage_objects_total ${stats?.totalObjectCount ?? 0}`,
|
||||||
|
'# HELP objectstorage_cluster_enabled Cluster mode enabled.',
|
||||||
|
'# TYPE objectstorage_cluster_enabled gauge',
|
||||||
|
`objectstorage_cluster_enabled ${clusterHealth.enabled ? 1 : 0}`,
|
||||||
|
'',
|
||||||
|
].join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
private get persistedAdminConfigPath(): string {
|
||||||
|
return `${this.config.storageDirectory}/.objectstorage/admin-config.json`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async loadPersistedAdminConfig(): Promise<void> {
|
||||||
|
if (this.envAccessCredentialsProvided) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const content = await Deno.readTextFile(this.persistedAdminConfigPath);
|
||||||
|
const persistedConfig = JSON.parse(content) as IPersistedAdminConfig;
|
||||||
|
const persistedCredentials = persistedConfig.accessCredentials;
|
||||||
|
|
||||||
|
if (!Array.isArray(persistedCredentials) || persistedCredentials.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const validCredentials = persistedCredentials
|
||||||
|
.filter((credential) => credential?.accessKeyId && credential?.secretAccessKey)
|
||||||
|
.map((credential) => ({
|
||||||
|
accessKeyId: credential.accessKeyId,
|
||||||
|
secretAccessKey: credential.secretAccessKey,
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (validCredentials.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.config.accessCredentials = validCredentials;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof Deno.errors.NotFound) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async savePersistedAdminConfig(): Promise<void> {
|
||||||
|
const dirPath = this.persistedAdminConfigPath.substring(
|
||||||
|
0,
|
||||||
|
this.persistedAdminConfigPath.lastIndexOf('/'),
|
||||||
|
);
|
||||||
|
await Deno.mkdir(dirPath, { recursive: true });
|
||||||
|
const persistedConfig: IPersistedAdminConfig = {
|
||||||
|
accessCredentials: this.config.accessCredentials,
|
||||||
|
};
|
||||||
|
await Deno.writeTextFile(
|
||||||
|
this.persistedAdminConfigPath,
|
||||||
|
JSON.stringify(persistedConfig, null, 2),
|
||||||
|
{ mode: 0o600 },
|
||||||
|
);
|
||||||
|
await this.restrictPersistedAdminConfigPermissions();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async refreshManagementClient(): Promise<void> {
|
||||||
|
const descriptor = await this.smartstorageInstance.getStorageDescriptor();
|
||||||
|
this.s3Client = new plugins.S3Client({
|
||||||
|
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
|
||||||
|
region: this.config.region,
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: descriptor.accessKey,
|
||||||
|
secretAccessKey: descriptor.accessSecret,
|
||||||
|
},
|
||||||
|
forcePathStyle: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildSmartstorageConfig(): any {
|
||||||
|
const smartstorageConfig: any = {
|
||||||
|
server: {
|
||||||
|
port: this.config.objstPort,
|
||||||
|
address: '0.0.0.0',
|
||||||
|
region: this.config.region,
|
||||||
|
},
|
||||||
|
storage: {
|
||||||
|
directory: this.config.storageDirectory,
|
||||||
|
},
|
||||||
|
auth: {
|
||||||
|
enabled: true,
|
||||||
|
credentials: this.config.accessCredentials,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
if (this.config.clusterEnabled) {
|
||||||
|
smartstorageConfig.cluster = {
|
||||||
|
enabled: true,
|
||||||
|
nodeId: this.config.clusterNodeId || crypto.randomUUID().slice(0, 8),
|
||||||
|
quicPort: this.config.clusterQuicPort,
|
||||||
|
seedNodes: this.config.clusterSeedNodes,
|
||||||
|
erasure: {
|
||||||
|
dataShards: this.config.erasureDataShards,
|
||||||
|
parityShards: this.config.erasureParityShards,
|
||||||
|
chunkSizeBytes: this.config.erasureChunkSizeBytes,
|
||||||
|
},
|
||||||
|
drives: {
|
||||||
|
paths: this.config.drivePaths.length > 0
|
||||||
|
? this.config.drivePaths
|
||||||
|
: [this.config.storageDirectory],
|
||||||
|
},
|
||||||
|
heartbeatIntervalMs: this.config.clusterHeartbeatIntervalMs,
|
||||||
|
heartbeatTimeoutMs: this.config.clusterHeartbeatTimeoutMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return smartstorageConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
private assertSecureStartupConfig(): void {
|
||||||
|
const allowInsecureDefaults = Deno.env.get('OBJST_ALLOW_INSECURE_DEFAULTS') === 'true';
|
||||||
|
const usesDefaultAdminPassword = this.config.adminPassword === 'admin';
|
||||||
|
const usesDefaultAccessCredential = this.config.accessCredentials.some((credential) => {
|
||||||
|
return credential.accessKeyId === 'admin' && credential.secretAccessKey === 'admin';
|
||||||
|
});
|
||||||
|
const looksLikePersistentProductionStorage = this.config.storageDirectory === '/data';
|
||||||
|
|
||||||
|
if (
|
||||||
|
looksLikePersistentProductionStorage &&
|
||||||
|
!allowInsecureDefaults &&
|
||||||
|
(usesDefaultAdminPassword || usesDefaultAccessCredential)
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
'Refusing to start with default admin credentials on persistent /data storage. Set OBJST_ADMIN_PASSWORD and OBJST_ACCESS_KEY/OBJST_SECRET_KEY, or set OBJST_ALLOW_INSECURE_DEFAULTS=true for disposable development.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async restrictPersistedAdminConfigPermissions(): Promise<void> {
|
||||||
|
try {
|
||||||
|
await Deno.chmod(this.persistedAdminConfigPath, 0o600);
|
||||||
|
} catch {
|
||||||
|
// chmod is not available on every platform Deno supports.
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,27 @@ export async function runCli(): Promise<void> {
|
|||||||
// Use a temp directory for storage
|
// Use a temp directory for storage
|
||||||
configOverrides.storageDirectory = './.nogit/objstdata';
|
configOverrides.storageDirectory = './.nogit/objstdata';
|
||||||
break;
|
break;
|
||||||
|
case '--cluster-enabled':
|
||||||
|
configOverrides.clusterEnabled = true;
|
||||||
|
break;
|
||||||
|
case '--cluster-node-id':
|
||||||
|
configOverrides.clusterNodeId = args[++i];
|
||||||
|
break;
|
||||||
|
case '--cluster-quic-port':
|
||||||
|
configOverrides.clusterQuicPort = parseInt(args[++i], 10);
|
||||||
|
break;
|
||||||
|
case '--cluster-seed-nodes':
|
||||||
|
configOverrides.clusterSeedNodes = args[++i].split(',').map(s => s.trim()).filter(Boolean);
|
||||||
|
break;
|
||||||
|
case '--drive-paths':
|
||||||
|
configOverrides.drivePaths = args[++i].split(',').map(s => s.trim()).filter(Boolean);
|
||||||
|
break;
|
||||||
|
case '--erasure-data-shards':
|
||||||
|
configOverrides.erasureDataShards = parseInt(args[++i], 10);
|
||||||
|
break;
|
||||||
|
case '--erasure-parity-shards':
|
||||||
|
configOverrides.erasureParityShards = parseInt(args[++i], 10);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -59,12 +80,21 @@ ObjectStorage - S3-compatible object storage server with management UI
|
|||||||
Usage:
|
Usage:
|
||||||
objectstorage server [options]
|
objectstorage server [options]
|
||||||
|
|
||||||
Options:
|
Server Options:
|
||||||
--ephemeral Use local .nogit/objstdata for storage
|
--ephemeral Use local .nogit/objstdata for storage
|
||||||
--storage-port PORT Storage API port (default: 9000, env: OBJST_PORT)
|
--storage-port PORT Storage API port (default: 9000, env: OBJST_PORT)
|
||||||
--ui-port PORT Management UI port (default: 3000, env: UI_PORT)
|
--ui-port PORT Management UI port (default: 3000, env: UI_PORT)
|
||||||
--storage-dir DIR Storage directory (default: /data, env: OBJST_STORAGE_DIR)
|
--storage-dir DIR Storage directory (default: /data, env: OBJST_STORAGE_DIR)
|
||||||
|
|
||||||
|
Clustering Options:
|
||||||
|
--cluster-enabled Enable cluster mode (env: OBJST_CLUSTER_ENABLED)
|
||||||
|
--cluster-node-id ID Unique node identifier (env: OBJST_CLUSTER_NODE_ID)
|
||||||
|
--cluster-quic-port PORT QUIC transport port (default: 4433, env: OBJST_CLUSTER_QUIC_PORT)
|
||||||
|
--cluster-seed-nodes LIST Comma-separated seed node addresses (env: OBJST_CLUSTER_SEED_NODES)
|
||||||
|
--drive-paths LIST Comma-separated drive mount paths (env: OBJST_DRIVE_PATHS)
|
||||||
|
--erasure-data-shards N Erasure coding data shards (default: 4, env: OBJST_ERASURE_DATA_SHARDS)
|
||||||
|
--erasure-parity-shards N Erasure coding parity shards (default: 2, env: OBJST_ERASURE_PARITY_SHARDS)
|
||||||
|
|
||||||
Environment Variables:
|
Environment Variables:
|
||||||
OBJST_PORT Storage API port
|
OBJST_PORT Storage API port
|
||||||
UI_PORT Management UI port
|
UI_PORT Management UI port
|
||||||
@@ -73,5 +103,16 @@ Environment Variables:
|
|||||||
OBJST_SECRET_KEY Secret key (default: admin)
|
OBJST_SECRET_KEY Secret key (default: admin)
|
||||||
OBJST_ADMIN_PASSWORD Admin UI password (default: admin)
|
OBJST_ADMIN_PASSWORD Admin UI password (default: admin)
|
||||||
OBJST_REGION Storage region (default: us-east-1)
|
OBJST_REGION Storage region (default: us-east-1)
|
||||||
|
OBJST_CLUSTER_ENABLED Enable clustering (true/false)
|
||||||
|
OBJST_CLUSTER_NODE_ID Unique node identifier
|
||||||
|
OBJST_CLUSTER_QUIC_PORT QUIC transport port (default: 4433)
|
||||||
|
OBJST_CLUSTER_SEED_NODES Comma-separated seed node addresses
|
||||||
|
OBJST_DRIVE_PATHS Comma-separated drive mount paths
|
||||||
|
OBJST_ERASURE_DATA_SHARDS Erasure data shards (default: 4)
|
||||||
|
OBJST_ERASURE_PARITY_SHARDS Erasure parity shards (default: 2)
|
||||||
|
OBJST_ERASURE_CHUNK_SIZE Erasure chunk size in bytes (default: 4194304)
|
||||||
|
OBJST_HEARTBEAT_INTERVAL_MS Cluster heartbeat interval (default: 5000)
|
||||||
|
OBJST_HEARTBEAT_TIMEOUT_MS Cluster heartbeat timeout (default: 30000)
|
||||||
|
OBJST_ALLOW_INSECURE_DEFAULTS Allow admin/admin defaults on /data for disposable development
|
||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ export class OpsServer {
|
|||||||
public configHandler!: handlers.ConfigHandler;
|
public configHandler!: handlers.ConfigHandler;
|
||||||
public credentialsHandler!: handlers.CredentialsHandler;
|
public credentialsHandler!: handlers.CredentialsHandler;
|
||||||
public policiesHandler!: handlers.PoliciesHandler;
|
public policiesHandler!: handlers.PoliciesHandler;
|
||||||
|
public auditHandler!: handlers.AuditHandler;
|
||||||
|
|
||||||
constructor(objectStorageRef: ObjectStorageContainer) {
|
constructor(objectStorageRef: ObjectStorageContainer) {
|
||||||
this.objectStorageRef = objectStorageRef;
|
this.objectStorageRef = objectStorageRef;
|
||||||
@@ -26,6 +27,27 @@ export class OpsServer {
|
|||||||
domain: 'localhost',
|
domain: 'localhost',
|
||||||
feedMetadata: undefined,
|
feedMetadata: undefined,
|
||||||
bundledContent: bundledFiles,
|
bundledContent: bundledFiles,
|
||||||
|
addCustomRoutes: async (typedserver) => {
|
||||||
|
typedserver.addRoute('/livez', 'GET', async () => {
|
||||||
|
return this.jsonResponse({ ok: true, status: 'alive' });
|
||||||
|
});
|
||||||
|
typedserver.addRoute('/readyz', 'GET', async () => {
|
||||||
|
const ready = await this.objectStorageRef.isReady();
|
||||||
|
return this.jsonResponse(
|
||||||
|
{ ok: ready, status: ready ? 'ready' : 'starting' },
|
||||||
|
ready ? 200 : 503,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
typedserver.addRoute('/healthz', 'GET', async () => {
|
||||||
|
return this.jsonResponse(await this.objectStorageRef.getOperationalHealth());
|
||||||
|
});
|
||||||
|
typedserver.addRoute('/metrics', 'GET', async () => {
|
||||||
|
const metrics = await this.objectStorageRef.getOperationalMetrics();
|
||||||
|
return new Response(metrics, {
|
||||||
|
headers: { 'content-type': 'text/plain; version=0.0.4' },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// Chain typedrouters: server -> opsServer -> individual handlers
|
// Chain typedrouters: server -> opsServer -> individual handlers
|
||||||
@@ -50,6 +72,7 @@ export class OpsServer {
|
|||||||
this.configHandler = new handlers.ConfigHandler(this);
|
this.configHandler = new handlers.ConfigHandler(this);
|
||||||
this.credentialsHandler = new handlers.CredentialsHandler(this);
|
this.credentialsHandler = new handlers.CredentialsHandler(this);
|
||||||
this.policiesHandler = new handlers.PoliciesHandler(this);
|
this.policiesHandler = new handlers.PoliciesHandler(this);
|
||||||
|
this.auditHandler = new handlers.AuditHandler(this);
|
||||||
|
|
||||||
console.log('OpsServer TypedRequest handlers initialized');
|
console.log('OpsServer TypedRequest handlers initialized');
|
||||||
}
|
}
|
||||||
@@ -60,4 +83,11 @@ export class OpsServer {
|
|||||||
console.log('OpsServer stopped');
|
console.log('OpsServer stopped');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private jsonResponse(data: unknown, status = 200): Response {
|
||||||
|
return new Response(JSON.stringify(data), {
|
||||||
|
status,
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import * as interfaces from '../../../ts_interfaces/index.ts';
|
|||||||
|
|
||||||
export interface IJwtData {
|
export interface IJwtData {
|
||||||
userId: string;
|
userId: string;
|
||||||
|
role: 'admin';
|
||||||
status: 'loggedIn' | 'loggedOut';
|
status: 'loggedIn' | 'loggedOut';
|
||||||
expiresAt: number;
|
expiresAt: number;
|
||||||
}
|
}
|
||||||
@@ -11,6 +12,8 @@ export interface IJwtData {
|
|||||||
export class AdminHandler {
|
export class AdminHandler {
|
||||||
public typedrouter = new plugins.typedrequest.TypedRouter();
|
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||||
public smartjwtInstance!: plugins.smartjwt.SmartJwt<IJwtData>;
|
public smartjwtInstance!: plugins.smartjwt.SmartJwt<IJwtData>;
|
||||||
|
private revokedTokens = new Set<string>();
|
||||||
|
private failedLoginAttempts = new Map<string, { count: number; firstAttemptAt: number }>();
|
||||||
|
|
||||||
constructor(private opsServerRef: OpsServer) {
|
constructor(private opsServerRef: OpsServer) {
|
||||||
this.opsServerRef.typedrouter.addTypedRouter(this.typedrouter);
|
this.opsServerRef.typedrouter.addTypedRouter(this.typedrouter);
|
||||||
@@ -29,19 +32,37 @@ export class AdminHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AdminLoginWithUsernameAndPassword>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AdminLoginWithUsernameAndPassword>(
|
||||||
'adminLoginWithUsernameAndPassword',
|
'adminLoginWithUsernameAndPassword',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
|
this.assertLoginNotRateLimited(dataArg.username);
|
||||||
const adminPassword = this.opsServerRef.objectStorageRef.config.adminPassword;
|
const adminPassword = this.opsServerRef.objectStorageRef.config.adminPassword;
|
||||||
if (dataArg.username !== 'admin' || dataArg.password !== adminPassword) {
|
if (dataArg.username !== 'admin' || dataArg.password !== adminPassword) {
|
||||||
|
this.recordFailedLogin(dataArg.username);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.username || 'anonymous',
|
||||||
|
action: 'admin.login',
|
||||||
|
targetType: 'adminSession',
|
||||||
|
success: false,
|
||||||
|
message: 'Invalid credentials',
|
||||||
|
});
|
||||||
throw new plugins.typedrequest.TypedResponseError('Invalid credentials');
|
throw new plugins.typedrequest.TypedResponseError('Invalid credentials');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.failedLoginAttempts.delete(dataArg.username);
|
||||||
const expiresAt = Date.now() + 24 * 3600 * 1000;
|
const expiresAt = Date.now() + 24 * 3600 * 1000;
|
||||||
const userId = 'admin';
|
const userId = 'admin';
|
||||||
const jwt = await this.smartjwtInstance.createJWT({
|
const jwt = await this.smartjwtInstance.createJWT({
|
||||||
userId,
|
userId,
|
||||||
|
role: 'admin',
|
||||||
status: 'loggedIn',
|
status: 'loggedIn',
|
||||||
expiresAt,
|
expiresAt,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: userId,
|
||||||
|
action: 'admin.login',
|
||||||
|
targetType: 'adminSession',
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
|
||||||
console.log('Admin user logged in');
|
console.log('Admin user logged in');
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -61,7 +82,16 @@ export class AdminHandler {
|
|||||||
this.typedrouter.addTypedHandler(
|
this.typedrouter.addTypedHandler(
|
||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AdminLogout>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AdminLogout>(
|
||||||
'adminLogout',
|
'adminLogout',
|
||||||
async (_dataArg) => {
|
async (dataArg) => {
|
||||||
|
if (dataArg.identity?.jwt) {
|
||||||
|
this.revokedTokens.add(dataArg.identity.jwt);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'admin.logout',
|
||||||
|
targetType: 'adminSession',
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -75,18 +105,20 @@ export class AdminHandler {
|
|||||||
if (!dataArg.identity?.jwt) {
|
if (!dataArg.identity?.jwt) {
|
||||||
return { valid: false };
|
return { valid: false };
|
||||||
}
|
}
|
||||||
|
if (this.revokedTokens.has(dataArg.identity.jwt)) return { valid: false };
|
||||||
try {
|
try {
|
||||||
const jwtData = await this.smartjwtInstance.verifyJWTAndGetData(dataArg.identity.jwt);
|
const jwtData = await this.smartjwtInstance.verifyJWTAndGetData(dataArg.identity.jwt);
|
||||||
if (jwtData.expiresAt < Date.now()) return { valid: false };
|
if (jwtData.expiresAt < Date.now()) return { valid: false };
|
||||||
if (jwtData.status !== 'loggedIn') return { valid: false };
|
if (jwtData.status !== 'loggedIn') return { valid: false };
|
||||||
|
if (jwtData.role !== 'admin') return { valid: false };
|
||||||
return {
|
return {
|
||||||
valid: true,
|
valid: true,
|
||||||
identity: {
|
identity: {
|
||||||
jwt: dataArg.identity.jwt,
|
jwt: dataArg.identity.jwt,
|
||||||
userId: jwtData.userId,
|
userId: jwtData.userId,
|
||||||
username: dataArg.identity.username,
|
username: jwtData.userId,
|
||||||
expiresAt: jwtData.expiresAt,
|
expiresAt: jwtData.expiresAt,
|
||||||
role: dataArg.identity.role,
|
role: jwtData.role,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
@@ -103,12 +135,15 @@ export class AdminHandler {
|
|||||||
}>(
|
}>(
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
if (!dataArg.identity?.jwt) return false;
|
if (!dataArg.identity?.jwt) return false;
|
||||||
|
if (this.revokedTokens.has(dataArg.identity.jwt)) return false;
|
||||||
try {
|
try {
|
||||||
const jwtData = await this.smartjwtInstance.verifyJWTAndGetData(dataArg.identity.jwt);
|
const jwtData = await this.smartjwtInstance.verifyJWTAndGetData(dataArg.identity.jwt);
|
||||||
if (jwtData.expiresAt < Date.now()) return false;
|
if (jwtData.expiresAt < Date.now()) return false;
|
||||||
if (jwtData.status !== 'loggedIn') return false;
|
if (jwtData.status !== 'loggedIn') return false;
|
||||||
|
if (jwtData.role !== 'admin') return false;
|
||||||
if (dataArg.identity.expiresAt !== jwtData.expiresAt) return false;
|
if (dataArg.identity.expiresAt !== jwtData.expiresAt) return false;
|
||||||
if (dataArg.identity.userId !== jwtData.userId) return false;
|
if (dataArg.identity.userId !== jwtData.userId) return false;
|
||||||
|
if (dataArg.identity.role !== jwtData.role) return false;
|
||||||
return true;
|
return true;
|
||||||
} catch {
|
} catch {
|
||||||
return false;
|
return false;
|
||||||
@@ -122,10 +157,33 @@ export class AdminHandler {
|
|||||||
identity: interfaces.data.IIdentity;
|
identity: interfaces.data.IIdentity;
|
||||||
}>(
|
}>(
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
const isValid = await this.validIdentityGuard.exec(dataArg);
|
return await this.validIdentityGuard.exec(dataArg);
|
||||||
if (!isValid) return false;
|
|
||||||
return dataArg.identity.role === 'admin';
|
|
||||||
},
|
},
|
||||||
{ failedHint: 'user is not admin', name: 'adminIdentityGuard' },
|
{ failedHint: 'user is not admin', name: 'adminIdentityGuard' },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
private assertLoginNotRateLimited(username: string): void {
|
||||||
|
const attempt = this.failedLoginAttempts.get(username);
|
||||||
|
if (!attempt) return;
|
||||||
|
|
||||||
|
const windowMs = 60 * 1000;
|
||||||
|
if (Date.now() - attempt.firstAttemptAt > windowMs) {
|
||||||
|
this.failedLoginAttempts.delete(username);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt.count >= 5) {
|
||||||
|
throw new plugins.typedrequest.TypedResponseError('Too many failed login attempts');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private recordFailedLogin(username: string): void {
|
||||||
|
const now = Date.now();
|
||||||
|
const attempt = this.failedLoginAttempts.get(username);
|
||||||
|
if (!attempt || now - attempt.firstAttemptAt > 60 * 1000) {
|
||||||
|
this.failedLoginAttempts.set(username, { count: 1, firstAttemptAt: now });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
attempt.count++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
import * as plugins from '../../plugins.ts';
|
||||||
|
import type { OpsServer } from '../classes.opsserver.ts';
|
||||||
|
import * as interfaces from '../../../ts_interfaces/index.ts';
|
||||||
|
import { requireAdminIdentity } from '../helpers/guards.ts';
|
||||||
|
|
||||||
|
export class AuditHandler {
|
||||||
|
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||||
|
|
||||||
|
constructor(private opsServerRef: OpsServer) {
|
||||||
|
this.opsServerRef.typedrouter.addTypedRouter(this.typedrouter);
|
||||||
|
this.registerHandlers();
|
||||||
|
}
|
||||||
|
|
||||||
|
private registerHandlers(): void {
|
||||||
|
this.typedrouter.addTypedHandler(
|
||||||
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_ListAuditEntries>(
|
||||||
|
'listAuditEntries',
|
||||||
|
async (dataArg) => {
|
||||||
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
|
const entries = await this.opsServerRef.objectStorageRef.auditLogger.listRecent(
|
||||||
|
dataArg.limit ?? 100,
|
||||||
|
);
|
||||||
|
return { entries };
|
||||||
|
},
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,14 @@
|
|||||||
import * as plugins from '../../plugins.ts';
|
import * as plugins from '../../plugins.ts';
|
||||||
import type { OpsServer } from '../classes.opsserver.ts';
|
import type { OpsServer } from '../classes.opsserver.ts';
|
||||||
import * as interfaces from '../../../ts_interfaces/index.ts';
|
import * as interfaces from '../../../ts_interfaces/index.ts';
|
||||||
import { requireValidIdentity } from '../helpers/guards.ts';
|
import { requireAdminIdentity, requireValidIdentity } from '../helpers/guards.ts';
|
||||||
|
|
||||||
|
const getStorageErrorCode = (error: unknown): string | undefined => {
|
||||||
|
if (!(error instanceof Error)) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return (error as Error & { Code?: string }).Code || error.name;
|
||||||
|
};
|
||||||
|
|
||||||
export class BucketsHandler {
|
export class BucketsHandler {
|
||||||
public typedrouter = new plugins.typedrequest.TypedRouter();
|
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||||
@@ -29,8 +36,27 @@ export class BucketsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_CreateBucket>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_CreateBucket>(
|
||||||
'createBucket',
|
'createBucket',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
|
try {
|
||||||
await this.opsServerRef.objectStorageRef.createBucket(dataArg.bucketName);
|
await this.opsServerRef.objectStorageRef.createBucket(dataArg.bucketName);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'bucket.create',
|
||||||
|
targetType: 'bucket',
|
||||||
|
targetId: dataArg.bucketName,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'bucket.create',
|
||||||
|
targetType: 'bucket',
|
||||||
|
targetId: dataArg.bucketName,
|
||||||
|
success: false,
|
||||||
|
message: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -41,9 +67,25 @@ export class BucketsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteBucket>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteBucket>(
|
||||||
'deleteBucket',
|
'deleteBucket',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
|
try {
|
||||||
await this.opsServerRef.objectStorageRef.deleteBucket(dataArg.bucketName);
|
await this.opsServerRef.objectStorageRef.deleteBucket(dataArg.bucketName);
|
||||||
await this.opsServerRef.objectStorageRef.policyManager.onBucketDeleted(dataArg.bucketName);
|
} catch (error) {
|
||||||
|
if (getStorageErrorCode(error) === 'NoSuchBucket') {
|
||||||
|
throw new plugins.typedrequest.TypedResponseError('Bucket not found');
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
await this.opsServerRef.objectStorageRef.policyManager.onBucketDeleted(
|
||||||
|
dataArg.bucketName,
|
||||||
|
);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'bucket.delete',
|
||||||
|
targetType: 'bucket',
|
||||||
|
targetId: dataArg.bucketName,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -54,8 +96,16 @@ export class BucketsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetBucketPolicy>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetBucketPolicy>(
|
||||||
'getBucketPolicy',
|
'getBucketPolicy',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
const policy = await this.opsServerRef.objectStorageRef.getBucketPolicy(dataArg.bucketName);
|
let policy: string | null;
|
||||||
|
try {
|
||||||
|
policy = await this.opsServerRef.objectStorageRef.getBucketPolicy(dataArg.bucketName);
|
||||||
|
} catch (error) {
|
||||||
|
if (getStorageErrorCode(error) === 'NoSuchBucket') {
|
||||||
|
throw new plugins.typedrequest.TypedResponseError('Bucket not found');
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
return { policy };
|
return { policy };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -66,14 +116,24 @@ export class BucketsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_PutBucketPolicy>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_PutBucketPolicy>(
|
||||||
'putBucketPolicy',
|
'putBucketPolicy',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
// Validate JSON
|
// Validate JSON
|
||||||
try {
|
try {
|
||||||
JSON.parse(dataArg.policy);
|
JSON.parse(dataArg.policy);
|
||||||
} catch {
|
} catch {
|
||||||
throw new Error('Invalid JSON policy document');
|
throw new plugins.typedrequest.TypedResponseError('Invalid JSON policy document');
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await this.opsServerRef.objectStorageRef.putBucketPolicy(
|
||||||
|
dataArg.bucketName,
|
||||||
|
dataArg.policy,
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
if (getStorageErrorCode(error) === 'NoSuchBucket') {
|
||||||
|
throw new plugins.typedrequest.TypedResponseError('Bucket not found');
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
}
|
}
|
||||||
await this.opsServerRef.objectStorageRef.putBucketPolicy(dataArg.bucketName, dataArg.policy);
|
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -84,8 +144,15 @@ export class BucketsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteBucketPolicy>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteBucketPolicy>(
|
||||||
'deleteBucketPolicy',
|
'deleteBucketPolicy',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
|
try {
|
||||||
await this.opsServerRef.objectStorageRef.deleteBucketPolicy(dataArg.bucketName);
|
await this.opsServerRef.objectStorageRef.deleteBucketPolicy(dataArg.bucketName);
|
||||||
|
} catch (error) {
|
||||||
|
if (getStorageErrorCode(error) === 'NoSuchBucket') {
|
||||||
|
throw new plugins.typedrequest.TypedResponseError('Bucket not found');
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -25,6 +25,16 @@ export class ConfigHandler {
|
|||||||
storageDirectory: containerConfig.storageDirectory,
|
storageDirectory: containerConfig.storageDirectory,
|
||||||
authEnabled: true,
|
authEnabled: true,
|
||||||
corsEnabled: false,
|
corsEnabled: false,
|
||||||
|
clusterEnabled: containerConfig.clusterEnabled,
|
||||||
|
clusterNodeId: containerConfig.clusterNodeId,
|
||||||
|
clusterQuicPort: containerConfig.clusterQuicPort,
|
||||||
|
clusterSeedNodes: containerConfig.clusterSeedNodes,
|
||||||
|
erasureDataShards: containerConfig.erasureDataShards,
|
||||||
|
erasureParityShards: containerConfig.erasureParityShards,
|
||||||
|
erasureChunkSizeBytes: containerConfig.erasureChunkSizeBytes,
|
||||||
|
drivePaths: containerConfig.drivePaths,
|
||||||
|
clusterHeartbeatIntervalMs: containerConfig.clusterHeartbeatIntervalMs,
|
||||||
|
clusterHeartbeatTimeoutMs: containerConfig.clusterHeartbeatTimeoutMs,
|
||||||
};
|
};
|
||||||
return { config };
|
return { config };
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import * as plugins from '../../plugins.ts';
|
import * as plugins from '../../plugins.ts';
|
||||||
import type { OpsServer } from '../classes.opsserver.ts';
|
import type { OpsServer } from '../classes.opsserver.ts';
|
||||||
import * as interfaces from '../../../ts_interfaces/index.ts';
|
import * as interfaces from '../../../ts_interfaces/index.ts';
|
||||||
import { requireValidIdentity } from '../helpers/guards.ts';
|
import { requireAdminIdentity, requireValidIdentity } from '../helpers/guards.ts';
|
||||||
|
|
||||||
export class CredentialsHandler {
|
export class CredentialsHandler {
|
||||||
public typedrouter = new plugins.typedrequest.TypedRouter();
|
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||||
@@ -18,10 +18,12 @@ export class CredentialsHandler {
|
|||||||
'getCredentials',
|
'getCredentials',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
const credentials = this.opsServerRef.objectStorageRef.config.accessCredentials.map(
|
const activeCredentials = await this.opsServerRef.objectStorageRef
|
||||||
|
.listAccessCredentials();
|
||||||
|
const credentials = activeCredentials.map(
|
||||||
(cred) => ({
|
(cred) => ({
|
||||||
accessKeyId: cred.accessKeyId,
|
accessKeyId: cred.accessKeyId,
|
||||||
secretAccessKey: cred.secretAccessKey.slice(0, 4) + '****',
|
secretAccessKey: '********',
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
return { credentials };
|
return { credentials };
|
||||||
@@ -34,14 +36,38 @@ export class CredentialsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AddCredential>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AddCredential>(
|
||||||
'addCredential',
|
'addCredential',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
this.opsServerRef.objectStorageRef.config.accessCredentials.push({
|
const credentials = this.opsServerRef.objectStorageRef.config.accessCredentials;
|
||||||
|
if (credentials.some((credential) => credential.accessKeyId === dataArg.accessKeyId)) {
|
||||||
|
throw new plugins.typedrequest.TypedResponseError('Credential already exists');
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.opsServerRef.objectStorageRef.replaceAccessCredentials([
|
||||||
|
...credentials,
|
||||||
|
{
|
||||||
accessKeyId: dataArg.accessKeyId,
|
accessKeyId: dataArg.accessKeyId,
|
||||||
secretAccessKey: dataArg.secretAccessKey,
|
secretAccessKey: dataArg.secretAccessKey,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'credential.add',
|
||||||
|
targetType: 'credential',
|
||||||
|
targetId: dataArg.accessKeyId,
|
||||||
|
success: true,
|
||||||
});
|
});
|
||||||
// Update the smartstorage auth config
|
} catch (error) {
|
||||||
this.opsServerRef.objectStorageRef.smartstorageInstance.config.auth!.credentials =
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
this.opsServerRef.objectStorageRef.config.accessCredentials;
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'credential.add',
|
||||||
|
targetType: 'credential',
|
||||||
|
targetId: dataArg.accessKeyId,
|
||||||
|
success: false,
|
||||||
|
message: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -52,19 +78,38 @@ export class CredentialsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_RemoveCredential>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_RemoveCredential>(
|
||||||
'removeCredential',
|
'removeCredential',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
const creds = this.opsServerRef.objectStorageRef.config.accessCredentials;
|
const creds = this.opsServerRef.objectStorageRef.config.accessCredentials;
|
||||||
|
if (!creds.some((credential) => credential.accessKeyId === dataArg.accessKeyId)) {
|
||||||
|
throw new plugins.typedrequest.TypedResponseError('Credential not found');
|
||||||
|
}
|
||||||
if (creds.length <= 1) {
|
if (creds.length <= 1) {
|
||||||
throw new plugins.typedrequest.TypedResponseError(
|
throw new plugins.typedrequest.TypedResponseError(
|
||||||
'Cannot remove the last credential',
|
'Cannot remove the last credential',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
this.opsServerRef.objectStorageRef.config.accessCredentials = creds.filter(
|
try {
|
||||||
(c) => c.accessKeyId !== dataArg.accessKeyId,
|
await this.opsServerRef.objectStorageRef.replaceAccessCredentials(
|
||||||
|
creds.filter((credential) => credential.accessKeyId !== dataArg.accessKeyId),
|
||||||
);
|
);
|
||||||
// Update the smartstorage auth config
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
this.opsServerRef.objectStorageRef.smartstorageInstance.config.auth!.credentials =
|
actorUserId: dataArg.identity.userId,
|
||||||
this.opsServerRef.objectStorageRef.config.accessCredentials;
|
action: 'credential.remove',
|
||||||
|
targetType: 'credential',
|
||||||
|
targetId: dataArg.accessKeyId,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'credential.remove',
|
||||||
|
targetType: 'credential',
|
||||||
|
targetId: dataArg.accessKeyId,
|
||||||
|
success: false,
|
||||||
|
message: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -5,3 +5,4 @@ export { ObjectsHandler } from './objects.handler.ts';
|
|||||||
export { ConfigHandler } from './config.handler.ts';
|
export { ConfigHandler } from './config.handler.ts';
|
||||||
export { CredentialsHandler } from './credentials.handler.ts';
|
export { CredentialsHandler } from './credentials.handler.ts';
|
||||||
export { PoliciesHandler } from './policies.handler.ts';
|
export { PoliciesHandler } from './policies.handler.ts';
|
||||||
|
export { AuditHandler } from './audit.handler.ts';
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import * as plugins from '../../plugins.ts';
|
import * as plugins from '../../plugins.ts';
|
||||||
import type { OpsServer } from '../classes.opsserver.ts';
|
import type { OpsServer } from '../classes.opsserver.ts';
|
||||||
import * as interfaces from '../../../ts_interfaces/index.ts';
|
import * as interfaces from '../../../ts_interfaces/index.ts';
|
||||||
import { requireValidIdentity } from '../helpers/guards.ts';
|
import { requireAdminIdentity, requireValidIdentity } from '../helpers/guards.ts';
|
||||||
|
|
||||||
export class ObjectsHandler {
|
export class ObjectsHandler {
|
||||||
public typedrouter = new plugins.typedrequest.TypedRouter();
|
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||||
@@ -34,8 +34,15 @@ export class ObjectsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteObject>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteObject>(
|
||||||
'deleteObject',
|
'deleteObject',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
await this.opsServerRef.objectStorageRef.deleteObject(dataArg.bucketName, dataArg.key);
|
await this.opsServerRef.objectStorageRef.deleteObject(dataArg.bucketName, dataArg.key);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'object.delete',
|
||||||
|
targetType: 'object',
|
||||||
|
targetId: `${dataArg.bucketName}/${dataArg.key}`,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -57,13 +64,20 @@ export class ObjectsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_PutObject>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_PutObject>(
|
||||||
'putObject',
|
'putObject',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
await this.opsServerRef.objectStorageRef.putObject(
|
await this.opsServerRef.objectStorageRef.putObject(
|
||||||
dataArg.bucketName,
|
dataArg.bucketName,
|
||||||
dataArg.key,
|
dataArg.key,
|
||||||
dataArg.base64Content,
|
dataArg.base64Content,
|
||||||
dataArg.contentType,
|
dataArg.contentType,
|
||||||
);
|
);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'object.put',
|
||||||
|
targetType: 'object',
|
||||||
|
targetId: `${dataArg.bucketName}/${dataArg.key}`,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -74,8 +88,15 @@ export class ObjectsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeletePrefix>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeletePrefix>(
|
||||||
'deletePrefix',
|
'deletePrefix',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
await this.opsServerRef.objectStorageRef.deletePrefix(dataArg.bucketName, dataArg.prefix);
|
await this.opsServerRef.objectStorageRef.deletePrefix(dataArg.bucketName, dataArg.prefix);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'objectPrefix.delete',
|
||||||
|
targetType: 'objectPrefix',
|
||||||
|
targetId: `${dataArg.bucketName}/${dataArg.prefix}`,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -98,12 +119,22 @@ export class ObjectsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_MoveObject>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_MoveObject>(
|
||||||
'moveObject',
|
'moveObject',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
return await this.opsServerRef.objectStorageRef.moveObject(
|
const result = await this.opsServerRef.objectStorageRef.moveObject(
|
||||||
dataArg.bucketName,
|
dataArg.bucketName,
|
||||||
dataArg.sourceKey,
|
dataArg.sourceKey,
|
||||||
dataArg.destKey,
|
dataArg.destKey,
|
||||||
);
|
);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'object.move',
|
||||||
|
targetType: 'object',
|
||||||
|
targetId: `${dataArg.bucketName}/${dataArg.sourceKey}`,
|
||||||
|
success: result.success,
|
||||||
|
metadata: { destKey: dataArg.destKey },
|
||||||
|
message: result.error,
|
||||||
|
});
|
||||||
|
return result;
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
@@ -113,12 +144,22 @@ export class ObjectsHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_MovePrefix>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_MovePrefix>(
|
||||||
'movePrefix',
|
'movePrefix',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
return await this.opsServerRef.objectStorageRef.movePrefix(
|
const result = await this.opsServerRef.objectStorageRef.movePrefix(
|
||||||
dataArg.bucketName,
|
dataArg.bucketName,
|
||||||
dataArg.sourcePrefix,
|
dataArg.sourcePrefix,
|
||||||
dataArg.destPrefix,
|
dataArg.destPrefix,
|
||||||
);
|
);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'objectPrefix.move',
|
||||||
|
targetType: 'objectPrefix',
|
||||||
|
targetId: `${dataArg.bucketName}/${dataArg.sourcePrefix}`,
|
||||||
|
success: result.success,
|
||||||
|
metadata: { destPrefix: dataArg.destPrefix },
|
||||||
|
message: result.error,
|
||||||
|
});
|
||||||
|
return result;
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import * as plugins from '../../plugins.ts';
|
import * as plugins from '../../plugins.ts';
|
||||||
import type { OpsServer } from '../classes.opsserver.ts';
|
import type { OpsServer } from '../classes.opsserver.ts';
|
||||||
import * as interfaces from '../../../ts_interfaces/index.ts';
|
import * as interfaces from '../../../ts_interfaces/index.ts';
|
||||||
import { requireValidIdentity } from '../helpers/guards.ts';
|
import { requireAdminIdentity, requireValidIdentity } from '../helpers/guards.ts';
|
||||||
|
|
||||||
export class PoliciesHandler {
|
export class PoliciesHandler {
|
||||||
public typedrouter = new plugins.typedrequest.TypedRouter();
|
public typedrouter = new plugins.typedrequest.TypedRouter();
|
||||||
@@ -30,8 +30,15 @@ export class PoliciesHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_CreateNamedPolicy>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_CreateNamedPolicy>(
|
||||||
'createNamedPolicy',
|
'createNamedPolicy',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
const policy = await pm().createPolicy(dataArg.name, dataArg.description, dataArg.statements);
|
const policy = await pm().createPolicy(dataArg.name, dataArg.description, dataArg.statements);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'policy.create',
|
||||||
|
targetType: 'policy',
|
||||||
|
targetId: policy.id,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
return { policy };
|
return { policy };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -42,8 +49,15 @@ export class PoliciesHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_UpdateNamedPolicy>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_UpdateNamedPolicy>(
|
||||||
'updateNamedPolicy',
|
'updateNamedPolicy',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
const policy = await pm().updatePolicy(dataArg.policyId, dataArg.name, dataArg.description, dataArg.statements);
|
const policy = await pm().updatePolicy(dataArg.policyId, dataArg.name, dataArg.description, dataArg.statements);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'policy.update',
|
||||||
|
targetType: 'policy',
|
||||||
|
targetId: dataArg.policyId,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
return { policy };
|
return { policy };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -54,8 +68,15 @@ export class PoliciesHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteNamedPolicy>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DeleteNamedPolicy>(
|
||||||
'deleteNamedPolicy',
|
'deleteNamedPolicy',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
await pm().deletePolicy(dataArg.policyId);
|
await pm().deletePolicy(dataArg.policyId);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'policy.delete',
|
||||||
|
targetType: 'policy',
|
||||||
|
targetId: dataArg.policyId,
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -77,8 +98,16 @@ export class PoliciesHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AttachPolicyToBucket>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_AttachPolicyToBucket>(
|
||||||
'attachPolicyToBucket',
|
'attachPolicyToBucket',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
await pm().attachPolicyToBucket(dataArg.policyId, dataArg.bucketName);
|
await pm().attachPolicyToBucket(dataArg.policyId, dataArg.bucketName);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'policy.attach',
|
||||||
|
targetType: 'bucket',
|
||||||
|
targetId: dataArg.bucketName,
|
||||||
|
success: true,
|
||||||
|
metadata: { policyId: dataArg.policyId },
|
||||||
|
});
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -89,8 +118,16 @@ export class PoliciesHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DetachPolicyFromBucket>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_DetachPolicyFromBucket>(
|
||||||
'detachPolicyFromBucket',
|
'detachPolicyFromBucket',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
await pm().detachPolicyFromBucket(dataArg.policyId, dataArg.bucketName);
|
await pm().detachPolicyFromBucket(dataArg.policyId, dataArg.bucketName);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'policy.detach',
|
||||||
|
targetType: 'bucket',
|
||||||
|
targetId: dataArg.bucketName,
|
||||||
|
success: true,
|
||||||
|
metadata: { policyId: dataArg.policyId },
|
||||||
|
});
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -118,8 +155,16 @@ export class PoliciesHandler {
|
|||||||
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_SetPolicyBuckets>(
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_SetPolicyBuckets>(
|
||||||
'setPolicyBuckets',
|
'setPolicyBuckets',
|
||||||
async (dataArg) => {
|
async (dataArg) => {
|
||||||
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
await requireAdminIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
await pm().setPolicyBuckets(dataArg.policyId, dataArg.bucketNames);
|
await pm().setPolicyBuckets(dataArg.policyId, dataArg.bucketNames);
|
||||||
|
await this.opsServerRef.objectStorageRef.auditLogger.log({
|
||||||
|
actorUserId: dataArg.identity.userId,
|
||||||
|
action: 'policy.setBuckets',
|
||||||
|
targetType: 'policy',
|
||||||
|
targetId: dataArg.policyId,
|
||||||
|
success: true,
|
||||||
|
metadata: { bucketCount: dataArg.bucketNames.length },
|
||||||
|
});
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -23,5 +23,16 @@ export class StatusHandler {
|
|||||||
},
|
},
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
this.typedrouter.addTypedHandler(
|
||||||
|
new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetClusterHealth>(
|
||||||
|
'getClusterHealth',
|
||||||
|
async (dataArg) => {
|
||||||
|
await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
|
||||||
|
const clusterHealth = await this.opsServerRef.objectStorageRef.getClusterHealth();
|
||||||
|
return { clusterHealth };
|
||||||
|
},
|
||||||
|
),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+24
@@ -5,6 +5,20 @@ export interface IObjectStorageConfig {
|
|||||||
accessCredentials: Array<{ accessKeyId: string; secretAccessKey: string }>;
|
accessCredentials: Array<{ accessKeyId: string; secretAccessKey: string }>;
|
||||||
adminPassword: string;
|
adminPassword: string;
|
||||||
region: string;
|
region: string;
|
||||||
|
// Cluster
|
||||||
|
clusterEnabled: boolean;
|
||||||
|
clusterNodeId: string;
|
||||||
|
clusterQuicPort: number;
|
||||||
|
clusterSeedNodes: string[];
|
||||||
|
// Erasure coding
|
||||||
|
erasureDataShards: number;
|
||||||
|
erasureParityShards: number;
|
||||||
|
erasureChunkSizeBytes: number;
|
||||||
|
// Multi-drive
|
||||||
|
drivePaths: string[];
|
||||||
|
// Cluster heartbeat
|
||||||
|
clusterHeartbeatIntervalMs: number;
|
||||||
|
clusterHeartbeatTimeoutMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const defaultConfig: IObjectStorageConfig = {
|
export const defaultConfig: IObjectStorageConfig = {
|
||||||
@@ -14,4 +28,14 @@ export const defaultConfig: IObjectStorageConfig = {
|
|||||||
accessCredentials: [{ accessKeyId: 'admin', secretAccessKey: 'admin' }],
|
accessCredentials: [{ accessKeyId: 'admin', secretAccessKey: 'admin' }],
|
||||||
adminPassword: 'admin',
|
adminPassword: 'admin',
|
||||||
region: 'us-east-1',
|
region: 'us-east-1',
|
||||||
|
clusterEnabled: false,
|
||||||
|
clusterNodeId: '',
|
||||||
|
clusterQuicPort: 4433,
|
||||||
|
clusterSeedNodes: [],
|
||||||
|
erasureDataShards: 4,
|
||||||
|
erasureParityShards: 2,
|
||||||
|
erasureChunkSizeBytes: 4194304,
|
||||||
|
drivePaths: [],
|
||||||
|
clusterHeartbeatIntervalMs: 5000,
|
||||||
|
clusterHeartbeatTimeoutMs: 30000,
|
||||||
};
|
};
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -19,6 +19,20 @@ export interface IServerConfig {
|
|||||||
storageDirectory: string;
|
storageDirectory: string;
|
||||||
authEnabled: boolean;
|
authEnabled: boolean;
|
||||||
corsEnabled: boolean;
|
corsEnabled: boolean;
|
||||||
|
// Cluster
|
||||||
|
clusterEnabled: boolean;
|
||||||
|
clusterNodeId: string;
|
||||||
|
clusterQuicPort: number;
|
||||||
|
clusterSeedNodes: string[];
|
||||||
|
// Erasure coding
|
||||||
|
erasureDataShards: number;
|
||||||
|
erasureParityShards: number;
|
||||||
|
erasureChunkSizeBytes: number;
|
||||||
|
// Multi-drive
|
||||||
|
drivePaths: string[];
|
||||||
|
// Cluster heartbeat
|
||||||
|
clusterHeartbeatIntervalMs: number;
|
||||||
|
clusterHeartbeatTimeoutMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IObjstCredential {
|
export interface IObjstCredential {
|
||||||
@@ -33,3 +47,70 @@ export interface IConnectionInfo {
|
|||||||
accessKey: string;
|
accessKey: string;
|
||||||
region: string;
|
region: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface IAuditEntry {
|
||||||
|
timestamp: number;
|
||||||
|
actorUserId: string;
|
||||||
|
action: string;
|
||||||
|
targetType: string;
|
||||||
|
targetId?: string;
|
||||||
|
success: boolean;
|
||||||
|
message?: string;
|
||||||
|
metadata?: Record<string, string | number | boolean>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IClusterPeerHealth {
|
||||||
|
nodeId: string;
|
||||||
|
status: 'online' | 'suspect' | 'offline';
|
||||||
|
quicAddress?: string;
|
||||||
|
s3Address?: string;
|
||||||
|
driveCount?: number;
|
||||||
|
lastHeartbeat?: number;
|
||||||
|
missedHeartbeats?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IClusterDriveHealth {
|
||||||
|
index: number;
|
||||||
|
path: string;
|
||||||
|
status: 'online' | 'degraded' | 'offline' | 'healing';
|
||||||
|
totalBytes?: number;
|
||||||
|
usedBytes?: number;
|
||||||
|
availableBytes?: number;
|
||||||
|
errorCount?: number;
|
||||||
|
lastError?: string;
|
||||||
|
lastCheck?: number;
|
||||||
|
erasureSetId?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IClusterErasureHealth {
|
||||||
|
dataShards: number;
|
||||||
|
parityShards: number;
|
||||||
|
chunkSizeBytes: number;
|
||||||
|
totalShards: number;
|
||||||
|
readQuorum: number;
|
||||||
|
writeQuorum: number;
|
||||||
|
erasureSetCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IClusterRepairHealth {
|
||||||
|
active: boolean;
|
||||||
|
scanIntervalMs?: number;
|
||||||
|
lastRunStartedAt?: number;
|
||||||
|
lastRunCompletedAt?: number;
|
||||||
|
lastDurationMs?: number;
|
||||||
|
shardsChecked?: number;
|
||||||
|
shardsHealed?: number;
|
||||||
|
failed?: number;
|
||||||
|
lastError?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IClusterHealth {
|
||||||
|
enabled: boolean;
|
||||||
|
nodeId?: string;
|
||||||
|
quorumHealthy?: boolean;
|
||||||
|
majorityHealthy?: boolean;
|
||||||
|
peers?: IClusterPeerHealth[];
|
||||||
|
drives?: IClusterDriveHealth[];
|
||||||
|
erasure?: IClusterErasureHealth;
|
||||||
|
repairs?: IClusterRepairHealth;
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import * as plugins from '../plugins.ts';
|
||||||
|
import * as data from '../data/index.ts';
|
||||||
|
|
||||||
|
export interface IReq_ListAuditEntries extends plugins.typedrequestInterfaces.implementsTR<
|
||||||
|
plugins.typedrequestInterfaces.ITypedRequest,
|
||||||
|
IReq_ListAuditEntries
|
||||||
|
> {
|
||||||
|
method: 'listAuditEntries';
|
||||||
|
request: {
|
||||||
|
identity: data.IIdentity;
|
||||||
|
limit?: number;
|
||||||
|
};
|
||||||
|
response: {
|
||||||
|
entries: data.IAuditEntry[];
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -5,3 +5,4 @@ export * from './objects.ts';
|
|||||||
export * from './config.ts';
|
export * from './config.ts';
|
||||||
export * from './credentials.ts';
|
export * from './credentials.ts';
|
||||||
export * from './policies.ts';
|
export * from './policies.ts';
|
||||||
|
export * from './audit.ts';
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
import * as plugins from '../plugins.ts';
|
import * as plugins from '../plugins.ts';
|
||||||
import * as data from '../data/index.ts';
|
import * as data from '../data/index.ts';
|
||||||
|
|
||||||
export interface IReq_GetServerStatus extends plugins.typedrequestInterfaces.implementsTR<
|
export interface IReq_GetServerStatus extends
|
||||||
|
plugins.typedrequestInterfaces.implementsTR<
|
||||||
plugins.typedrequestInterfaces.ITypedRequest,
|
plugins.typedrequestInterfaces.ITypedRequest,
|
||||||
IReq_GetServerStatus
|
IReq_GetServerStatus
|
||||||
> {
|
> {
|
||||||
method: 'getServerStatus';
|
method: 'getServerStatus';
|
||||||
request: {
|
request: {
|
||||||
identity: data.IIdentity;
|
identity: data.IIdentity;
|
||||||
@@ -14,3 +15,17 @@ export interface IReq_GetServerStatus extends plugins.typedrequestInterfaces.imp
|
|||||||
connectionInfo: data.IConnectionInfo;
|
connectionInfo: data.IConnectionInfo;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface IReq_GetClusterHealth extends
|
||||||
|
plugins.typedrequestInterfaces.implementsTR<
|
||||||
|
plugins.typedrequestInterfaces.ITypedRequest,
|
||||||
|
IReq_GetClusterHealth
|
||||||
|
> {
|
||||||
|
method: 'getClusterHealth';
|
||||||
|
request: {
|
||||||
|
identity: data.IIdentity;
|
||||||
|
};
|
||||||
|
response: {
|
||||||
|
clusterHealth: data.IClusterHealth;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@lossless.zone/objectstorage',
|
name: '@lossless.zone/objectstorage',
|
||||||
version: '1.4.2',
|
version: '1.9.0',
|
||||||
description: 'object storage server with management UI powered by smartstorage'
|
description: 'object storage server with management UI powered by smartstorage'
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-2
@@ -40,6 +40,7 @@ export interface IPoliciesState {
|
|||||||
|
|
||||||
export interface IConfigState {
|
export interface IConfigState {
|
||||||
config: interfaces.data.IServerConfig | null;
|
config: interfaces.data.IServerConfig | null;
|
||||||
|
clusterHealth: interfaces.data.IClusterHealth | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IUiState {
|
export interface IUiState {
|
||||||
@@ -58,7 +59,7 @@ export const loginStatePart = await appState.getStatePart<ILoginState>(
|
|||||||
identity: null,
|
identity: null,
|
||||||
isLoggedIn: false,
|
isLoggedIn: false,
|
||||||
},
|
},
|
||||||
'persistent',
|
'soft',
|
||||||
);
|
);
|
||||||
|
|
||||||
export const serverStatePart = await appState.getStatePart<IServerState>(
|
export const serverStatePart = await appState.getStatePart<IServerState>(
|
||||||
@@ -108,6 +109,7 @@ export const configStatePart = await appState.getStatePart<IConfigState>(
|
|||||||
'config',
|
'config',
|
||||||
{
|
{
|
||||||
config: null,
|
config: null,
|
||||||
|
clusterHealth: null,
|
||||||
},
|
},
|
||||||
'soft',
|
'soft',
|
||||||
);
|
);
|
||||||
@@ -531,7 +533,11 @@ export const fetchConfigAction = configStatePart.createAction(async (statePartAr
|
|||||||
interfaces.requests.IReq_GetServerConfig
|
interfaces.requests.IReq_GetServerConfig
|
||||||
>('/typedrequest', 'getServerConfig');
|
>('/typedrequest', 'getServerConfig');
|
||||||
const response = await typedRequest.fire({ identity: context.identity! });
|
const response = await typedRequest.fire({ identity: context.identity! });
|
||||||
return { config: response.config };
|
const clusterHealthRequest = new plugins.domtools.plugins.typedrequest.TypedRequest<
|
||||||
|
interfaces.requests.IReq_GetClusterHealth
|
||||||
|
>('/typedrequest', 'getClusterHealth');
|
||||||
|
const clusterHealthResponse = await clusterHealthRequest.fire({ identity: context.identity! });
|
||||||
|
return { config: response.config, clusterHealth: clusterHealthResponse.clusterHealth };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Failed to fetch config:', err);
|
console.error('Failed to fetch config:', err);
|
||||||
return statePartArg.getState();
|
return statePartArg.getState();
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.ts';
|
||||||
import * as appstate from './appstate.js';
|
import * as appstate from './appstate.ts';
|
||||||
import * as interfaces from '../ts_interfaces/index.js';
|
import * as interfaces from '../ts_interfaces/index.ts';
|
||||||
import type { IS3DataProvider } from '@design.estate/dees-catalog';
|
import type { IStorageDataProvider } from '@design.estate/dees-catalog';
|
||||||
|
|
||||||
const getIdentity = (): interfaces.data.IIdentity => {
|
const getIdentity = (): interfaces.data.IIdentity => {
|
||||||
return appstate.loginStatePart.getState().identity!;
|
return appstate.loginStatePart.getState().identity!;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const createDataProvider = (): IS3DataProvider => ({
|
export const createDataProvider = (): IStorageDataProvider => ({
|
||||||
async listObjects(bucket: string, prefix?: string, delimiter?: string) {
|
async listObjects(bucket: string, prefix?: string, delimiter?: string) {
|
||||||
const typedRequest = new plugins.domtools.plugins.typedrequest.TypedRequest<
|
const typedRequest = new plugins.domtools.plugins.typedrequest.TypedRequest<
|
||||||
interfaces.requests.IReq_ListObjects
|
interfaces.requests.IReq_ListObjects
|
||||||
|
|||||||
@@ -3,12 +3,12 @@ import * as appstate from '../appstate.js';
|
|||||||
import * as shared from './shared/index.js';
|
import * as shared from './shared/index.js';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
DeesElement,
|
|
||||||
customElement,
|
|
||||||
html,
|
|
||||||
state,
|
|
||||||
css,
|
css,
|
||||||
cssManager,
|
cssManager,
|
||||||
|
customElement,
|
||||||
|
DeesElement,
|
||||||
|
html,
|
||||||
|
state,
|
||||||
type TemplateResult,
|
type TemplateResult,
|
||||||
} from '@design.estate/dees-element';
|
} from '@design.estate/dees-element';
|
||||||
import { type IStatsTile } from '@design.estate/dees-catalog';
|
import { type IStatsTile } from '@design.estate/dees-catalog';
|
||||||
@@ -16,7 +16,7 @@ import { type IStatsTile } from '@design.estate/dees-catalog';
|
|||||||
@customElement('objst-view-config')
|
@customElement('objst-view-config')
|
||||||
export class ObjstViewConfig extends DeesElement {
|
export class ObjstViewConfig extends DeesElement {
|
||||||
@state()
|
@state()
|
||||||
accessor configState: appstate.IConfigState = { config: null };
|
accessor configState: appstate.IConfigState = { config: null, clusterHealth: null };
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
@@ -36,12 +36,99 @@ export class ObjstViewConfig extends DeesElement {
|
|||||||
public static styles = [
|
public static styles = [
|
||||||
cssManager.defaultStyles,
|
cssManager.defaultStyles,
|
||||||
shared.viewHostCss,
|
shared.viewHostCss,
|
||||||
|
css`
|
||||||
|
.sectionSpacer {
|
||||||
|
margin-top: 32px;
|
||||||
|
}
|
||||||
|
.infoPanel {
|
||||||
|
margin-top: 32px;
|
||||||
|
padding: 24px;
|
||||||
|
border-radius: 8px;
|
||||||
|
background: ${cssManager.bdTheme('#f5f5f5', '#1a1a2e')};
|
||||||
|
border: 1px solid ${cssManager.bdTheme('#e0e0e0', '#2a2a4a')};
|
||||||
|
}
|
||||||
|
.infoPanel h2 {
|
||||||
|
margin: 0 0 16px 0;
|
||||||
|
font-size: 18px;
|
||||||
|
font-weight: 600;
|
||||||
|
color: ${cssManager.bdTheme('#333', '#ccc')};
|
||||||
|
}
|
||||||
|
.infoPanel p {
|
||||||
|
margin: 0 0 16px 0;
|
||||||
|
font-size: 14px;
|
||||||
|
color: ${cssManager.bdTheme('#666', '#999')};
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
.infoPanel .row {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
.infoPanel .label {
|
||||||
|
min-width: 260px;
|
||||||
|
font-family: monospace;
|
||||||
|
font-weight: 500;
|
||||||
|
color: ${cssManager.bdTheme('#1565c0', '#64b5f6')};
|
||||||
|
padding: 4px 8px;
|
||||||
|
border-radius: 4px;
|
||||||
|
background: ${cssManager.bdTheme('#e3f2fd', '#1a237e30')};
|
||||||
|
}
|
||||||
|
.infoPanel .value {
|
||||||
|
color: ${cssManager.bdTheme('#666', '#999')};
|
||||||
|
margin-left: 12px;
|
||||||
|
}
|
||||||
|
.driveList {
|
||||||
|
margin-top: 16px;
|
||||||
|
}
|
||||||
|
.driveList .driveItem {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
.driveList .driveIndex {
|
||||||
|
width: 32px;
|
||||||
|
height: 32px;
|
||||||
|
border-radius: 6px;
|
||||||
|
background: ${cssManager.bdTheme('#e8eaf6', '#1a237e40')};
|
||||||
|
color: ${cssManager.bdTheme('#3f51b5', '#7986cb')};
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 13px;
|
||||||
|
margin-right: 12px;
|
||||||
|
}
|
||||||
|
.driveList .drivePath {
|
||||||
|
font-family: monospace;
|
||||||
|
color: ${cssManager.bdTheme('#333', '#e0e0e0')};
|
||||||
|
padding: 6px 12px;
|
||||||
|
border-radius: 4px;
|
||||||
|
background: ${cssManager.bdTheme('#e8e8e8', '#252540')};
|
||||||
|
}
|
||||||
|
.driveList .driveStatus {
|
||||||
|
margin-left: 12px;
|
||||||
|
padding: 4px 8px;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-size: 12px;
|
||||||
|
font-weight: 600;
|
||||||
|
color: ${cssManager.bdTheme('#1b5e20', '#a5d6a7')};
|
||||||
|
background: ${cssManager.bdTheme('#e8f5e9', '#1b5e2030')};
|
||||||
|
}
|
||||||
|
.driveList .driveMeta {
|
||||||
|
margin-left: 12px;
|
||||||
|
color: ${cssManager.bdTheme('#666', '#999')};
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
`,
|
||||||
];
|
];
|
||||||
|
|
||||||
public render(): TemplateResult {
|
public render(): TemplateResult {
|
||||||
const config = this.configState.config;
|
const config = this.configState.config;
|
||||||
|
const clusterHealth = this.configState.clusterHealth;
|
||||||
|
|
||||||
const tiles: IStatsTile[] = [
|
const serverTiles: IStatsTile[] = [
|
||||||
{
|
{
|
||||||
id: 'objstPort',
|
id: 'objstPort',
|
||||||
title: 'Storage API Port',
|
title: 'Storage API Port',
|
||||||
@@ -92,20 +179,240 @@ export class ObjstViewConfig extends DeesElement {
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
return html`
|
const clusterEnabled = clusterHealth?.enabled ?? config?.clusterEnabled ?? false;
|
||||||
<objst-sectionheading>Configuration</objst-sectionheading>
|
const clusterTiles: IStatsTile[] = [
|
||||||
<dees-statsgrid
|
|
||||||
.tiles=${tiles}
|
|
||||||
.gridActions=${[
|
|
||||||
{
|
{
|
||||||
|
id: 'clusterStatus',
|
||||||
|
title: 'Cluster Status',
|
||||||
|
value: clusterEnabled ? 'Enabled' : 'Disabled',
|
||||||
|
type: 'text',
|
||||||
|
icon: 'lucide:network',
|
||||||
|
color: clusterEnabled ? '#4caf50' : '#ff9800',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'nodeId',
|
||||||
|
title: 'Node ID',
|
||||||
|
value: clusterHealth?.nodeId || config?.clusterNodeId || '(auto)',
|
||||||
|
type: 'text',
|
||||||
|
icon: 'lucide:fingerprint',
|
||||||
|
color: '#607d8b',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'quicPort',
|
||||||
|
title: 'QUIC Port',
|
||||||
|
value: config?.clusterQuicPort ?? 4433,
|
||||||
|
type: 'number',
|
||||||
|
icon: 'lucide:radio',
|
||||||
|
color: '#00bcd4',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'seedNodes',
|
||||||
|
title: 'Seed Nodes',
|
||||||
|
value: config?.clusterSeedNodes?.length ?? 0,
|
||||||
|
type: 'number',
|
||||||
|
icon: 'lucide:gitBranch',
|
||||||
|
color: '#3f51b5',
|
||||||
|
description: config?.clusterSeedNodes?.length
|
||||||
|
? config.clusterSeedNodes.join(', ')
|
||||||
|
: 'No seed nodes configured',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'heartbeatInterval',
|
||||||
|
title: 'Heartbeat Interval',
|
||||||
|
value: `${config?.clusterHeartbeatIntervalMs ?? 5000}ms`,
|
||||||
|
type: 'text',
|
||||||
|
icon: 'lucide:heartPulse',
|
||||||
|
color: '#e91e63',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'heartbeatTimeout',
|
||||||
|
title: 'Heartbeat Timeout',
|
||||||
|
value: `${config?.clusterHeartbeatTimeoutMs ?? 30000}ms`,
|
||||||
|
type: 'text',
|
||||||
|
icon: 'lucide:timer',
|
||||||
|
color: '#ff5722',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'quorum',
|
||||||
|
title: 'Quorum',
|
||||||
|
value: clusterHealth?.enabled
|
||||||
|
? clusterHealth.quorumHealthy ? 'Healthy' : 'Degraded'
|
||||||
|
: 'Standalone',
|
||||||
|
type: 'text',
|
||||||
|
icon: 'lucide:activity',
|
||||||
|
color: clusterHealth?.quorumHealthy ? '#4caf50' : '#ff9800',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'peers',
|
||||||
|
title: 'Peers',
|
||||||
|
value: clusterHealth?.peers?.length ?? 0,
|
||||||
|
type: 'number',
|
||||||
|
icon: 'lucide:share2',
|
||||||
|
color: '#3f51b5',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const erasureTiles: IStatsTile[] = [
|
||||||
|
{
|
||||||
|
id: 'dataShards',
|
||||||
|
title: 'Data Shards',
|
||||||
|
value: clusterHealth?.erasure?.dataShards ?? config?.erasureDataShards ?? 4,
|
||||||
|
type: 'number',
|
||||||
|
icon: 'lucide:layers',
|
||||||
|
color: '#2196f3',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'parityShards',
|
||||||
|
title: 'Parity Shards',
|
||||||
|
value: clusterHealth?.erasure?.parityShards ?? config?.erasureParityShards ?? 2,
|
||||||
|
type: 'number',
|
||||||
|
icon: 'lucide:shieldCheck',
|
||||||
|
color: '#4caf50',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'chunkSize',
|
||||||
|
title: 'Chunk Size',
|
||||||
|
value: this.formatBytes(
|
||||||
|
clusterHealth?.erasure?.chunkSizeBytes ?? config?.erasureChunkSizeBytes ?? 4194304,
|
||||||
|
),
|
||||||
|
type: 'text',
|
||||||
|
icon: 'lucide:puzzle',
|
||||||
|
color: '#9c27b0',
|
||||||
|
description: `${clusterHealth?.erasure?.dataShards ?? config?.erasureDataShards ?? 4}+${
|
||||||
|
clusterHealth?.erasure?.parityShards ?? config?.erasureParityShards ?? 2
|
||||||
|
}`,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const drivePaths = clusterHealth?.drives?.length
|
||||||
|
? clusterHealth.drives.map((drive) => drive.path)
|
||||||
|
: config?.drivePaths?.length
|
||||||
|
? config.drivePaths
|
||||||
|
: config?.storageDirectory
|
||||||
|
? [config.storageDirectory]
|
||||||
|
: ['/data'];
|
||||||
|
|
||||||
|
const driveTiles: IStatsTile[] = [
|
||||||
|
{
|
||||||
|
id: 'driveCount',
|
||||||
|
title: 'Drive Count',
|
||||||
|
value: clusterHealth?.drives?.length ?? drivePaths.length,
|
||||||
|
type: 'number',
|
||||||
|
icon: 'lucide:hardDrive',
|
||||||
|
color: '#3f51b5',
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const refreshAction = {
|
||||||
name: 'Refresh',
|
name: 'Refresh',
|
||||||
iconName: 'lucide:refreshCw',
|
iconName: 'lucide:refreshCw',
|
||||||
action: async () => {
|
action: async () => {
|
||||||
await appstate.configStatePart.dispatchAction(appstate.fetchConfigAction, null);
|
await appstate.configStatePart.dispatchAction(appstate.fetchConfigAction, null);
|
||||||
},
|
},
|
||||||
},
|
};
|
||||||
]}
|
|
||||||
|
return html`
|
||||||
|
<objst-sectionheading>Server Configuration</objst-sectionheading>
|
||||||
|
<dees-statsgrid
|
||||||
|
.tiles="${serverTiles}"
|
||||||
|
.gridActions="${[refreshAction]}"
|
||||||
></dees-statsgrid>
|
></dees-statsgrid>
|
||||||
|
|
||||||
|
<div class="sectionSpacer">
|
||||||
|
<objst-sectionheading>Cluster Configuration</objst-sectionheading>
|
||||||
|
</div>
|
||||||
|
<dees-statsgrid .tiles="${clusterTiles}"></dees-statsgrid>
|
||||||
|
|
||||||
|
${clusterEnabled
|
||||||
|
? html`
|
||||||
|
<div class="sectionSpacer">
|
||||||
|
<objst-sectionheading>Erasure Coding</objst-sectionheading>
|
||||||
|
</div>
|
||||||
|
<dees-statsgrid .tiles="${erasureTiles}"></dees-statsgrid>
|
||||||
|
`
|
||||||
|
: ''}
|
||||||
|
|
||||||
|
<div class="sectionSpacer">
|
||||||
|
<objst-sectionheading>Storage Drives</objst-sectionheading>
|
||||||
|
</div>
|
||||||
|
<dees-statsgrid .tiles="${driveTiles}"></dees-statsgrid>
|
||||||
|
<div class="driveList">
|
||||||
|
${(clusterHealth?.drives?.length
|
||||||
|
? clusterHealth.drives
|
||||||
|
: drivePaths.map((path, index) => ({ path, index, status: 'configured' }))).map((
|
||||||
|
drive,
|
||||||
|
i,
|
||||||
|
) =>
|
||||||
|
html`
|
||||||
|
<div class="driveItem">
|
||||||
|
<div class="driveIndex">${i + 1}</div>
|
||||||
|
<span class="drivePath">${drive.path}</span>
|
||||||
|
<span class="driveStatus">${drive.status}</span>
|
||||||
|
${drive.usedBytes !== undefined && drive.totalBytes !== undefined
|
||||||
|
? html`
|
||||||
|
<span class="driveMeta">${this.formatBytes(drive.usedBytes)} / ${this
|
||||||
|
.formatBytes(drive.totalBytes)}</span>
|
||||||
|
`
|
||||||
|
: ''}
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="infoPanel">
|
||||||
|
<h2>Configuration Reference</h2>
|
||||||
|
<p>
|
||||||
|
Cluster and drive settings are applied at server startup. To change them, set the environment
|
||||||
|
variables and restart the server.
|
||||||
|
</p>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_CLUSTER_ENABLED</span><span class="value"
|
||||||
|
>Enable clustering (true/false)</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_CLUSTER_NODE_ID</span><span class="value"
|
||||||
|
>Unique node identifier</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_CLUSTER_QUIC_PORT</span><span class="value"
|
||||||
|
>QUIC transport port (default: 4433)</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_CLUSTER_SEED_NODES</span><span class="value"
|
||||||
|
>Comma-separated seed node addresses</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_DRIVE_PATHS</span><span class="value"
|
||||||
|
>Comma-separated drive mount paths</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_ERASURE_DATA_SHARDS</span><span class="value"
|
||||||
|
>Data shards for erasure coding (default: 4)</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_ERASURE_PARITY_SHARDS</span><span class="value"
|
||||||
|
>Parity shards for erasure coding (default: 2)</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_ERASURE_CHUNK_SIZE</span><span class="value"
|
||||||
|
>Chunk size in bytes (default: 4194304)</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_HEARTBEAT_INTERVAL_MS</span><span class="value"
|
||||||
|
>Heartbeat interval in ms (default: 5000)</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="label">OBJST_HEARTBEAT_TIMEOUT_MS</span><span class="value"
|
||||||
|
>Heartbeat timeout in ms (default: 30000)</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
`;
|
`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private formatBytes(bytes: number): string {
|
||||||
|
if (bytes === 0) return '0 B';
|
||||||
|
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||||
|
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
||||||
|
return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -123,10 +123,10 @@ export class ObjstViewObjects extends DeesElement {
|
|||||||
${this.selectedBucket
|
${this.selectedBucket
|
||||||
? html`
|
? html`
|
||||||
<div class="browser-container">
|
<div class="browser-container">
|
||||||
<dees-s3-browser
|
<dees-storage-browser
|
||||||
.dataProvider=${this.dataProvider}
|
.dataProvider=${this.dataProvider}
|
||||||
.bucketName=${this.selectedBucket}
|
.bucketName=${this.selectedBucket}
|
||||||
></dees-s3-browser>
|
></dees-storage-browser>
|
||||||
</div>
|
</div>
|
||||||
`
|
`
|
||||||
: html`
|
: html`
|
||||||
|
|||||||
Reference in New Issue
Block a user