feat: add corestore service
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
node_modules
|
||||
.git
|
||||
.nogit
|
||||
.pnpm-store
|
||||
dist
|
||||
dist_*
|
||||
coverage
|
||||
*.log
|
||||
@@ -0,0 +1,7 @@
|
||||
node_modules/
|
||||
dist/
|
||||
dist_*/
|
||||
.nogit/
|
||||
.pnpm-store/
|
||||
coverage/
|
||||
*.log
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"gitzone": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "code.foss.global",
|
||||
"gitscope": "serve.zone",
|
||||
"gitrepo": "corestore",
|
||||
"gitRemote": "ssh://git@code.foss.global:29419/serve.zone/corestore.git",
|
||||
"description": "Node-local database and S3 storage provider for serve.zone workloads.",
|
||||
"npmPackagename": "@serve.zone/corestore",
|
||||
"license": "MIT",
|
||||
"projectDomain": "serve.zone"
|
||||
}
|
||||
}
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# gitzone dockerfile_service
|
||||
## STAGE 1 // BUILD
|
||||
FROM code.foss.global/host.today/ht-docker-node:lts AS build
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json pnpm-lock.yaml ./
|
||||
RUN pnpm config set store-dir .pnpm-store
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
||||
COPY . ./
|
||||
RUN pnpm run build
|
||||
RUN rm -rf .pnpm-store
|
||||
RUN pnpm prune --prod
|
||||
|
||||
## STAGE 2 // PRODUCTION
|
||||
FROM code.foss.global/host.today/ht-docker-node:alpine-node AS production
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV CORESTORE_DATA_DIR=/data/corestore
|
||||
|
||||
COPY --from=build /app/package.json ./package.json
|
||||
COPY --from=build /app/node_modules ./node_modules
|
||||
COPY --from=build /app/cli.js ./cli.js
|
||||
COPY --from=build /app/dist_ts ./dist_ts
|
||||
|
||||
LABEL org.opencontainers.image.title="corestore" \
|
||||
org.opencontainers.image.description="serve.zone node-local database and S3 storage provider" \
|
||||
org.opencontainers.image.source="https://code.foss.global/serve.zone/corestore"
|
||||
|
||||
VOLUME ["/data/corestore"]
|
||||
EXPOSE 3000 9000 27017
|
||||
CMD ["node", "cli.js"]
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env node
|
||||
process.env.CLI_CALL = 'true';
|
||||
const cliTool = await import('./dist_ts/index.js');
|
||||
await cliTool.runCli();
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env node
|
||||
process.env.CLI_CALL = 'true';
|
||||
|
||||
import * as tsrun from '@git.zone/tsrun';
|
||||
tsrun.runPath('./ts/index.ts', import.meta.url);
|
||||
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"name": "@serve.zone/corestore",
|
||||
"version": "1.0.0",
|
||||
"description": "Node-local database and S3 storage provider for serve.zone workloads.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "(tstest test/)",
|
||||
"start": "node cli.js",
|
||||
"startTs": "node cli.ts.js",
|
||||
"watch": "tswatch service",
|
||||
"build": "tsbuild tsfolders --allowimplicitany",
|
||||
"build:docker": "tsdocker build --verbose",
|
||||
"release:docker": "tsdocker push --verbose"
|
||||
},
|
||||
"bin": {
|
||||
"corestore": "cli.js"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "ssh://git@code.foss.global:29419/serve.zone/corestore.git"
|
||||
},
|
||||
"keywords": [
|
||||
"serve.zone",
|
||||
"corestore",
|
||||
"smartstorage",
|
||||
"smartdb",
|
||||
"s3",
|
||||
"mongodb",
|
||||
"database",
|
||||
"storage",
|
||||
"docker"
|
||||
],
|
||||
"author": "Lossless GmbH",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^4.4.0",
|
||||
"@git.zone/tsdocker": "^2.2.4",
|
||||
"@git.zone/tsrun": "^2.0.2",
|
||||
"@git.zone/tstest": "^3.6.3",
|
||||
"@git.zone/tswatch": "^3.3.2",
|
||||
"@types/node": "^25.6.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@push.rocks/projectinfo": "^5.1.0",
|
||||
"@push.rocks/smartdb": "^2.10.0",
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@push.rocks/smartstorage": "^6.5.1"
|
||||
},
|
||||
"private": false,
|
||||
"files": [
|
||||
"ts/**/*",
|
||||
"dist/**/*",
|
||||
"dist_*/**/*",
|
||||
"dist_ts/**/*",
|
||||
"assets/**/*",
|
||||
"cli.js",
|
||||
".smartconfig.json",
|
||||
"readme.md"
|
||||
],
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
],
|
||||
"packageManager": "pnpm@10.28.2"
|
||||
}
|
||||
Generated
+9167
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,93 @@
|
||||
# @serve.zone/corestore
|
||||
|
||||
`corestore` is the node-local serve.zone storage provider. It runs one container that starts:
|
||||
|
||||
- `@push.rocks/smartdb` as a MongoDB-compatible database endpoint on port `27017`.
|
||||
- `@push.rocks/smartstorage` as an S3-compatible object-storage endpoint on port `9000`.
|
||||
- A small control API on port `3000` for Coreflow provisioning.
|
||||
|
||||
## Purpose
|
||||
|
||||
Coreflow can run `corestore` on every node and provision per-service resources on the node that hosts a workload requiring `database` or `objectstorage`.
|
||||
|
||||
The first implementation exposes the provider container and provisioning API. Coreflow should call the control API when reconciling platform bindings, then inject the returned environment variables into the workload secret.
|
||||
|
||||
## Runtime
|
||||
|
||||
```bash
|
||||
pnpm install
|
||||
pnpm build
|
||||
node cli.js
|
||||
```
|
||||
|
||||
Default ports:
|
||||
|
||||
| Service | Port | Purpose |
|
||||
| ------- | ---- | ------- |
|
||||
| Control API | `3000` | Provisioning, deprovisioning, health, metrics |
|
||||
| S3 | `9000` | S3-compatible API from smartstorage |
|
||||
| DB | `27017` | MongoDB wire protocol from smartdb |
|
||||
|
||||
Default data directory: `/data/corestore`.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Env var | Default | Purpose |
|
||||
| ------- | ------- | ------- |
|
||||
| `CORESTORE_DATA_DIR` | `/data/corestore` | Persistent data root |
|
||||
| `CORESTORE_BIND_ADDRESS` | `0.0.0.0` | Bind address for all endpoints |
|
||||
| `CORESTORE_PUBLIC_HOST` | `corestore` | Hostname injected into service credentials |
|
||||
| `CORESTORE_CONTROL_PORT` | `3000` | Control API port |
|
||||
| `CORESTORE_S3_PORT` | `9000` | S3 endpoint port |
|
||||
| `CORESTORE_DB_PORT` | `27017` | Mongo-compatible DB endpoint port |
|
||||
| `CORESTORE_REGION` | `us-east-1` | S3 region |
|
||||
| `CORESTORE_API_TOKEN` | unset | Optional bearer token for mutating/read-sensitive control APIs |
|
||||
| `CORESTORE_MASTER_SECRET` | generated and persisted | Seed for deterministic tenant credentials |
|
||||
|
||||
When Coreflow creates the global `corestore` service, it forwards its own `CORESTORE_API_TOKEN` environment variable into the service. Set the same value on Coreflow to protect provisioning APIs from workload containers on the same overlay network.
|
||||
|
||||
## Control API
|
||||
|
||||
Health is unauthenticated:
|
||||
|
||||
```bash
|
||||
curl http://corestore:3000/health
|
||||
```
|
||||
|
||||
Provision per-service DB and S3 resources:
|
||||
|
||||
```bash
|
||||
curl -X POST http://corestore:3000/resources/provision \
|
||||
-H 'content-type: application/json' \
|
||||
-H 'authorization: Bearer <CORESTORE_API_TOKEN>' \
|
||||
-d '{"serviceId":"svc-123","serviceName":"api","capabilities":["database","objectstorage"]}'
|
||||
```
|
||||
|
||||
The response contains service-specific env vars such as `MONGODB_URI`, `S3_BUCKET`, `AWS_ACCESS_KEY_ID`, and `AWS_ENDPOINT_URL`.
|
||||
|
||||
Deprovision a service:
|
||||
|
||||
```bash
|
||||
curl -X POST http://corestore:3000/resources/deprovision \
|
||||
-H 'content-type: application/json' \
|
||||
-H 'authorization: Bearer <CORESTORE_API_TOKEN>' \
|
||||
-d '{"serviceId":"svc-123"}'
|
||||
```
|
||||
|
||||
## Docker
|
||||
|
||||
```bash
|
||||
pnpm run build:docker
|
||||
```
|
||||
|
||||
The image exposes `3000`, `9000`, and `27017` and stores all runtime data under `/data/corestore`.
|
||||
|
||||
## Coreflow Integration Notes
|
||||
|
||||
The intended cluster behavior is:
|
||||
|
||||
- deploy `corestore` as a node-local/global service so every workload node has a local storage provider;
|
||||
- provision `database` and `objectstorage` bindings through `/resources/provision`;
|
||||
- merge the returned env vars into the workload Docker secret before service creation;
|
||||
- mark Cloudly platform bindings `ready` with endpoint metadata and credential env refs;
|
||||
- deprovision resources when the service binding or workload is deleted.
|
||||
@@ -0,0 +1,586 @@
|
||||
import * as plugins from './corestore.plugins.js';
|
||||
import type * as interfaces from './corestore.interfaces.js';
|
||||
|
||||
export interface ICoreStoreOptions {
|
||||
dataDir?: string;
|
||||
bindAddress?: string;
|
||||
publicHost?: string;
|
||||
controlPort?: number;
|
||||
s3Port?: number;
|
||||
dbPort?: number;
|
||||
region?: string;
|
||||
apiToken?: string;
|
||||
}
|
||||
|
||||
type TResolvedCoreStoreOptions = Required<Omit<ICoreStoreOptions, 'apiToken'>> & {
|
||||
apiToken?: string;
|
||||
};
|
||||
|
||||
export class CoreStore {
|
||||
private options: TResolvedCoreStoreOptions;
|
||||
private smartStorage: plugins.smartstorage.SmartStorage | null = null;
|
||||
private smartDb: plugins.smartdb.SmartdbServer | null = null;
|
||||
private controlServer: plugins.http.Server | null = null;
|
||||
private manifest: interfaces.ICoreStoreManifest = { version: 1, services: {} };
|
||||
private secretFile: interfaces.ICoreStoreSecretFile | null = null;
|
||||
|
||||
constructor(optionsArg: ICoreStoreOptions = {}) {
|
||||
this.options = {
|
||||
dataDir: optionsArg.dataDir || process.env.CORESTORE_DATA_DIR || '/data/corestore',
|
||||
bindAddress: optionsArg.bindAddress || process.env.CORESTORE_BIND_ADDRESS || '0.0.0.0',
|
||||
publicHost: optionsArg.publicHost || process.env.CORESTORE_PUBLIC_HOST || 'corestore',
|
||||
controlPort: optionsArg.controlPort || this.getNumberEnv('CORESTORE_CONTROL_PORT', 3000),
|
||||
s3Port: optionsArg.s3Port || this.getNumberEnv('CORESTORE_S3_PORT', 9000),
|
||||
dbPort: optionsArg.dbPort || this.getNumberEnv('CORESTORE_DB_PORT', 27017),
|
||||
region: optionsArg.region || process.env.CORESTORE_REGION || 'us-east-1',
|
||||
...(optionsArg.apiToken || process.env.CORESTORE_API_TOKEN
|
||||
? { apiToken: optionsArg.apiToken || process.env.CORESTORE_API_TOKEN }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
public async start() {
|
||||
await this.ensureDirectories();
|
||||
this.secretFile = await this.loadOrCreateSecretFile();
|
||||
this.manifest = await this.loadManifest();
|
||||
await this.startSmartStorage();
|
||||
await this.startSmartDb();
|
||||
await this.startControlApi();
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
if (this.controlServer) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
this.controlServer!.close((errorArg) => (errorArg ? reject(errorArg) : resolve()));
|
||||
});
|
||||
this.controlServer = null;
|
||||
}
|
||||
if (this.smartStorage) {
|
||||
await this.smartStorage.stop();
|
||||
this.smartStorage = null;
|
||||
}
|
||||
if (this.smartDb) {
|
||||
await this.smartDb.stop();
|
||||
this.smartDb = null;
|
||||
}
|
||||
}
|
||||
|
||||
private getNumberEnv(envNameArg: string, defaultArg: number) {
|
||||
const value = process.env[envNameArg];
|
||||
const parsedValue = value ? Number(value) : NaN;
|
||||
return Number.isInteger(parsedValue) && parsedValue > 0 ? parsedValue : defaultArg;
|
||||
}
|
||||
|
||||
private getStorageDir() {
|
||||
return plugins.path.join(this.options.dataDir, 'smartstorage');
|
||||
}
|
||||
|
||||
private getDbDir() {
|
||||
return plugins.path.join(this.options.dataDir, 'smartdb');
|
||||
}
|
||||
|
||||
private getManifestPath() {
|
||||
return plugins.path.join(this.options.dataDir, 'corestore-manifest.json');
|
||||
}
|
||||
|
||||
private getSecretPath() {
|
||||
return plugins.path.join(this.options.dataDir, 'corestore-secret.json');
|
||||
}
|
||||
|
||||
private async ensureDirectories() {
|
||||
await plugins.fs.mkdir(this.options.dataDir, { recursive: true });
|
||||
await plugins.fs.mkdir(this.getStorageDir(), { recursive: true });
|
||||
await plugins.fs.mkdir(this.getDbDir(), { recursive: true });
|
||||
}
|
||||
|
||||
private async loadOrCreateSecretFile(): Promise<interfaces.ICoreStoreSecretFile> {
|
||||
const envMasterSecret = process.env.CORESTORE_MASTER_SECRET;
|
||||
const existingSecretFile = await this.readJsonFile<interfaces.ICoreStoreSecretFile>(this.getSecretPath());
|
||||
const masterSecret = envMasterSecret || existingSecretFile?.masterSecret || plugins.crypto.randomBytes(32).toString('hex');
|
||||
const secretFile: interfaces.ICoreStoreSecretFile = {
|
||||
masterSecret,
|
||||
dbRootPassword:
|
||||
process.env.CORESTORE_DB_ROOT_PASSWORD ||
|
||||
existingSecretFile?.dbRootPassword ||
|
||||
this.deriveHex(masterSecret, 'db-root-password', 48),
|
||||
s3AdminAccessKeyId:
|
||||
process.env.CORESTORE_S3_ADMIN_ACCESS_KEY_ID ||
|
||||
existingSecretFile?.s3AdminAccessKeyId ||
|
||||
`CSADMIN${this.deriveHex(masterSecret, 's3-admin-access-key', 20).toUpperCase()}`,
|
||||
s3AdminSecretAccessKey:
|
||||
process.env.CORESTORE_S3_ADMIN_SECRET_ACCESS_KEY ||
|
||||
existingSecretFile?.s3AdminSecretAccessKey ||
|
||||
this.deriveHex(masterSecret, 's3-admin-secret-key', 64),
|
||||
};
|
||||
await this.writeJsonFile(this.getSecretPath(), secretFile);
|
||||
return secretFile;
|
||||
}
|
||||
|
||||
private async loadManifest(): Promise<interfaces.ICoreStoreManifest> {
|
||||
return (await this.readJsonFile<interfaces.ICoreStoreManifest>(this.getManifestPath())) || {
|
||||
version: 1,
|
||||
services: {},
|
||||
};
|
||||
}
|
||||
|
||||
private async saveManifest() {
|
||||
await this.writeJsonFile(this.getManifestPath(), this.manifest);
|
||||
}
|
||||
|
||||
private async readJsonFile<T>(filePathArg: string): Promise<T | null> {
|
||||
try {
|
||||
return JSON.parse(await plugins.fs.readFile(filePathArg, 'utf8')) as T;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async writeJsonFile(filePathArg: string, dataArg: unknown) {
|
||||
await plugins.fs.writeFile(`${filePathArg}.tmp`, `${JSON.stringify(dataArg, null, 2)}\n`);
|
||||
await plugins.fs.rename(`${filePathArg}.tmp`, filePathArg);
|
||||
}
|
||||
|
||||
private deriveHex(secretArg: string, labelArg: string, lengthArg: number) {
|
||||
return plugins.crypto.createHmac('sha256', secretArg).update(labelArg).digest('hex').slice(0, lengthArg);
|
||||
}
|
||||
|
||||
private async startSmartStorage() {
|
||||
if (!this.secretFile) {
|
||||
throw new Error('corestore secret file must be loaded before smartstorage starts');
|
||||
}
|
||||
this.smartStorage = await plugins.smartstorage.SmartStorage.createAndStart({
|
||||
server: {
|
||||
address: this.options.bindAddress,
|
||||
port: this.options.s3Port,
|
||||
region: this.options.region,
|
||||
silent: true,
|
||||
},
|
||||
storage: {
|
||||
directory: this.getStorageDir(),
|
||||
cleanSlate: false,
|
||||
},
|
||||
auth: {
|
||||
enabled: true,
|
||||
credentials: [
|
||||
{
|
||||
accessKeyId: this.secretFile.s3AdminAccessKeyId,
|
||||
secretAccessKey: this.secretFile.s3AdminSecretAccessKey,
|
||||
region: this.options.region,
|
||||
},
|
||||
],
|
||||
},
|
||||
logging: {
|
||||
enabled: process.env.CORESTORE_VERBOSE === 'true',
|
||||
level: process.env.CORESTORE_VERBOSE === 'true' ? 'info' : 'error',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private async startSmartDb() {
|
||||
if (!this.secretFile) {
|
||||
throw new Error('corestore secret file must be loaded before smartdb starts');
|
||||
}
|
||||
this.smartDb = new plugins.smartdb.SmartdbServer({
|
||||
host: this.options.bindAddress,
|
||||
port: this.options.dbPort,
|
||||
storage: 'file',
|
||||
storagePath: this.getDbDir(),
|
||||
auth: {
|
||||
enabled: true,
|
||||
usersPath: plugins.path.join(this.getDbDir(), 'smartdb-users.json'),
|
||||
users: [
|
||||
{
|
||||
username: process.env.CORESTORE_DB_ROOT_USER || 'corestore_root',
|
||||
password: this.secretFile.dbRootPassword,
|
||||
database: 'admin',
|
||||
roles: ['root'],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await this.smartDb.start();
|
||||
}
|
||||
|
||||
private async startControlApi() {
|
||||
this.controlServer = plugins.http.createServer(async (reqArg, resArg) => {
|
||||
try {
|
||||
await this.handleControlRequest(reqArg, resArg);
|
||||
} catch (error) {
|
||||
this.sendJson(resArg, 500, {
|
||||
ok: false,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
this.controlServer!.listen(this.options.controlPort, this.options.bindAddress, resolve);
|
||||
});
|
||||
}
|
||||
|
||||
private async handleControlRequest(
|
||||
reqArg: plugins.http.IncomingMessage,
|
||||
resArg: plugins.http.ServerResponse,
|
||||
) {
|
||||
const url = new URL(reqArg.url || '/', `http://${reqArg.headers.host || 'localhost'}`);
|
||||
const method = reqArg.method || 'GET';
|
||||
|
||||
if (method === 'GET' && url.pathname === '/health') {
|
||||
this.sendJson(resArg, 200, await this.getHealth());
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.isAuthorized(reqArg)) {
|
||||
this.sendJson(resArg, 401, { ok: false, error: 'unauthorized' });
|
||||
return;
|
||||
}
|
||||
|
||||
if (method === 'GET' && url.pathname === '/metrics') {
|
||||
this.sendJson(resArg, 200, await this.getMetrics());
|
||||
return;
|
||||
}
|
||||
|
||||
if (method === 'GET' && url.pathname === '/resources') {
|
||||
this.sendJson(resArg, 200, {
|
||||
services: Object.values(this.manifest.services).map((serviceArg) => ({
|
||||
serviceId: serviceArg.serviceId,
|
||||
serviceName: serviceArg.serviceName,
|
||||
resources: Object.values(serviceArg.resources).map((resourceArg) => ({
|
||||
capability: resourceArg!.capability,
|
||||
provider: resourceArg!.provider,
|
||||
resourceName: resourceArg!.resourceName,
|
||||
createdAt: resourceArg!.createdAt,
|
||||
})),
|
||||
updatedAt: serviceArg.updatedAt,
|
||||
})),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (method === 'POST' && url.pathname === '/resources/provision') {
|
||||
const body = await this.readRequestBody<interfaces.ICoreStoreProvisionRequest>(reqArg);
|
||||
this.sendJson(resArg, 200, await this.provisionForService(body));
|
||||
return;
|
||||
}
|
||||
|
||||
if (method === 'POST' && url.pathname === '/resources/deprovision') {
|
||||
const body = await this.readRequestBody<interfaces.ICoreStoreDeprovisionRequest>(reqArg);
|
||||
this.sendJson(resArg, 200, await this.deprovisionForService(body));
|
||||
return;
|
||||
}
|
||||
|
||||
this.sendJson(resArg, 404, { ok: false, error: 'not found' });
|
||||
}
|
||||
|
||||
private isAuthorized(reqArg: plugins.http.IncomingMessage) {
|
||||
if (!this.options.apiToken) {
|
||||
return true;
|
||||
}
|
||||
const authHeader = reqArg.headers.authorization;
|
||||
const tokenHeader = reqArg.headers['x-corestore-token'];
|
||||
return (
|
||||
authHeader === `Bearer ${this.options.apiToken}` ||
|
||||
tokenHeader === this.options.apiToken ||
|
||||
(Array.isArray(tokenHeader) && tokenHeader.includes(this.options.apiToken))
|
||||
);
|
||||
}
|
||||
|
||||
private async readRequestBody<T>(reqArg: plugins.http.IncomingMessage): Promise<T> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of reqArg) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
const rawBody = Buffer.concat(chunks).toString('utf8').trim();
|
||||
if (!rawBody) {
|
||||
return {} as T;
|
||||
}
|
||||
return JSON.parse(rawBody) as T;
|
||||
}
|
||||
|
||||
private sendJson(resArg: plugins.http.ServerResponse, statusCodeArg: number, dataArg: unknown) {
|
||||
const body = JSON.stringify(dataArg, null, 2);
|
||||
resArg.writeHead(statusCodeArg, {
|
||||
'content-type': 'application/json; charset=utf-8',
|
||||
'content-length': Buffer.byteLength(body),
|
||||
});
|
||||
resArg.end(body);
|
||||
}
|
||||
|
||||
private async getHealth() {
|
||||
const [dbHealth, storageHealth] = await Promise.all([
|
||||
this.smartDb?.getHealth(),
|
||||
this.smartStorage?.getHealth(),
|
||||
]);
|
||||
return {
|
||||
ok: Boolean(dbHealth?.running && storageHealth?.ok),
|
||||
control: {
|
||||
port: this.options.controlPort,
|
||||
},
|
||||
database: {
|
||||
host: this.options.publicHost,
|
||||
port: this.options.dbPort,
|
||||
health: dbHealth,
|
||||
},
|
||||
objectstorage: {
|
||||
host: this.options.publicHost,
|
||||
port: this.options.s3Port,
|
||||
health: storageHealth,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async getMetrics() {
|
||||
const [dbMetrics, storageMetrics] = await Promise.all([
|
||||
this.smartDb?.getMetrics(),
|
||||
this.smartStorage?.getMetrics(),
|
||||
]);
|
||||
return {
|
||||
database: dbMetrics,
|
||||
objectstorage: storageMetrics,
|
||||
};
|
||||
}
|
||||
|
||||
private normalizeCapabilities(
|
||||
requestArg: interfaces.ICoreStoreProvisionRequest,
|
||||
): interfaces.TCoreStoreCapability[] {
|
||||
const capabilities = new Set<interfaces.TCoreStoreCapability>();
|
||||
for (const capability of requestArg.capabilities || []) {
|
||||
capabilities.add(capability);
|
||||
}
|
||||
if (requestArg.database || requestArg.db) {
|
||||
capabilities.add('database');
|
||||
}
|
||||
if (requestArg.objectstorage || requestArg.s3) {
|
||||
capabilities.add('objectstorage');
|
||||
}
|
||||
if (capabilities.size === 0) {
|
||||
throw new Error('No corestore capabilities requested');
|
||||
}
|
||||
return Array.from(capabilities);
|
||||
}
|
||||
|
||||
private async provisionForService(
|
||||
requestArg: interfaces.ICoreStoreProvisionRequest,
|
||||
): Promise<interfaces.ICoreStoreProvisionResponse> {
|
||||
if (!requestArg.serviceId) {
|
||||
throw new Error('serviceId is required');
|
||||
}
|
||||
const capabilities = this.normalizeCapabilities(requestArg);
|
||||
const now = Date.now();
|
||||
const serviceEntry = this.manifest.services[requestArg.serviceId] || {
|
||||
serviceId: requestArg.serviceId,
|
||||
serviceName: requestArg.serviceName,
|
||||
resources: {},
|
||||
env: {},
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
serviceEntry.serviceName = requestArg.serviceName || serviceEntry.serviceName;
|
||||
|
||||
for (const capability of capabilities) {
|
||||
if (serviceEntry.resources[capability]) {
|
||||
continue;
|
||||
}
|
||||
const resource = capability === 'database'
|
||||
? await this.provisionDatabase(requestArg)
|
||||
: await this.provisionObjectStorage(requestArg);
|
||||
serviceEntry.resources[capability] = resource;
|
||||
serviceEntry.env = {
|
||||
...serviceEntry.env,
|
||||
...resource.env,
|
||||
};
|
||||
}
|
||||
|
||||
serviceEntry.updatedAt = Date.now();
|
||||
this.manifest.services[requestArg.serviceId] = serviceEntry;
|
||||
await this.saveManifest();
|
||||
|
||||
return {
|
||||
serviceId: serviceEntry.serviceId,
|
||||
serviceName: serviceEntry.serviceName,
|
||||
resources: Object.values(serviceEntry.resources) as interfaces.TCoreStoreResource[],
|
||||
env: serviceEntry.env,
|
||||
};
|
||||
}
|
||||
|
||||
private async deprovisionForService(requestArg: interfaces.ICoreStoreDeprovisionRequest) {
|
||||
if (!requestArg.serviceId) {
|
||||
throw new Error('serviceId is required');
|
||||
}
|
||||
const serviceEntry = this.manifest.services[requestArg.serviceId];
|
||||
if (!serviceEntry) {
|
||||
return { ok: true, deleted: [] };
|
||||
}
|
||||
|
||||
const deleted: Array<{ capability: string; resourceName: string }> = [];
|
||||
const databaseResource = serviceEntry.resources.database as interfaces.ICoreStoreDatabaseResource | undefined;
|
||||
if (databaseResource && this.smartDb) {
|
||||
await this.smartDb.deleteDatabaseTenant({
|
||||
databaseName: databaseResource.databaseName,
|
||||
username: databaseResource.username,
|
||||
}).catch((errorArg) => {
|
||||
console.warn(`Failed to delete database tenant ${databaseResource.databaseName}: ${(errorArg as Error).message}`);
|
||||
});
|
||||
deleted.push({ capability: 'database', resourceName: databaseResource.resourceName });
|
||||
}
|
||||
|
||||
const storageResource = serviceEntry.resources.objectstorage as interfaces.ICoreStoreObjectStorageResource | undefined;
|
||||
if (storageResource && this.smartStorage) {
|
||||
await this.smartStorage.deleteBucketTenant({
|
||||
bucketName: storageResource.bucketName,
|
||||
}).catch((errorArg) => {
|
||||
console.warn(`Failed to delete bucket tenant ${storageResource.bucketName}: ${(errorArg as Error).message}`);
|
||||
});
|
||||
deleted.push({ capability: 'objectstorage', resourceName: storageResource.resourceName });
|
||||
}
|
||||
|
||||
delete this.manifest.services[requestArg.serviceId];
|
||||
await this.saveManifest();
|
||||
return { ok: true, deleted };
|
||||
}
|
||||
|
||||
private async provisionDatabase(
|
||||
requestArg: interfaces.ICoreStoreProvisionRequest,
|
||||
): Promise<interfaces.ICoreStoreDatabaseResource> {
|
||||
if (!this.smartDb || !this.secretFile) {
|
||||
throw new Error('smartdb is not running');
|
||||
}
|
||||
const names = this.getResourceNames(requestArg.serviceId, requestArg.serviceName);
|
||||
const password = this.deriveHex(this.secretFile.masterSecret, `db-password:${requestArg.serviceId}`, 48);
|
||||
const existingTenants = await this.smartDb.listDatabaseTenants().catch(() => []);
|
||||
const existingTenant = existingTenants.find((tenantArg) => {
|
||||
return tenantArg.databaseName === names.databaseName && tenantArg.username === names.databaseUsername;
|
||||
});
|
||||
if (existingTenant) {
|
||||
await this.smartDb.rotateDatabaseTenantPassword({
|
||||
username: names.databaseUsername,
|
||||
password,
|
||||
});
|
||||
} else {
|
||||
await this.smartDb.createDatabaseTenant({
|
||||
databaseName: names.databaseName,
|
||||
username: names.databaseUsername,
|
||||
password,
|
||||
roles: ['readWrite'],
|
||||
});
|
||||
}
|
||||
const mongodbUri = this.buildMongoDbUri(names.databaseName, names.databaseUsername, password);
|
||||
const env = {
|
||||
MONGODB_URI: mongodbUri,
|
||||
MONGODB_HOST: this.options.publicHost,
|
||||
MONGODB_PORT: String(this.options.dbPort),
|
||||
MONGODB_DATABASE: names.databaseName,
|
||||
MONGODB_USERNAME: names.databaseUsername,
|
||||
MONGODB_PASSWORD: password,
|
||||
};
|
||||
return {
|
||||
capability: 'database',
|
||||
provider: 'smartdb',
|
||||
resourceName: names.databaseName,
|
||||
databaseName: names.databaseName,
|
||||
username: names.databaseUsername,
|
||||
env,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
private async provisionObjectStorage(
|
||||
requestArg: interfaces.ICoreStoreProvisionRequest,
|
||||
): Promise<interfaces.ICoreStoreObjectStorageResource> {
|
||||
if (!this.smartStorage || !this.secretFile) {
|
||||
throw new Error('smartstorage is not running');
|
||||
}
|
||||
const names = this.getResourceNames(requestArg.serviceId, requestArg.serviceName);
|
||||
const accessKeyId = `CS${this.deriveHex(this.secretFile.masterSecret, `s3-access:${requestArg.serviceId}`, 22).toUpperCase()}`;
|
||||
const secretAccessKey = this.deriveHex(this.secretFile.masterSecret, `s3-secret:${requestArg.serviceId}`, 64);
|
||||
const existingTenants = await this.smartStorage.listBucketTenants().catch(() => []);
|
||||
const existingTenant = existingTenants.find((tenantArg) => tenantArg.bucketName === names.bucketName);
|
||||
if (existingTenant) {
|
||||
await this.smartStorage.rotateBucketTenantCredentials({
|
||||
bucketName: names.bucketName,
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
region: this.options.region,
|
||||
});
|
||||
} else {
|
||||
await this.smartStorage.createBucketTenant({
|
||||
bucketName: names.bucketName,
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
region: this.options.region,
|
||||
});
|
||||
}
|
||||
const endpointUrl = `http://${this.options.publicHost}:${this.options.s3Port}`;
|
||||
const env = {
|
||||
S3_ENDPOINT: endpointUrl,
|
||||
S3_ENDPOINT_HOST: this.options.publicHost,
|
||||
S3_PORT: String(this.options.s3Port),
|
||||
S3_REGION: this.options.region,
|
||||
S3_BUCKET: names.bucketName,
|
||||
S3_ACCESS_KEY: accessKeyId,
|
||||
S3_ACCESS_KEY_ID: accessKeyId,
|
||||
S3_SECRET_KEY: secretAccessKey,
|
||||
S3_SECRET_ACCESS_KEY: secretAccessKey,
|
||||
S3_USE_SSL: 'false',
|
||||
AWS_ACCESS_KEY_ID: accessKeyId,
|
||||
AWS_SECRET_ACCESS_KEY: secretAccessKey,
|
||||
AWS_REGION: this.options.region,
|
||||
AWS_ENDPOINT_URL: endpointUrl,
|
||||
AWS_S3_FORCE_PATH_STYLE: 'true',
|
||||
};
|
||||
return {
|
||||
capability: 'objectstorage',
|
||||
provider: 'smartstorage',
|
||||
resourceName: names.bucketName,
|
||||
bucketName: names.bucketName,
|
||||
accessKeyId,
|
||||
env,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
private getResourceNames(serviceIdArg: string, serviceNameArg?: string) {
|
||||
const hash = plugins.crypto.createHash('sha1').update(serviceIdArg).digest('hex').slice(0, 12);
|
||||
const base = serviceNameArg || serviceIdArg;
|
||||
const bucketBase = this.sanitizeBucketPart(base).slice(0, 42) || 'service';
|
||||
const databaseBase = this.sanitizeIdentifierPart(base).slice(0, 36) || 'service';
|
||||
return {
|
||||
bucketName: this.trimBucketName(`sz-${bucketBase}-${hash}`),
|
||||
databaseName: `sz_${databaseBase}_${hash}`,
|
||||
databaseUsername: `u_${databaseBase}_${hash}`,
|
||||
};
|
||||
}
|
||||
|
||||
private sanitizeBucketPart(valueArg: string) {
|
||||
return valueArg
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.replace(/--+/g, '-');
|
||||
}
|
||||
|
||||
private sanitizeIdentifierPart(valueArg: string) {
|
||||
return valueArg
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_]+/g, '_')
|
||||
.replace(/^_+|_+$/g, '')
|
||||
.replace(/_+/g, '_');
|
||||
}
|
||||
|
||||
private trimBucketName(bucketNameArg: string) {
|
||||
const trimmed = bucketNameArg.slice(0, 63).replace(/^-+|-+$/g, '');
|
||||
return trimmed.length >= 3 ? trimmed : `${trimmed}000`.slice(0, 3);
|
||||
}
|
||||
|
||||
private buildMongoDbUri(databaseNameArg: string, usernameArg: string, passwordArg: string) {
|
||||
const auth = `${encodeURIComponent(usernameArg)}:${encodeURIComponent(passwordArg)}@`;
|
||||
const host = `${this.options.publicHost}:${this.options.dbPort}`;
|
||||
const query = new URLSearchParams({
|
||||
authSource: databaseNameArg,
|
||||
directConnection: 'true',
|
||||
});
|
||||
return `mongodb://${auth}${host}/${encodeURIComponent(databaseNameArg)}?${query.toString()}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
export type TCoreStoreCapability = 'database' | 'objectstorage';
|
||||
|
||||
export interface ICoreStoreProvisionRequest {
|
||||
serviceId: string;
|
||||
serviceName?: string;
|
||||
capabilities?: TCoreStoreCapability[];
|
||||
database?: boolean;
|
||||
db?: boolean;
|
||||
objectstorage?: boolean;
|
||||
s3?: boolean;
|
||||
}
|
||||
|
||||
export interface ICoreStoreDeprovisionRequest {
|
||||
serviceId: string;
|
||||
}
|
||||
|
||||
export interface ICoreStoreResourceBase {
|
||||
capability: TCoreStoreCapability;
|
||||
provider: string;
|
||||
resourceName: string;
|
||||
env: Record<string, string>;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
export interface ICoreStoreDatabaseResource extends ICoreStoreResourceBase {
|
||||
capability: 'database';
|
||||
provider: 'smartdb';
|
||||
databaseName: string;
|
||||
username: string;
|
||||
}
|
||||
|
||||
export interface ICoreStoreObjectStorageResource extends ICoreStoreResourceBase {
|
||||
capability: 'objectstorage';
|
||||
provider: 'smartstorage';
|
||||
bucketName: string;
|
||||
accessKeyId: string;
|
||||
}
|
||||
|
||||
export type TCoreStoreResource = ICoreStoreDatabaseResource | ICoreStoreObjectStorageResource;
|
||||
|
||||
export interface ICoreStoreServiceManifestEntry {
|
||||
serviceId: string;
|
||||
serviceName?: string;
|
||||
resources: Partial<Record<TCoreStoreCapability, TCoreStoreResource>>;
|
||||
env: Record<string, string>;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
export interface ICoreStoreManifest {
|
||||
version: 1;
|
||||
services: Record<string, ICoreStoreServiceManifestEntry>;
|
||||
}
|
||||
|
||||
export interface ICoreStoreProvisionResponse {
|
||||
serviceId: string;
|
||||
serviceName?: string;
|
||||
resources: TCoreStoreResource[];
|
||||
env: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface ICoreStoreSecretFile {
|
||||
masterSecret: string;
|
||||
dbRootPassword: string;
|
||||
s3AdminAccessKeyId: string;
|
||||
s3AdminSecretAccessKey: string;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import * as plugins from './corestore.plugins.js';
|
||||
|
||||
export const packageDir = plugins.path.join(
|
||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||
'../',
|
||||
);
|
||||
@@ -0,0 +1,15 @@
|
||||
// native
|
||||
import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import * as http from 'node:http';
|
||||
import * as path from 'node:path';
|
||||
|
||||
export { crypto, fs, http, path };
|
||||
|
||||
// @push.rocks scope
|
||||
import * as projectinfo from '@push.rocks/projectinfo';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import * as smartstorage from '@push.rocks/smartstorage';
|
||||
import * as smartdb from '@push.rocks/smartdb';
|
||||
|
||||
export { projectinfo, smartpath, smartstorage, smartdb };
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
import * as plugins from './corestore.plugins.js';
|
||||
import * as paths from './corestore.paths.js';
|
||||
import { CoreStore } from './corestore.classes.corestore.js';
|
||||
|
||||
export { CoreStore };
|
||||
|
||||
let coreStoreInstance: CoreStore | null = null;
|
||||
|
||||
export const runCli = async () => {
|
||||
const projectinfo = await plugins.projectinfo.ProjectInfo.create(paths.packageDir);
|
||||
console.log(`corestore@v${projectinfo.npm.version}`);
|
||||
coreStoreInstance = new CoreStore();
|
||||
await coreStoreInstance.start();
|
||||
console.log('corestore successfully started');
|
||||
};
|
||||
|
||||
export const stop = async () => {
|
||||
if (coreStoreInstance) {
|
||||
await coreStoreInstance.stop();
|
||||
coreStoreInstance = null;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"experimentalDecorators": true,
|
||||
"useDefineForClassFields": false,
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"types": [
|
||||
"node"
|
||||
],
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"exclude": [
|
||||
"dist_*/**/*.d.ts"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user