10 Commits

23 changed files with 1620 additions and 40 deletions

View File

@@ -48,7 +48,7 @@ RUN deno cache mod.ts
# Create storage directory
RUN mkdir -p /data
EXPOSE 9000 3000
EXPOSE 9000 3000 4433
VOLUME ["/data"]
ENTRYPOINT ["/sbin/tini", "--"]
CMD ["deno", "run", "--allow-all", "mod.ts", "server"]

View File

@@ -1,5 +1,34 @@
# Changelog
## 2026-03-22 - 1.7.1 - fix(repo)
no changes to commit
## 2026-03-22 - 1.7.0 - feat(cluster)
add cluster configuration support across server, CLI, and admin UI
- add environment variable and CLI support for cluster, QUIC, seed node, drive, erasure coding, and heartbeat settings
- pass cluster-aware configuration into smartstorage and expose the QUIC port in Docker
- extend config APIs and admin UI to display cluster, erasure coding, and storage drive configuration
- upgrade @push.rocks/smartstorage to ^6.3.1 to support the new cluster capabilities
## 2026-03-21 - 1.6.0 - feat(scripts)
add release script for committing and pushing docker images
- Adds a new npm release script to automate git commit and Docker image push steps.
## 2026-03-21 - 1.5.1 - fix(project)
no changes to commit
## 2026-03-21 - 1.5.0 - feat(test)
add end-to-end test coverage for container lifecycle, auth, buckets, objects, policies, credentials, status, and S3 compatibility
- adds a reusable test helper for creating isolated ObjectStorageContainer instances and logging in as admin
- introduces a test script in package.json to run the new Deno test suite
- covers core management API flows including authentication, bucket operations, object operations, named policies, credentials, and server status/config
- verifies S3 SDK interoperability including bucket and object operations plus credential rejection cases
## 2026-03-15 - 1.4.2 - fix(license)
add missing license file

View File

@@ -1,6 +1,6 @@
{
"name": "@lossless.zone/objectstorage",
"version": "1.4.2",
"version": "1.7.1",
"exports": "./mod.ts",
"nodeModulesDir": "auto",
"tasks": {
@@ -8,7 +8,7 @@
"dev": "pnpm run watch"
},
"imports": {
"@push.rocks/smartstorage": "npm:@push.rocks/smartstorage@^6.0.1",
"@push.rocks/smartstorage": "npm:@push.rocks/smartstorage@^6.3.1",
"@push.rocks/smartbucket": "npm:@push.rocks/smartbucket@^4.3.0",
"@aws-sdk/client-s3": "npm:@aws-sdk/client-s3@^3.937.0",
"@api.global/typedrequest-interfaces": "npm:@api.global/typedrequest-interfaces@^3.0.19",

41
deno.lock generated
View File

@@ -1,6 +1,10 @@
{
"version": "5",
"specifiers": {
"jsr:@std/assert@*": "1.0.19",
"jsr:@std/assert@^1.0.17": "1.0.19",
"jsr:@std/internal@^1.0.12": "1.0.12",
"jsr:@std/testing@*": "1.0.17",
"npm:@api.global/typedrequest-interfaces@^3.0.19": "3.0.19",
"npm:@api.global/typedrequest@^3.2.6": "3.3.0",
"npm:@api.global/typedserver@^8.3.1": "8.4.2_@push.rocks+smartserve@2.0.1",
@@ -13,7 +17,25 @@
"npm:@push.rocks/smartbucket@^4.3.0": "4.5.1",
"npm:@push.rocks/smartguard@^3.1.0": "3.1.0",
"npm:@push.rocks/smartjwt@^2.2.1": "2.2.1",
"npm:@push.rocks/smartstorage@^6.0.1": "6.0.1"
"npm:@push.rocks/smartstorage@^6.3.1": "6.3.1"
},
"jsr": {
"@std/assert@1.0.19": {
"integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e",
"dependencies": [
"jsr:@std/internal"
]
},
"@std/internal@1.0.12": {
"integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027"
},
"@std/testing@1.0.17": {
"integrity": "87bdc2700fa98249d48a17cd72413352d3d3680dcfbdb64947fd0982d6bbf681",
"dependencies": [
"jsr:@std/assert@^1.0.17",
"jsr:@std/internal"
]
}
},
"npm": {
"@api.global/typedrequest-interfaces@2.0.2": {
@@ -1929,14 +1951,14 @@
],
"tarball": "https://verdaccio.lossless.digital/@push.rocks/smartstate/-/smartstate-2.2.1.tgz"
},
"@push.rocks/smartstorage@6.0.1": {
"integrity": "sha512-W5PEVwO0J2K9YUZRTbKXadC11h6/IBzzqU+P0TIE/xpJZC4K1duEXwEhxGWcbfhCkPRRa51xH8Z5mAmzzm8qxA==",
"@push.rocks/smartstorage@6.3.1": {
"integrity": "sha512-Emxwyeb/BH//q32IB2Z34Clz0yR3mjWKgd+qhnAKA6UADRmF3DMemdz5cQVkmpQeD1YE8hu633gHmynsLTFMRA==",
"dependencies": [
"@push.rocks/smartpath@6.0.0",
"@push.rocks/smartrust",
"@tsclass/tsclass@9.4.0"
"@tsclass/tsclass@9.5.0"
],
"tarball": "https://verdaccio.lossless.digital/@push.rocks/smartstorage/-/smartstorage-6.0.1.tgz"
"tarball": "https://verdaccio.lossless.digital/@push.rocks/smartstorage/-/smartstorage-6.3.1.tgz"
},
"@push.rocks/smartstream@2.0.8": {
"integrity": "sha512-GlF/9cCkvBHwKa3DK4DO5wjfSgqkj6gAS4TrY9uD5NMHu9RQv4WiNrElTYj7iCEpnZgUnLO3tzw1JA3NRIMnnA==",
@@ -3108,6 +3130,13 @@
],
"tarball": "https://verdaccio.lossless.digital/@tsclass/tsclass/-/tsclass-9.4.0.tgz"
},
"@tsclass/tsclass@9.5.0": {
"integrity": "sha512-HwMVwkrBnEFMjwOsMkGwWN/q+XEczSpf4a/PBAXgkDdV6sXdxAMFXUH1tW8Y5ecuvXFYMvFry4X57MCCT7Dm8A==",
"dependencies": [
"type-fest@5.4.4"
],
"tarball": "https://verdaccio.lossless.digital/@tsclass/tsclass/-/tsclass-9.5.0.tgz"
},
"@tybys/wasm-util@0.10.1": {
"integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==",
"dependencies": [
@@ -6028,7 +6057,7 @@
"npm:@push.rocks/smartbucket@^4.3.0",
"npm:@push.rocks/smartguard@^3.1.0",
"npm:@push.rocks/smartjwt@^2.2.1",
"npm:@push.rocks/smartstorage@^6.0.1"
"npm:@push.rocks/smartstorage@^6.3.1"
],
"packageJson": {
"dependencies": [

View File

@@ -1,14 +1,16 @@
{
"name": "@lossless.zone/objectstorage",
"version": "1.4.2",
"version": "1.7.1",
"description": "object storage server with management UI powered by smartstorage",
"main": "mod.ts",
"type": "module",
"scripts": {
"test": "deno task test",
"watch": "tswatch",
"build": "tsbundle",
"bundle": "tsbundle",
"build:docker": "tsdocker build --verbose",
"release": "gitzone commit -yp && tsdocker push --verbose",
"start:docker": "docker stop objectstorage 2>/dev/null; docker rm objectstorage 2>/dev/null; docker build --load -t objectstorage:latest . && docker run --rm --name objectstorage -p 9000:9000 -p 3000:3000 -v objectstorage-data:/data objectstorage:latest"
},
"author": "Lossless GmbH",

View File

@@ -0,0 +1,49 @@
import { ObjectStorageContainer } from '../../ts/index.ts';
import type { IObjectStorageConfig } from '../../ts/types.ts';
import type * as interfaces from '../../ts_interfaces/index.ts';
import { TypedRequest } from '@api.global/typedrequest';
import type { IReq_AdminLoginWithUsernameAndPassword } from '../../ts_interfaces/requests/admin.ts';
export const TEST_ADMIN_PASSWORD = 'testpassword';
export const TEST_ACCESS_KEY = 'testkey';
export const TEST_SECRET_KEY = 'testsecret';
export function getTestPorts(index: number): { objstPort: number; uiPort: number } {
return {
objstPort: 19000 + index * 10,
uiPort: 19001 + index * 10,
};
}
export function createTestContainer(
index: number,
extraConfig?: Partial<IObjectStorageConfig>,
): ObjectStorageContainer {
const ports = getTestPorts(index);
return new ObjectStorageContainer({
objstPort: ports.objstPort,
uiPort: ports.uiPort,
storageDirectory: `.nogit/testdata-${index}`,
accessCredentials: [{ accessKeyId: TEST_ACCESS_KEY, secretAccessKey: TEST_SECRET_KEY }],
adminPassword: TEST_ADMIN_PASSWORD,
region: 'us-east-1',
...extraConfig,
});
}
export async function loginAndGetIdentity(
uiPort: number,
): Promise<interfaces.data.IIdentity> {
const req = new TypedRequest<IReq_AdminLoginWithUsernameAndPassword>(
`http://localhost:${uiPort}/typedrequest`,
'adminLoginWithUsernameAndPassword',
);
const response = await req.fire({
username: 'admin',
password: TEST_ADMIN_PASSWORD,
});
if (!response.identity) {
throw new Error('Login failed: no identity returned');
}
return response.identity;
}

103
test/test.auth.test.ts Normal file
View File

@@ -0,0 +1,103 @@
import { assertEquals, assertExists } from 'jsr:@std/assert';
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
import { TypedRequest } from '@api.global/typedrequest';
import { createTestContainer, getTestPorts, loginAndGetIdentity, TEST_ADMIN_PASSWORD } from './helpers/server.helper.ts';
import { ObjectStorageContainer } from '../ts/index.ts';
import type * as interfaces from '../ts_interfaces/index.ts';
import type { IReq_AdminLoginWithUsernameAndPassword } from '../ts_interfaces/requests/admin.ts';
import type { IReq_VerifyIdentity } from '../ts_interfaces/requests/admin.ts';
import type { IReq_AdminLogout } from '../ts_interfaces/requests/admin.ts';
import type { IReq_GetServerStatus } from '../ts_interfaces/requests/status.ts';
const PORT_INDEX = 1;
const ports = getTestPorts(PORT_INDEX);
const url = `http://localhost:${ports.uiPort}/typedrequest`;
describe('Authentication', { sanitizeResources: false, sanitizeOps: false }, () => {
let container: ObjectStorageContainer;
let identity: interfaces.data.IIdentity;
beforeAll(async () => {
container = createTestContainer(PORT_INDEX);
await container.start();
});
afterAll(async () => {
await container.stop();
});
it('should login with valid credentials', async () => {
identity = await loginAndGetIdentity(ports.uiPort);
assertExists(identity.jwt);
assertEquals(identity.userId, 'admin');
assertEquals(identity.username, 'admin');
assertEquals(identity.role, 'admin');
assertEquals(identity.expiresAt > Date.now(), true);
});
it('should reject login with wrong password', async () => {
const req = new TypedRequest<IReq_AdminLoginWithUsernameAndPassword>(
url,
'adminLoginWithUsernameAndPassword',
);
let threw = false;
try {
await req.fire({ username: 'admin', password: 'wrongpassword' });
} catch {
threw = true;
}
assertEquals(threw, true);
});
it('should reject login with wrong username', async () => {
const req = new TypedRequest<IReq_AdminLoginWithUsernameAndPassword>(
url,
'adminLoginWithUsernameAndPassword',
);
let threw = false;
try {
await req.fire({ username: 'notadmin', password: TEST_ADMIN_PASSWORD });
} catch {
threw = true;
}
assertEquals(threw, true);
});
it('should verify a valid identity', async () => {
const req = new TypedRequest<IReq_VerifyIdentity>(url, 'verifyIdentity');
const response = await req.fire({ identity });
assertEquals(response.valid, true);
assertExists(response.identity);
assertEquals(response.identity!.userId, 'admin');
});
it('should reject verification with tampered JWT', async () => {
const req = new TypedRequest<IReq_VerifyIdentity>(url, 'verifyIdentity');
const tamperedIdentity = { ...identity, jwt: identity.jwt + 'tampered' };
const response = await req.fire({ identity: tamperedIdentity });
assertEquals(response.valid, false);
});
it('should reject verification with missing identity', async () => {
const req = new TypedRequest<IReq_VerifyIdentity>(url, 'verifyIdentity');
const response = await req.fire({ identity: null as any });
assertEquals(response.valid, false);
});
it('should logout successfully', async () => {
const req = new TypedRequest<IReq_AdminLogout>(url, 'adminLogout');
const response = await req.fire({ identity });
assertEquals(response.ok, true);
});
it('should reject protected endpoint without identity', async () => {
const req = new TypedRequest<IReq_GetServerStatus>(url, 'getServerStatus');
let threw = false;
try {
await req.fire({ identity: null as any });
} catch {
threw = true;
}
assertEquals(threw, true);
});
});

133
test/test.buckets.test.ts Normal file
View File

@@ -0,0 +1,133 @@
import { assertEquals, assertExists } from 'jsr:@std/assert';
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
import { TypedRequest } from '@api.global/typedrequest';
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
import { ObjectStorageContainer } from '../ts/index.ts';
import type * as interfaces from '../ts_interfaces/index.ts';
import type {
IReq_ListBuckets,
IReq_CreateBucket,
IReq_DeleteBucket,
IReq_GetBucketPolicy,
IReq_PutBucketPolicy,
IReq_DeleteBucketPolicy,
} from '../ts_interfaces/requests/buckets.ts';
const PORT_INDEX = 2;
const ports = getTestPorts(PORT_INDEX);
const url = `http://localhost:${ports.uiPort}/typedrequest`;
describe('Bucket management', { sanitizeResources: false, sanitizeOps: false }, () => {
let container: ObjectStorageContainer;
let identity: interfaces.data.IIdentity;
beforeAll(async () => {
container = createTestContainer(PORT_INDEX);
await container.start();
identity = await loginAndGetIdentity(ports.uiPort);
});
afterAll(async () => {
await container.stop();
});
it('should list buckets (initially empty)', async () => {
const req = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
const response = await req.fire({ identity });
assertEquals(response.buckets.length, 0);
});
it('should create a bucket', async () => {
const req = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
assertEquals(response.ok, true);
});
it('should create a second bucket', async () => {
const req = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
const response = await req.fire({ identity, bucketName: 'test-bucket-2' });
assertEquals(response.ok, true);
});
it('should list both buckets', async () => {
const req = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
const response = await req.fire({ identity });
assertEquals(response.buckets.length, 2);
const names = response.buckets.map((b) => b.name).sort();
assertEquals(names, ['test-bucket-1', 'test-bucket-2']);
});
it('should have valid bucket metadata', async () => {
const req = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
const response = await req.fire({ identity });
for (const bucket of response.buckets) {
assertEquals(bucket.objectCount, 0);
assertEquals(bucket.totalSizeBytes, 0);
assertEquals(bucket.creationDate > 0, true);
}
});
it('should delete a bucket', async () => {
const req = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
const response = await req.fire({ identity, bucketName: 'test-bucket-2' });
assertEquals(response.ok, true);
});
it('should list one remaining bucket', async () => {
const req = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
const response = await req.fire({ identity });
assertEquals(response.buckets.length, 1);
assertEquals(response.buckets[0].name, 'test-bucket-1');
});
it('should get bucket policy (initially null)', async () => {
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
assertEquals(response.policy, null);
});
it('should put a bucket policy', async () => {
const policy = JSON.stringify({
Version: '2012-10-17',
Statement: [
{
Sid: 'PublicRead',
Effect: 'Allow',
Principal: '*',
Action: 's3:GetObject',
Resource: 'arn:aws:s3:::test-bucket-1/*',
},
],
});
const req = new TypedRequest<IReq_PutBucketPolicy>(url, 'putBucketPolicy');
const response = await req.fire({ identity, bucketName: 'test-bucket-1', policy });
assertEquals(response.ok, true);
});
it('should get the bucket policy back', async () => {
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
assertExists(response.policy);
const parsed = JSON.parse(response.policy!);
assertExists(parsed.Statement);
assertEquals(parsed.Statement[0].Sid, 'PublicRead');
});
it('should delete bucket policy', async () => {
const req = new TypedRequest<IReq_DeleteBucketPolicy>(url, 'deleteBucketPolicy');
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
assertEquals(response.ok, true);
});
it('should confirm policy is deleted', async () => {
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
assertEquals(response.policy, null);
});
it('cleanup: delete remaining bucket', async () => {
const req = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
const response = await req.fire({ identity, bucketName: 'test-bucket-1' });
assertEquals(response.ok, true);
});
});

View File

@@ -0,0 +1,56 @@
import { assertEquals, assertExists } from 'jsr:@std/assert';
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
import { createTestContainer, getTestPorts } from './helpers/server.helper.ts';
import { ObjectStorageContainer } from '../ts/index.ts';
import { defaultConfig } from '../ts/types.ts';
const PORT_INDEX = 0;
const ports = getTestPorts(PORT_INDEX);
describe('ObjectStorageContainer lifecycle', { sanitizeResources: false, sanitizeOps: false }, () => {
let container: ObjectStorageContainer;
it('should create container with default config values', () => {
const c = new ObjectStorageContainer();
assertEquals(c.config.objstPort, defaultConfig.objstPort);
assertEquals(c.config.uiPort, defaultConfig.uiPort);
assertEquals(c.config.region, defaultConfig.region);
assertEquals(c.config.adminPassword, defaultConfig.adminPassword);
});
it('should create container with custom config overrides', () => {
container = createTestContainer(PORT_INDEX);
assertEquals(container.config.objstPort, ports.objstPort);
assertEquals(container.config.uiPort, ports.uiPort);
assertEquals(container.config.adminPassword, 'testpassword');
assertEquals(container.config.accessCredentials[0].accessKeyId, 'testkey');
});
it('should start successfully', async () => {
await container.start();
assertEquals(container.startedAt > 0, true);
});
it('should have s3Client initialized after start', () => {
assertExists(container.s3Client);
});
it('should have smartstorageInstance initialized after start', () => {
assertExists(container.smartstorageInstance);
});
it('should have policyManager with empty policies', () => {
assertExists(container.policyManager);
const policies = container.policyManager.listPolicies();
assertEquals(policies.length, 0);
});
it('should have opsServer started', () => {
assertExists(container.opsServer);
assertExists(container.opsServer.server);
});
it('should stop cleanly', async () => {
await container.stop();
});
});

View File

@@ -0,0 +1,93 @@
import { assertEquals } from 'jsr:@std/assert';
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
import { TypedRequest } from '@api.global/typedrequest';
import {
createTestContainer,
getTestPorts,
loginAndGetIdentity,
TEST_ACCESS_KEY,
} from './helpers/server.helper.ts';
import { ObjectStorageContainer } from '../ts/index.ts';
import type * as interfaces from '../ts_interfaces/index.ts';
import type {
IReq_GetCredentials,
IReq_AddCredential,
IReq_RemoveCredential,
} from '../ts_interfaces/requests/credentials.ts';
const PORT_INDEX = 6;
const ports = getTestPorts(PORT_INDEX);
const url = `http://localhost:${ports.uiPort}/typedrequest`;
describe('Credential management', { sanitizeResources: false, sanitizeOps: false }, () => {
let container: ObjectStorageContainer;
let identity: interfaces.data.IIdentity;
beforeAll(async () => {
container = createTestContainer(PORT_INDEX);
await container.start();
identity = await loginAndGetIdentity(ports.uiPort);
});
afterAll(async () => {
await container.stop();
});
it('should get credentials with masked secrets', async () => {
const req = new TypedRequest<IReq_GetCredentials>(url, 'getCredentials');
const response = await req.fire({ identity });
assertEquals(response.credentials.length, 1);
assertEquals(response.credentials[0].accessKeyId, TEST_ACCESS_KEY);
// Secret should be masked: first 4 chars + ****
assertEquals(response.credentials[0].secretAccessKey.includes('****'), true);
});
it('should add a new credential', async () => {
const req = new TypedRequest<IReq_AddCredential>(url, 'addCredential');
const response = await req.fire({
identity,
accessKeyId: 'newkey',
secretAccessKey: 'newsecret',
});
assertEquals(response.ok, true);
});
it('should list credentials showing both', async () => {
const req = new TypedRequest<IReq_GetCredentials>(url, 'getCredentials');
const response = await req.fire({ identity });
assertEquals(response.credentials.length, 2);
const keys = response.credentials.map((c) => c.accessKeyId);
assertEquals(keys.includes(TEST_ACCESS_KEY), true);
assertEquals(keys.includes('newkey'), true);
});
it('should verify new credential is stored in config', () => {
const creds = container.config.accessCredentials;
const newCred = creds.find((c) => c.accessKeyId === 'newkey');
assertEquals(newCred?.secretAccessKey, 'newsecret');
});
it('should remove a credential', async () => {
const req = new TypedRequest<IReq_RemoveCredential>(url, 'removeCredential');
const response = await req.fire({ identity, accessKeyId: 'newkey' });
assertEquals(response.ok, true);
});
it('should list credentials showing one remaining', async () => {
const req = new TypedRequest<IReq_GetCredentials>(url, 'getCredentials');
const response = await req.fire({ identity });
assertEquals(response.credentials.length, 1);
assertEquals(response.credentials[0].accessKeyId, TEST_ACCESS_KEY);
});
it('should reject removing the last credential', async () => {
const req = new TypedRequest<IReq_RemoveCredential>(url, 'removeCredential');
let threw = false;
try {
await req.fire({ identity, accessKeyId: TEST_ACCESS_KEY });
} catch {
threw = true;
}
assertEquals(threw, true);
});
});

215
test/test.objects.test.ts Normal file
View File

@@ -0,0 +1,215 @@
import { assertEquals, assertExists } from 'jsr:@std/assert';
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
import { TypedRequest } from '@api.global/typedrequest';
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
import { ObjectStorageContainer } from '../ts/index.ts';
import type * as interfaces from '../ts_interfaces/index.ts';
import type { IReq_CreateBucket, IReq_DeleteBucket } from '../ts_interfaces/requests/buckets.ts';
import type {
IReq_ListObjects,
IReq_PutObject,
IReq_GetObject,
IReq_DeleteObject,
IReq_DeletePrefix,
IReq_GetObjectUrl,
IReq_MoveObject,
IReq_MovePrefix,
} from '../ts_interfaces/requests/objects.ts';
const PORT_INDEX = 3;
const ports = getTestPorts(PORT_INDEX);
const url = `http://localhost:${ports.uiPort}/typedrequest`;
const BUCKET = 'obj-test-bucket';
function toBase64(text: string): string {
return btoa(text);
}
function fromBase64(b64: string): string {
return atob(b64);
}
describe('Object operations', { sanitizeResources: false, sanitizeOps: false }, () => {
let container: ObjectStorageContainer;
let identity: interfaces.data.IIdentity;
beforeAll(async () => {
container = createTestContainer(PORT_INDEX);
await container.start();
identity = await loginAndGetIdentity(ports.uiPort);
// Create test bucket
const req = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
await req.fire({ identity, bucketName: BUCKET });
});
afterAll(async () => {
// Delete test bucket
try {
const req = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
await req.fire({ identity, bucketName: BUCKET });
} catch { /* bucket may already be empty/deleted */ }
await container.stop();
});
it('should list objects (initially empty)', async () => {
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
const response = await req.fire({ identity, bucketName: BUCKET });
assertEquals(response.result.objects.length, 0);
assertEquals(response.result.commonPrefixes.length, 0);
});
it('should put a text object', async () => {
const req = new TypedRequest<IReq_PutObject>(url, 'putObject');
const response = await req.fire({
identity,
bucketName: BUCKET,
key: 'hello.txt',
base64Content: toBase64('Hello World'),
contentType: 'text/plain',
});
assertEquals(response.ok, true);
});
it('should put nested objects', async () => {
const req = new TypedRequest<IReq_PutObject>(url, 'putObject');
await req.fire({
identity,
bucketName: BUCKET,
key: 'folder/nested.txt',
base64Content: toBase64('Nested content'),
contentType: 'text/plain',
});
await req.fire({
identity,
bucketName: BUCKET,
key: 'folder/a.txt',
base64Content: toBase64('File A'),
contentType: 'text/plain',
});
await req.fire({
identity,
bucketName: BUCKET,
key: 'folder/b.txt',
base64Content: toBase64('File B'),
contentType: 'text/plain',
});
await req.fire({
identity,
bucketName: BUCKET,
key: 'folder/sub/c.txt',
base64Content: toBase64('File C'),
contentType: 'text/plain',
});
});
it('should list objects at root with delimiter', async () => {
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
const response = await req.fire({ identity, bucketName: BUCKET, delimiter: '/' });
// Root should have hello.txt as direct object
const rootKeys = response.result.objects.map((o) => o.key);
assertEquals(rootKeys.includes('hello.txt'), true);
// folder/ should appear as a common prefix
assertEquals(response.result.commonPrefixes.includes('folder/'), true);
});
it('should list objects with prefix', async () => {
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
const response = await req.fire({
identity,
bucketName: BUCKET,
prefix: 'folder/',
delimiter: '/',
});
const keys = response.result.objects.map((o) => o.key);
assertEquals(keys.includes('folder/nested.txt'), true);
assertEquals(keys.includes('folder/a.txt'), true);
assertEquals(keys.includes('folder/b.txt'), true);
// sub/ should be a common prefix
assertEquals(response.result.commonPrefixes.includes('folder/sub/'), true);
});
it('should get a text object', async () => {
const req = new TypedRequest<IReq_GetObject>(url, 'getObject');
const response = await req.fire({ identity, bucketName: BUCKET, key: 'hello.txt' });
assertEquals(fromBase64(response.content), 'Hello World');
assertEquals(response.size > 0, true);
assertExists(response.lastModified);
});
it('should get object URL', async () => {
const req = new TypedRequest<IReq_GetObjectUrl>(url, 'getObjectUrl');
const response = await req.fire({ identity, bucketName: BUCKET, key: 'hello.txt' });
assertExists(response.url);
assertEquals(response.url.includes(BUCKET), true);
assertEquals(response.url.includes('hello.txt'), true);
});
it('should move an object', async () => {
const req = new TypedRequest<IReq_MoveObject>(url, 'moveObject');
const response = await req.fire({
identity,
bucketName: BUCKET,
sourceKey: 'hello.txt',
destKey: 'moved-hello.txt',
});
assertEquals(response.success, true);
});
it('should verify moved object exists at new key', async () => {
const req = new TypedRequest<IReq_GetObject>(url, 'getObject');
const response = await req.fire({ identity, bucketName: BUCKET, key: 'moved-hello.txt' });
assertEquals(fromBase64(response.content), 'Hello World');
});
it('should verify source key no longer exists after move', async () => {
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
const response = await req.fire({ identity, bucketName: BUCKET, delimiter: '/' });
const rootKeys = response.result.objects.map((o) => o.key);
assertEquals(rootKeys.includes('hello.txt'), false);
assertEquals(rootKeys.includes('moved-hello.txt'), true);
});
it('should move a prefix', async () => {
const req = new TypedRequest<IReq_MovePrefix>(url, 'movePrefix');
const response = await req.fire({
identity,
bucketName: BUCKET,
sourcePrefix: 'folder/',
destPrefix: 'renamed/',
});
assertEquals(response.success, true);
assertEquals((response.movedCount ?? 0) >= 4, true);
});
it('should verify moved prefix contents', async () => {
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
const response = await req.fire({ identity, bucketName: BUCKET, prefix: 'renamed/' });
assertEquals(response.result.objects.length >= 1, true);
});
it('should verify old prefix is empty', async () => {
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
const response = await req.fire({ identity, bucketName: BUCKET, prefix: 'folder/' });
assertEquals(response.result.objects.length, 0);
assertEquals(response.result.commonPrefixes.length, 0);
});
it('should delete a single object', async () => {
const req = new TypedRequest<IReq_DeleteObject>(url, 'deleteObject');
const response = await req.fire({ identity, bucketName: BUCKET, key: 'moved-hello.txt' });
assertEquals(response.ok, true);
});
it('should delete a prefix', async () => {
const req = new TypedRequest<IReq_DeletePrefix>(url, 'deletePrefix');
const response = await req.fire({ identity, bucketName: BUCKET, prefix: 'renamed/' });
assertEquals(response.ok, true);
});
it('should verify bucket is empty after cleanup', async () => {
const req = new TypedRequest<IReq_ListObjects>(url, 'listObjects');
const response = await req.fire({ identity, bucketName: BUCKET });
assertEquals(response.result.objects.length, 0);
});
});

258
test/test.policies.test.ts Normal file
View File

@@ -0,0 +1,258 @@
import { assertEquals, assertExists } from 'jsr:@std/assert';
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
import { TypedRequest } from '@api.global/typedrequest';
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
import { ObjectStorageContainer } from '../ts/index.ts';
import type * as interfaces from '../ts_interfaces/index.ts';
import type { IReq_CreateBucket, IReq_DeleteBucket, IReq_GetBucketPolicy } from '../ts_interfaces/requests/buckets.ts';
import type {
IReq_ListNamedPolicies,
IReq_CreateNamedPolicy,
IReq_UpdateNamedPolicy,
IReq_DeleteNamedPolicy,
IReq_GetBucketNamedPolicies,
IReq_AttachPolicyToBucket,
IReq_DetachPolicyFromBucket,
IReq_GetPolicyBuckets,
IReq_SetPolicyBuckets,
} from '../ts_interfaces/requests/policies.ts';
const PORT_INDEX = 5;
const ports = getTestPorts(PORT_INDEX);
const url = `http://localhost:${ports.uiPort}/typedrequest`;
describe('Named policy management', { sanitizeResources: false, sanitizeOps: false }, () => {
let container: ObjectStorageContainer;
let identity: interfaces.data.IIdentity;
let policy1Id: string;
let policy2Id: string;
beforeAll(async () => {
container = createTestContainer(PORT_INDEX);
await container.start();
identity = await loginAndGetIdentity(ports.uiPort);
// Create test buckets
const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
await createBucket.fire({ identity, bucketName: 'pol-bucket-1' });
await createBucket.fire({ identity, bucketName: 'pol-bucket-2' });
});
afterAll(async () => {
// Cleanup
try {
const del = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
await del.fire({ identity, bucketName: 'pol-bucket-1' });
} catch { /* may already be deleted */ }
try {
const del = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
await del.fire({ identity, bucketName: 'pol-bucket-2' });
} catch { /* may already be deleted */ }
await container.stop();
});
it('should list policies (initially empty)', async () => {
const req = new TypedRequest<IReq_ListNamedPolicies>(url, 'listNamedPolicies');
const response = await req.fire({ identity });
assertEquals(response.policies.length, 0);
});
it('should create a named policy', async () => {
const req = new TypedRequest<IReq_CreateNamedPolicy>(url, 'createNamedPolicy');
const response = await req.fire({
identity,
name: 'Public Read',
description: 'Allows public read access',
statements: [
{
Sid: 'PublicRead',
Effect: 'Allow',
Principal: '*',
Action: 's3:GetObject',
Resource: 'arn:aws:s3:::${bucket}/*',
},
],
});
assertExists(response.policy.id);
assertEquals(response.policy.name, 'Public Read');
assertEquals(response.policy.statements.length, 1);
assertEquals(response.policy.createdAt > 0, true);
policy1Id = response.policy.id;
});
it('should create a second policy', async () => {
const req = new TypedRequest<IReq_CreateNamedPolicy>(url, 'createNamedPolicy');
const response = await req.fire({
identity,
name: 'Deny Delete',
description: 'Denies delete actions',
statements: [
{
Sid: 'DenyDelete',
Effect: 'Deny',
Principal: '*',
Action: 's3:DeleteObject',
Resource: 'arn:aws:s3:::${bucket}/*',
},
],
});
policy2Id = response.policy.id;
assertExists(policy2Id);
});
it('should list both policies', async () => {
const req = new TypedRequest<IReq_ListNamedPolicies>(url, 'listNamedPolicies');
const response = await req.fire({ identity });
assertEquals(response.policies.length, 2);
});
it('should get bucket named policies (none attached)', async () => {
const req = new TypedRequest<IReq_GetBucketNamedPolicies>(url, 'getBucketNamedPolicies');
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
assertEquals(response.attachedPolicies.length, 0);
assertEquals(response.availablePolicies.length, 2);
});
it('should attach policy to bucket', async () => {
const req = new TypedRequest<IReq_AttachPolicyToBucket>(url, 'attachPolicyToBucket');
const response = await req.fire({ identity, policyId: policy1Id, bucketName: 'pol-bucket-1' });
assertEquals(response.ok, true);
});
it('should verify bucket policy was applied with placeholder replaced', async () => {
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
assertExists(response.policy);
const parsed = JSON.parse(response.policy!);
const resource = parsed.Statement[0].Resource;
// Resource may be a string or array depending on the S3 engine
const resourceStr = Array.isArray(resource) ? resource.join(' ') : resource;
// ${bucket} should be replaced with actual bucket name
assertEquals(resourceStr.includes('pol-bucket-1'), true);
assertEquals(resourceStr.includes('${bucket}'), false);
});
it('should get bucket named policies (one attached)', async () => {
const req = new TypedRequest<IReq_GetBucketNamedPolicies>(url, 'getBucketNamedPolicies');
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
assertEquals(response.attachedPolicies.length, 1);
assertEquals(response.availablePolicies.length, 1);
assertEquals(response.attachedPolicies[0].id, policy1Id);
});
it('should attach second policy to same bucket', async () => {
const req = new TypedRequest<IReq_AttachPolicyToBucket>(url, 'attachPolicyToBucket');
const response = await req.fire({ identity, policyId: policy2Id, bucketName: 'pol-bucket-1' });
assertEquals(response.ok, true);
});
it('should verify merged policy has statements from both', async () => {
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
assertExists(response.policy);
const parsed = JSON.parse(response.policy!);
assertEquals(parsed.Statement.length >= 2, true);
const sids = parsed.Statement.map((s: any) => s.Sid);
assertEquals(sids.includes('PublicRead'), true);
assertEquals(sids.includes('DenyDelete'), true);
});
it('should get policy buckets', async () => {
const req = new TypedRequest<IReq_GetPolicyBuckets>(url, 'getPolicyBuckets');
const response = await req.fire({ identity, policyId: policy1Id });
assertEquals(response.attachedBuckets.includes('pol-bucket-1'), true);
assertEquals(response.availableBuckets.includes('pol-bucket-2'), true);
});
it('should set policy buckets (batch)', async () => {
const req = new TypedRequest<IReq_SetPolicyBuckets>(url, 'setPolicyBuckets');
const response = await req.fire({
identity,
policyId: policy1Id,
bucketNames: ['pol-bucket-1', 'pol-bucket-2'],
});
assertEquals(response.ok, true);
});
it('should verify policy applied to second bucket', async () => {
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
const response = await req.fire({ identity, bucketName: 'pol-bucket-2' });
assertExists(response.policy);
const parsed = JSON.parse(response.policy!);
const resource = parsed.Statement[0].Resource;
const resourceStr = Array.isArray(resource) ? resource.join(' ') : resource;
assertEquals(resourceStr.includes('pol-bucket-2'), true);
});
it('should update a policy', async () => {
const req = new TypedRequest<IReq_UpdateNamedPolicy>(url, 'updateNamedPolicy');
const response = await req.fire({
identity,
policyId: policy1Id,
name: 'Public Read Updated',
description: 'Updated policy',
statements: [
{
Sid: 'PublicReadUpdated',
Effect: 'Allow',
Principal: '*',
Action: ['s3:GetObject', 's3:ListBucket'],
Resource: 'arn:aws:s3:::${bucket}/*',
},
],
});
assertEquals(response.policy.name, 'Public Read Updated');
assertEquals(response.policy.updatedAt >= response.policy.createdAt, true);
});
it('should verify updated policy cascaded to attached buckets', async () => {
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
assertExists(response.policy);
const parsed = JSON.parse(response.policy!);
const sids = parsed.Statement.map((s: any) => s.Sid);
assertEquals(sids.includes('PublicReadUpdated'), true);
});
it('should detach policy from bucket', async () => {
const req = new TypedRequest<IReq_DetachPolicyFromBucket>(url, 'detachPolicyFromBucket');
const response = await req.fire({ identity, policyId: policy2Id, bucketName: 'pol-bucket-1' });
assertEquals(response.ok, true);
});
it('should verify bucket policy updated after detach', async () => {
const req = new TypedRequest<IReq_GetBucketPolicy>(url, 'getBucketPolicy');
const response = await req.fire({ identity, bucketName: 'pol-bucket-1' });
assertExists(response.policy);
const parsed = JSON.parse(response.policy!);
const sids = parsed.Statement.map((s: any) => s.Sid);
assertEquals(sids.includes('DenyDelete'), false);
assertEquals(sids.includes('PublicReadUpdated'), true);
});
it('should delete a named policy', async () => {
const req = new TypedRequest<IReq_DeleteNamedPolicy>(url, 'deleteNamedPolicy');
const response = await req.fire({ identity, policyId: policy2Id });
assertEquals(response.ok, true);
});
it('should handle bucket deletion cleaning up attachments', async () => {
// Delete pol-bucket-1 which has policy1 attached
const delBucket = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
await delBucket.fire({ identity, bucketName: 'pol-bucket-1' });
// Verify policy1 no longer lists pol-bucket-1
const req = new TypedRequest<IReq_GetPolicyBuckets>(url, 'getPolicyBuckets');
const response = await req.fire({ identity, policyId: policy1Id });
assertEquals(response.attachedBuckets.includes('pol-bucket-1'), false);
});
it('cleanup: delete remaining policy and bucket', async () => {
const delPolicy = new TypedRequest<IReq_DeleteNamedPolicy>(url, 'deleteNamedPolicy');
await delPolicy.fire({ identity, policyId: policy1Id });
const listPolicies = new TypedRequest<IReq_ListNamedPolicies>(url, 'listNamedPolicies');
const response = await listPolicies.fire({ identity });
assertEquals(response.policies.length, 0);
});
});

132
test/test.s3-compat.test.ts Normal file
View File

@@ -0,0 +1,132 @@
import { assertEquals, assertExists } from 'jsr:@std/assert';
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
import {
S3Client,
ListBucketsCommand,
CreateBucketCommand,
DeleteBucketCommand,
PutObjectCommand,
GetObjectCommand,
ListObjectsV2Command,
CopyObjectCommand,
DeleteObjectCommand,
} from '@aws-sdk/client-s3';
import { createTestContainer, getTestPorts, TEST_ACCESS_KEY, TEST_SECRET_KEY } from './helpers/server.helper.ts';
import { ObjectStorageContainer } from '../ts/index.ts';
const PORT_INDEX = 4;
const ports = getTestPorts(PORT_INDEX);
const BUCKET = 's3-test-bucket';
describe('S3 SDK compatibility', { sanitizeResources: false, sanitizeOps: false }, () => {
let container: ObjectStorageContainer;
let s3: S3Client;
beforeAll(async () => {
container = createTestContainer(PORT_INDEX);
await container.start();
s3 = new S3Client({
endpoint: `http://localhost:${ports.objstPort}`,
region: 'us-east-1',
credentials: {
accessKeyId: TEST_ACCESS_KEY,
secretAccessKey: TEST_SECRET_KEY,
},
forcePathStyle: true,
});
});
afterAll(async () => {
s3.destroy();
await container.stop();
});
it('should list buckets (empty)', async () => {
const response = await s3.send(new ListBucketsCommand({}));
assertEquals((response.Buckets || []).length, 0);
});
it('should create bucket via S3', async () => {
await s3.send(new CreateBucketCommand({ Bucket: BUCKET }));
const response = await s3.send(new ListBucketsCommand({}));
const names = (response.Buckets || []).map((b) => b.Name);
assertEquals(names.includes(BUCKET), true);
});
it('should put object via S3', async () => {
await s3.send(
new PutObjectCommand({
Bucket: BUCKET,
Key: 'test.txt',
Body: new TextEncoder().encode('Hello from S3 SDK'),
ContentType: 'text/plain',
}),
);
});
it('should get object via S3', async () => {
const response = await s3.send(
new GetObjectCommand({ Bucket: BUCKET, Key: 'test.txt' }),
);
const body = await response.Body!.transformToString();
assertEquals(body, 'Hello from S3 SDK');
});
it('should list objects via S3', async () => {
const response = await s3.send(
new ListObjectsV2Command({ Bucket: BUCKET }),
);
const keys = (response.Contents || []).map((o) => o.Key);
assertEquals(keys.includes('test.txt'), true);
});
it('should copy object via S3', async () => {
await s3.send(
new CopyObjectCommand({
Bucket: BUCKET,
CopySource: `${BUCKET}/test.txt`,
Key: 'copy.txt',
}),
);
const response = await s3.send(
new GetObjectCommand({ Bucket: BUCKET, Key: 'copy.txt' }),
);
const body = await response.Body!.transformToString();
assertEquals(body, 'Hello from S3 SDK');
});
it('should delete object via S3', async () => {
await s3.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'test.txt' }));
await s3.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'copy.txt' }));
const response = await s3.send(
new ListObjectsV2Command({ Bucket: BUCKET }),
);
assertEquals((response.Contents || []).length, 0);
});
it('should reject requests with wrong credentials', async () => {
const badS3 = new S3Client({
endpoint: `http://localhost:${ports.objstPort}`,
region: 'us-east-1',
credentials: {
accessKeyId: 'wrongkey',
secretAccessKey: 'wrongsecret',
},
forcePathStyle: true,
});
let threw = false;
try {
await badS3.send(new ListBucketsCommand({}));
} catch {
threw = true;
}
badS3.destroy();
assertEquals(threw, true);
});
it('cleanup: delete bucket', async () => {
await s3.send(new DeleteBucketCommand({ Bucket: BUCKET }));
const response = await s3.send(new ListBucketsCommand({}));
assertEquals((response.Buckets || []).length, 0);
});
});

View File

@@ -0,0 +1,112 @@
import { assertEquals, assertExists } from 'jsr:@std/assert';
import { afterAll, beforeAll, describe, it } from 'jsr:@std/testing/bdd';
import { TypedRequest } from '@api.global/typedrequest';
import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
import { ObjectStorageContainer } from '../ts/index.ts';
import type * as interfaces from '../ts_interfaces/index.ts';
import type { IReq_CreateBucket, IReq_DeleteBucket } from '../ts_interfaces/requests/buckets.ts';
import type { IReq_PutObject, IReq_DeletePrefix } from '../ts_interfaces/requests/objects.ts';
import type { IReq_GetServerStatus } from '../ts_interfaces/requests/status.ts';
import type { IReq_GetServerConfig } from '../ts_interfaces/requests/config.ts';
const PORT_INDEX = 7;
const ports = getTestPorts(PORT_INDEX);
const url = `http://localhost:${ports.uiPort}/typedrequest`;
const BUCKET = 'stats-bucket';
describe('Status and config', { sanitizeResources: false, sanitizeOps: false }, () => {
let container: ObjectStorageContainer;
let identity: interfaces.data.IIdentity;
beforeAll(async () => {
container = createTestContainer(PORT_INDEX);
await container.start();
identity = await loginAndGetIdentity(ports.uiPort);
// Create bucket with objects for stats testing
const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
await createBucket.fire({ identity, bucketName: BUCKET });
const putObj = new TypedRequest<IReq_PutObject>(url, 'putObject');
await putObj.fire({
identity,
bucketName: BUCKET,
key: 'file1.txt',
base64Content: btoa('Content of file 1'),
contentType: 'text/plain',
});
await putObj.fire({
identity,
bucketName: BUCKET,
key: 'file2.txt',
base64Content: btoa('Content of file 2'),
contentType: 'text/plain',
});
});
afterAll(async () => {
try {
const delPrefix = new TypedRequest<IReq_DeletePrefix>(url, 'deletePrefix');
await delPrefix.fire({ identity, bucketName: BUCKET, prefix: '' });
const delBucket = new TypedRequest<IReq_DeleteBucket>(url, 'deleteBucket');
await delBucket.fire({ identity, bucketName: BUCKET });
} catch { /* cleanup best-effort */ }
await container.stop();
});
it('should get server status', async () => {
const req = new TypedRequest<IReq_GetServerStatus>(url, 'getServerStatus');
const response = await req.fire({ identity });
const status = response.status;
assertEquals(status.running, true);
assertEquals(status.objstPort, ports.objstPort);
assertEquals(status.uiPort, ports.uiPort);
assertEquals(status.bucketCount, 1);
assertEquals(status.totalObjectCount, 2);
assertEquals(status.totalStorageBytes > 0, true);
assertEquals(status.uptime >= 0, true);
assertEquals(status.startedAt > 0, true);
assertEquals(status.region, 'us-east-1');
assertEquals(status.authEnabled, true);
});
it('should get connection info from status', async () => {
const req = new TypedRequest<IReq_GetServerStatus>(url, 'getServerStatus');
const response = await req.fire({ identity });
const info = response.connectionInfo;
assertExists(info.endpoint);
assertEquals(info.port, ports.objstPort);
assertExists(info.accessKey);
assertEquals(info.region, 'us-east-1');
});
it('should get server config', async () => {
const req = new TypedRequest<IReq_GetServerConfig>(url, 'getServerConfig');
const response = await req.fire({ identity });
const config = response.config;
assertEquals(config.objstPort, ports.objstPort);
assertEquals(config.uiPort, ports.uiPort);
assertEquals(config.region, 'us-east-1');
assertEquals(config.authEnabled, true);
assertEquals(config.corsEnabled, false);
});
it('should reflect correct stats after adding objects', async () => {
// Add a third object
const putObj = new TypedRequest<IReq_PutObject>(url, 'putObject');
await putObj.fire({
identity,
bucketName: BUCKET,
key: 'file3.txt',
base64Content: btoa('Content of file 3'),
contentType: 'text/plain',
});
const req = new TypedRequest<IReq_GetServerStatus>(url, 'getServerStatus');
const response = await req.fire({ identity });
assertEquals(response.status.totalObjectCount, 3);
});
});

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@lossless.zone/objectstorage',
version: '1.4.2',
version: '1.7.1',
description: 'object storage server with management UI powered by smartstorage'
}

View File

@@ -39,6 +39,37 @@ export class ObjectStorageContainer {
const envRegion = Deno.env.get('OBJST_REGION');
if (envRegion) this.config.region = envRegion;
// Cluster environment variables
const envClusterEnabled = Deno.env.get('OBJST_CLUSTER_ENABLED');
if (envClusterEnabled) this.config.clusterEnabled = envClusterEnabled === 'true' || envClusterEnabled === '1';
const envClusterNodeId = Deno.env.get('OBJST_CLUSTER_NODE_ID');
if (envClusterNodeId) this.config.clusterNodeId = envClusterNodeId;
const envClusterQuicPort = Deno.env.get('OBJST_CLUSTER_QUIC_PORT');
if (envClusterQuicPort) this.config.clusterQuicPort = parseInt(envClusterQuicPort, 10);
const envClusterSeedNodes = Deno.env.get('OBJST_CLUSTER_SEED_NODES');
if (envClusterSeedNodes) this.config.clusterSeedNodes = envClusterSeedNodes.split(',').map(s => s.trim()).filter(Boolean);
const envDrivePaths = Deno.env.get('OBJST_DRIVE_PATHS');
if (envDrivePaths) this.config.drivePaths = envDrivePaths.split(',').map(s => s.trim()).filter(Boolean);
const envErasureDataShards = Deno.env.get('OBJST_ERASURE_DATA_SHARDS');
if (envErasureDataShards) this.config.erasureDataShards = parseInt(envErasureDataShards, 10);
const envErasureParityShards = Deno.env.get('OBJST_ERASURE_PARITY_SHARDS');
if (envErasureParityShards) this.config.erasureParityShards = parseInt(envErasureParityShards, 10);
const envErasureChunkSize = Deno.env.get('OBJST_ERASURE_CHUNK_SIZE');
if (envErasureChunkSize) this.config.erasureChunkSizeBytes = parseInt(envErasureChunkSize, 10);
const envHeartbeatInterval = Deno.env.get('OBJST_HEARTBEAT_INTERVAL_MS');
if (envHeartbeatInterval) this.config.clusterHeartbeatIntervalMs = parseInt(envHeartbeatInterval, 10);
const envHeartbeatTimeout = Deno.env.get('OBJST_HEARTBEAT_TIMEOUT_MS');
if (envHeartbeatTimeout) this.config.clusterHeartbeatTimeoutMs = parseInt(envHeartbeatTimeout, 10);
this.opsServer = new OpsServer(this);
this.policyManager = new PolicyManager(this);
}
@@ -49,9 +80,17 @@ export class ObjectStorageContainer {
console.log(` UI port: ${this.config.uiPort}`);
console.log(` Storage: ${this.config.storageDirectory}`);
console.log(` Region: ${this.config.region}`);
console.log(` Cluster: ${this.config.clusterEnabled ? 'enabled' : 'disabled'}`);
if (this.config.clusterEnabled) {
console.log(` Node ID: ${this.config.clusterNodeId || '(auto-generated)'}`);
console.log(` QUIC Port: ${this.config.clusterQuicPort}`);
console.log(` Seed Nodes: ${this.config.clusterSeedNodes.join(', ') || '(none)'}`);
console.log(` Drives: ${this.config.drivePaths.length > 0 ? this.config.drivePaths.join(', ') : this.config.storageDirectory}`);
console.log(` Erasure: ${this.config.erasureDataShards}+${this.config.erasureParityShards}`);
}
// Start smartstorage
this.smartstorageInstance = await plugins.smartstorage.SmartStorage.createAndStart({
// Build smartstorage config
const smartstorageConfig: any = {
server: {
port: this.config.objstPort,
address: '0.0.0.0',
@@ -64,7 +103,31 @@ export class ObjectStorageContainer {
enabled: true,
credentials: this.config.accessCredentials,
},
});
};
if (this.config.clusterEnabled) {
smartstorageConfig.cluster = {
enabled: true,
nodeId: this.config.clusterNodeId || crypto.randomUUID().slice(0, 8),
quicPort: this.config.clusterQuicPort,
seedNodes: this.config.clusterSeedNodes,
erasure: {
dataShards: this.config.erasureDataShards,
parityShards: this.config.erasureParityShards,
chunkSizeBytes: this.config.erasureChunkSizeBytes,
},
drives: {
paths: this.config.drivePaths.length > 0
? this.config.drivePaths
: [this.config.storageDirectory],
},
heartbeatIntervalMs: this.config.clusterHeartbeatIntervalMs,
heartbeatTimeoutMs: this.config.clusterHeartbeatTimeoutMs,
};
}
// Start smartstorage
this.smartstorageInstance = await plugins.smartstorage.SmartStorage.createAndStart(smartstorageConfig);
this.startedAt = Date.now();
console.log(`Storage server started on port ${this.config.objstPort}`);

View File

@@ -29,6 +29,27 @@ export async function runCli(): Promise<void> {
// Use a temp directory for storage
configOverrides.storageDirectory = './.nogit/objstdata';
break;
case '--cluster-enabled':
configOverrides.clusterEnabled = true;
break;
case '--cluster-node-id':
configOverrides.clusterNodeId = args[++i];
break;
case '--cluster-quic-port':
configOverrides.clusterQuicPort = parseInt(args[++i], 10);
break;
case '--cluster-seed-nodes':
configOverrides.clusterSeedNodes = args[++i].split(',').map(s => s.trim()).filter(Boolean);
break;
case '--drive-paths':
configOverrides.drivePaths = args[++i].split(',').map(s => s.trim()).filter(Boolean);
break;
case '--erasure-data-shards':
configOverrides.erasureDataShards = parseInt(args[++i], 10);
break;
case '--erasure-parity-shards':
configOverrides.erasureParityShards = parseInt(args[++i], 10);
break;
}
}
@@ -59,19 +80,38 @@ ObjectStorage - S3-compatible object storage server with management UI
Usage:
objectstorage server [options]
Options:
--ephemeral Use local .nogit/objstdata for storage
--storage-port PORT Storage API port (default: 9000, env: OBJST_PORT)
--ui-port PORT Management UI port (default: 3000, env: UI_PORT)
--storage-dir DIR Storage directory (default: /data, env: OBJST_STORAGE_DIR)
Server Options:
--ephemeral Use local .nogit/objstdata for storage
--storage-port PORT Storage API port (default: 9000, env: OBJST_PORT)
--ui-port PORT Management UI port (default: 3000, env: UI_PORT)
--storage-dir DIR Storage directory (default: /data, env: OBJST_STORAGE_DIR)
Clustering Options:
--cluster-enabled Enable cluster mode (env: OBJST_CLUSTER_ENABLED)
--cluster-node-id ID Unique node identifier (env: OBJST_CLUSTER_NODE_ID)
--cluster-quic-port PORT QUIC transport port (default: 4433, env: OBJST_CLUSTER_QUIC_PORT)
--cluster-seed-nodes LIST Comma-separated seed node addresses (env: OBJST_CLUSTER_SEED_NODES)
--drive-paths LIST Comma-separated drive mount paths (env: OBJST_DRIVE_PATHS)
--erasure-data-shards N Erasure coding data shards (default: 4, env: OBJST_ERASURE_DATA_SHARDS)
--erasure-parity-shards N Erasure coding parity shards (default: 2, env: OBJST_ERASURE_PARITY_SHARDS)
Environment Variables:
OBJST_PORT Storage API port
UI_PORT Management UI port
OBJST_STORAGE_DIR Storage directory
OBJST_ACCESS_KEY Access key (default: admin)
OBJST_SECRET_KEY Secret key (default: admin)
OBJST_ADMIN_PASSWORD Admin UI password (default: admin)
OBJST_REGION Storage region (default: us-east-1)
OBJST_PORT Storage API port
UI_PORT Management UI port
OBJST_STORAGE_DIR Storage directory
OBJST_ACCESS_KEY Access key (default: admin)
OBJST_SECRET_KEY Secret key (default: admin)
OBJST_ADMIN_PASSWORD Admin UI password (default: admin)
OBJST_REGION Storage region (default: us-east-1)
OBJST_CLUSTER_ENABLED Enable clustering (true/false)
OBJST_CLUSTER_NODE_ID Unique node identifier
OBJST_CLUSTER_QUIC_PORT QUIC transport port (default: 4433)
OBJST_CLUSTER_SEED_NODES Comma-separated seed node addresses
OBJST_DRIVE_PATHS Comma-separated drive mount paths
OBJST_ERASURE_DATA_SHARDS Erasure data shards (default: 4)
OBJST_ERASURE_PARITY_SHARDS Erasure parity shards (default: 2)
OBJST_ERASURE_CHUNK_SIZE Erasure chunk size in bytes (default: 4194304)
OBJST_HEARTBEAT_INTERVAL_MS Cluster heartbeat interval (default: 5000)
OBJST_HEARTBEAT_TIMEOUT_MS Cluster heartbeat timeout (default: 30000)
`);
}

View File

@@ -25,6 +25,16 @@ export class ConfigHandler {
storageDirectory: containerConfig.storageDirectory,
authEnabled: true,
corsEnabled: false,
clusterEnabled: containerConfig.clusterEnabled,
clusterNodeId: containerConfig.clusterNodeId,
clusterQuicPort: containerConfig.clusterQuicPort,
clusterSeedNodes: containerConfig.clusterSeedNodes,
erasureDataShards: containerConfig.erasureDataShards,
erasureParityShards: containerConfig.erasureParityShards,
erasureChunkSizeBytes: containerConfig.erasureChunkSizeBytes,
drivePaths: containerConfig.drivePaths,
clusterHeartbeatIntervalMs: containerConfig.clusterHeartbeatIntervalMs,
clusterHeartbeatTimeoutMs: containerConfig.clusterHeartbeatTimeoutMs,
};
return { config };
},

View File

@@ -5,6 +5,20 @@ export interface IObjectStorageConfig {
accessCredentials: Array<{ accessKeyId: string; secretAccessKey: string }>;
adminPassword: string;
region: string;
// Cluster
clusterEnabled: boolean;
clusterNodeId: string;
clusterQuicPort: number;
clusterSeedNodes: string[];
// Erasure coding
erasureDataShards: number;
erasureParityShards: number;
erasureChunkSizeBytes: number;
// Multi-drive
drivePaths: string[];
// Cluster heartbeat
clusterHeartbeatIntervalMs: number;
clusterHeartbeatTimeoutMs: number;
}
export const defaultConfig: IObjectStorageConfig = {
@@ -14,4 +28,14 @@ export const defaultConfig: IObjectStorageConfig = {
accessCredentials: [{ accessKeyId: 'admin', secretAccessKey: 'admin' }],
adminPassword: 'admin',
region: 'us-east-1',
clusterEnabled: false,
clusterNodeId: '',
clusterQuicPort: 4433,
clusterSeedNodes: [],
erasureDataShards: 4,
erasureParityShards: 2,
erasureChunkSizeBytes: 4194304,
drivePaths: [],
clusterHeartbeatIntervalMs: 5000,
clusterHeartbeatTimeoutMs: 30000,
};

File diff suppressed because one or more lines are too long

View File

@@ -19,6 +19,20 @@ export interface IServerConfig {
storageDirectory: string;
authEnabled: boolean;
corsEnabled: boolean;
// Cluster
clusterEnabled: boolean;
clusterNodeId: string;
clusterQuicPort: number;
clusterSeedNodes: string[];
// Erasure coding
erasureDataShards: number;
erasureParityShards: number;
erasureChunkSizeBytes: number;
// Multi-drive
drivePaths: string[];
// Cluster heartbeat
clusterHeartbeatIntervalMs: number;
clusterHeartbeatTimeoutMs: number;
}
export interface IObjstCredential {

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@lossless.zone/objectstorage',
version: '1.4.2',
version: '1.7.1',
description: 'object storage server with management UI powered by smartstorage'
}

View File

@@ -36,12 +36,84 @@ export class ObjstViewConfig extends DeesElement {
public static styles = [
cssManager.defaultStyles,
shared.viewHostCss,
css`
.sectionSpacer {
margin-top: 32px;
}
.infoPanel {
margin-top: 32px;
padding: 24px;
border-radius: 8px;
background: ${cssManager.bdTheme('#f5f5f5', '#1a1a2e')};
border: 1px solid ${cssManager.bdTheme('#e0e0e0', '#2a2a4a')};
}
.infoPanel h2 {
margin: 0 0 16px 0;
font-size: 18px;
font-weight: 600;
color: ${cssManager.bdTheme('#333', '#ccc')};
}
.infoPanel p {
margin: 0 0 16px 0;
font-size: 14px;
color: ${cssManager.bdTheme('#666', '#999')};
line-height: 1.5;
}
.infoPanel .row {
display: flex;
align-items: center;
margin-bottom: 8px;
font-size: 14px;
}
.infoPanel .label {
min-width: 260px;
font-family: monospace;
font-weight: 500;
color: ${cssManager.bdTheme('#1565c0', '#64b5f6')};
padding: 4px 8px;
border-radius: 4px;
background: ${cssManager.bdTheme('#e3f2fd', '#1a237e30')};
}
.infoPanel .value {
color: ${cssManager.bdTheme('#666', '#999')};
margin-left: 12px;
}
.driveList {
margin-top: 16px;
}
.driveList .driveItem {
display: flex;
align-items: center;
margin-bottom: 8px;
font-size: 14px;
}
.driveList .driveIndex {
width: 32px;
height: 32px;
border-radius: 6px;
background: ${cssManager.bdTheme('#e8eaf6', '#1a237e40')};
color: ${cssManager.bdTheme('#3f51b5', '#7986cb')};
display: flex;
align-items: center;
justify-content: center;
font-weight: 600;
font-size: 13px;
margin-right: 12px;
}
.driveList .drivePath {
font-family: monospace;
color: ${cssManager.bdTheme('#333', '#e0e0e0')};
padding: 6px 12px;
border-radius: 4px;
background: ${cssManager.bdTheme('#e8e8e8', '#252540')};
}
`,
];
public render(): TemplateResult {
const config = this.configState.config;
const tiles: IStatsTile[] = [
const serverTiles: IStatsTile[] = [
{
id: 'objstPort',
title: 'Storage API Port',
@@ -92,20 +164,166 @@ export class ObjstViewConfig extends DeesElement {
},
];
const clusterTiles: IStatsTile[] = [
{
id: 'clusterStatus',
title: 'Cluster Status',
value: config?.clusterEnabled ? 'Enabled' : 'Disabled',
type: 'text',
icon: 'lucide:network',
color: config?.clusterEnabled ? '#4caf50' : '#ff9800',
},
{
id: 'nodeId',
title: 'Node ID',
value: config?.clusterNodeId || '(auto)',
type: 'text',
icon: 'lucide:fingerprint',
color: '#607d8b',
},
{
id: 'quicPort',
title: 'QUIC Port',
value: config?.clusterQuicPort ?? 4433,
type: 'number',
icon: 'lucide:radio',
color: '#00bcd4',
},
{
id: 'seedNodes',
title: 'Seed Nodes',
value: config?.clusterSeedNodes?.length ?? 0,
type: 'number',
icon: 'lucide:gitBranch',
color: '#3f51b5',
description: config?.clusterSeedNodes?.length
? config.clusterSeedNodes.join(', ')
: 'No seed nodes configured',
},
{
id: 'heartbeatInterval',
title: 'Heartbeat Interval',
value: `${config?.clusterHeartbeatIntervalMs ?? 5000}ms`,
type: 'text',
icon: 'lucide:heartPulse',
color: '#e91e63',
},
{
id: 'heartbeatTimeout',
title: 'Heartbeat Timeout',
value: `${config?.clusterHeartbeatTimeoutMs ?? 30000}ms`,
type: 'text',
icon: 'lucide:timer',
color: '#ff5722',
},
];
const erasureTiles: IStatsTile[] = [
{
id: 'dataShards',
title: 'Data Shards',
value: config?.erasureDataShards ?? 4,
type: 'number',
icon: 'lucide:layers',
color: '#2196f3',
},
{
id: 'parityShards',
title: 'Parity Shards',
value: config?.erasureParityShards ?? 2,
type: 'number',
icon: 'lucide:shieldCheck',
color: '#4caf50',
},
{
id: 'chunkSize',
title: 'Chunk Size',
value: this.formatBytes(config?.erasureChunkSizeBytes ?? 4194304),
type: 'text',
icon: 'lucide:puzzle',
color: '#9c27b0',
description: `${config?.erasureDataShards ?? 4}+${config?.erasureParityShards ?? 2} = ${Math.round(((config?.erasureParityShards ?? 2) / (config?.erasureDataShards ?? 4)) * 100)}% overhead`,
},
];
const drivePaths = config?.drivePaths?.length
? config.drivePaths
: config?.storageDirectory
? [config.storageDirectory]
: ['/data'];
const driveTiles: IStatsTile[] = [
{
id: 'driveCount',
title: 'Drive Count',
value: drivePaths.length,
type: 'number',
icon: 'lucide:hardDrive',
color: '#3f51b5',
},
];
const refreshAction = {
name: 'Refresh',
iconName: 'lucide:refreshCw',
action: async () => {
await appstate.configStatePart.dispatchAction(appstate.fetchConfigAction, null);
},
};
return html`
<objst-sectionheading>Configuration</objst-sectionheading>
<objst-sectionheading>Server Configuration</objst-sectionheading>
<dees-statsgrid
.tiles=${tiles}
.gridActions=${[
{
name: 'Refresh',
iconName: 'lucide:refreshCw',
action: async () => {
await appstate.configStatePart.dispatchAction(appstate.fetchConfigAction, null);
},
},
]}
.tiles=${serverTiles}
.gridActions=${[refreshAction]}
></dees-statsgrid>
<div class="sectionSpacer">
<objst-sectionheading>Cluster Configuration</objst-sectionheading>
</div>
<dees-statsgrid .tiles=${clusterTiles}></dees-statsgrid>
${config?.clusterEnabled ? html`
<div class="sectionSpacer">
<objst-sectionheading>Erasure Coding</objst-sectionheading>
</div>
<dees-statsgrid .tiles=${erasureTiles}></dees-statsgrid>
` : ''}
<div class="sectionSpacer">
<objst-sectionheading>Storage Drives</objst-sectionheading>
</div>
<dees-statsgrid .tiles=${driveTiles}></dees-statsgrid>
<div class="driveList">
${drivePaths.map((path, i) => html`
<div class="driveItem">
<div class="driveIndex">${i + 1}</div>
<span class="drivePath">${path}</span>
</div>
`)}
</div>
<div class="infoPanel">
<h2>Configuration Reference</h2>
<p>Cluster and drive settings are applied at server startup. To change them, set the environment variables and restart the server.</p>
<div class="row"><span class="label">OBJST_CLUSTER_ENABLED</span><span class="value">Enable clustering (true/false)</span></div>
<div class="row"><span class="label">OBJST_CLUSTER_NODE_ID</span><span class="value">Unique node identifier</span></div>
<div class="row"><span class="label">OBJST_CLUSTER_QUIC_PORT</span><span class="value">QUIC transport port (default: 4433)</span></div>
<div class="row"><span class="label">OBJST_CLUSTER_SEED_NODES</span><span class="value">Comma-separated seed node addresses</span></div>
<div class="row"><span class="label">OBJST_DRIVE_PATHS</span><span class="value">Comma-separated drive mount paths</span></div>
<div class="row"><span class="label">OBJST_ERASURE_DATA_SHARDS</span><span class="value">Data shards for erasure coding (default: 4)</span></div>
<div class="row"><span class="label">OBJST_ERASURE_PARITY_SHARDS</span><span class="value">Parity shards for erasure coding (default: 2)</span></div>
<div class="row"><span class="label">OBJST_ERASURE_CHUNK_SIZE</span><span class="value">Chunk size in bytes (default: 4194304)</span></div>
<div class="row"><span class="label">OBJST_HEARTBEAT_INTERVAL_MS</span><span class="value">Heartbeat interval in ms (default: 5000)</span></div>
<div class="row"><span class="label">OBJST_HEARTBEAT_TIMEOUT_MS</span><span class="value">Heartbeat timeout in ms (default: 30000)</span></div>
</div>
`;
}
private formatBytes(bytes: number): string {
if (bytes === 0) return '0 B';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(1024));
return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
}
}