lossless.zone/objectstorage
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(opsserver): add health, audit, cluster health, and durable credential management hardening

Browse Source
This commit is contained in:
Jürgen Kunz
2026-04-30 07:10:21 +00:00
parent c3e5cabe3d
commit f4e5f02d0c
34 changed files with 1722 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ts/classes/objectstoragecontainer.ts
+280 -101
View File
@@ -1,8 +1,13 @@
import * as plugins from '../plugins.ts';
import { type IObjectStorageConfig, defaultConfig } from '../types.ts';
import { defaultConfig, type IObjectStorageConfig } from '../types.ts';
import type * as interfaces from '../../ts_interfaces/index.ts';
import { OpsServer } from '../opsserver/index.ts';
import { PolicyManager } from './policymanager.ts';
import { AuditLogger } from './auditlogger.ts';
interface IPersistedAdminConfig {
accessCredentials?: Array<{ accessKeyId: string; secretAccessKey: string }>;
}
export class ObjectStorageContainer {
public config: IObjectStorageConfig;
@@ -10,7 +15,9 @@ export class ObjectStorageContainer {
public s3Client!: plugins.S3Client;
public opsServer: OpsServer;
public policyManager: PolicyManager;
public auditLogger: AuditLogger;
public startedAt: number = 0;
private envAccessCredentialsProvided = false;
constructor(configArg?: Partial<IObjectStorageConfig>) {
this.config = { ...defaultConfig, ...configArg };
@@ -28,6 +35,7 @@ export class ObjectStorageContainer {
const envAccessKey = Deno.env.get('OBJST_ACCESS_KEY');
const envSecretKey = Deno.env.get('OBJST_SECRET_KEY');
if (envAccessKey && envSecretKey) {
this.envAccessCredentialsProvided = true;
this.config.accessCredentials = [
{ accessKeyId: envAccessKey, secretAccessKey: envSecretKey },
];
@@ -41,7 +49,9 @@ export class ObjectStorageContainer {
// Cluster environment variables
const envClusterEnabled = Deno.env.get('OBJST_CLUSTER_ENABLED');
if (envClusterEnabled) this.config.clusterEnabled = envClusterEnabled === 'true' || envClusterEnabled === '1';
if (envClusterEnabled) {
this.config.clusterEnabled = envClusterEnabled === 'true' || envClusterEnabled === '1';
}
const envClusterNodeId = Deno.env.get('OBJST_CLUSTER_NODE_ID');
if (envClusterNodeId) this.config.clusterNodeId = envClusterNodeId;
@@ -50,31 +60,47 @@ export class ObjectStorageContainer {
if (envClusterQuicPort) this.config.clusterQuicPort = parseInt(envClusterQuicPort, 10);
const envClusterSeedNodes = Deno.env.get('OBJST_CLUSTER_SEED_NODES');
if (envClusterSeedNodes) this.config.clusterSeedNodes = envClusterSeedNodes.split(',').map(s => s.trim()).filter(Boolean);
if (envClusterSeedNodes) {
this.config.clusterSeedNodes = envClusterSeedNodes.split(',').map((s) => s.trim()).filter(
Boolean,
);
}
const envDrivePaths = Deno.env.get('OBJST_DRIVE_PATHS');
if (envDrivePaths) this.config.drivePaths = envDrivePaths.split(',').map(s => s.trim()).filter(Boolean);
if (envDrivePaths) {
this.config.drivePaths = envDrivePaths.split(',').map((s) => s.trim()).filter(Boolean);
}
const envErasureDataShards = Deno.env.get('OBJST_ERASURE_DATA_SHARDS');
if (envErasureDataShards) this.config.erasureDataShards = parseInt(envErasureDataShards, 10);
const envErasureParityShards = Deno.env.get('OBJST_ERASURE_PARITY_SHARDS');
if (envErasureParityShards) this.config.erasureParityShards = parseInt(envErasureParityShards, 10);
if (envErasureParityShards) {
this.config.erasureParityShards = parseInt(envErasureParityShards, 10);
}
const envErasureChunkSize = Deno.env.get('OBJST_ERASURE_CHUNK_SIZE');
if (envErasureChunkSize) this.config.erasureChunkSizeBytes = parseInt(envErasureChunkSize, 10);
const envHeartbeatInterval = Deno.env.get('OBJST_HEARTBEAT_INTERVAL_MS');
if (envHeartbeatInterval) this.config.clusterHeartbeatIntervalMs = parseInt(envHeartbeatInterval, 10);
if (envHeartbeatInterval) {
this.config.clusterHeartbeatIntervalMs = parseInt(envHeartbeatInterval, 10);
}
const envHeartbeatTimeout = Deno.env.get('OBJST_HEARTBEAT_TIMEOUT_MS');
if (envHeartbeatTimeout) this.config.clusterHeartbeatTimeoutMs = parseInt(envHeartbeatTimeout, 10);
if (envHeartbeatTimeout) {
this.config.clusterHeartbeatTimeoutMs = parseInt(envHeartbeatTimeout, 10);
}
this.opsServer = new OpsServer(this);
this.policyManager = new PolicyManager(this);
this.auditLogger = new AuditLogger(this.config.storageDirectory);
}
public async start(): Promise<void> {
this.assertSecureStartupConfig();
await this.loadPersistedAdminConfig();
console.log(`Starting ObjectStorage...`);
console.log(` Storage port: ${this.config.objstPort}`);
console.log(` UI port: ${this.config.uiPort}`);
@@ -85,64 +111,24 @@ export class ObjectStorageContainer {
console.log(` Node ID: ${this.config.clusterNodeId || '(auto-generated)'}`);
console.log(` QUIC Port: ${this.config.clusterQuicPort}`);
console.log(` Seed Nodes: ${this.config.clusterSeedNodes.join(', ') || '(none)'}`);
console.log(` Drives: ${this.config.drivePaths.length > 0 ? this.config.drivePaths.join(', ') : this.config.storageDirectory}`);
console.log(
` Drives: ${
this.config.drivePaths.length > 0
? this.config.drivePaths.join(', ')
: this.config.storageDirectory
}`,
);
console.log(` Erasure: ${this.config.erasureDataShards}+${this.config.erasureParityShards}`);
}
// Build smartstorage config
const smartstorageConfig: any = {
server: {
port: this.config.objstPort,
address: '0.0.0.0',
region: this.config.region,
},
storage: {
directory: this.config.storageDirectory,
},
auth: {
enabled: true,
credentials: this.config.accessCredentials,
},
};
if (this.config.clusterEnabled) {
smartstorageConfig.cluster = {
enabled: true,
nodeId: this.config.clusterNodeId || crypto.randomUUID().slice(0, 8),
quicPort: this.config.clusterQuicPort,
seedNodes: this.config.clusterSeedNodes,
erasure: {
dataShards: this.config.erasureDataShards,
parityShards: this.config.erasureParityShards,
chunkSizeBytes: this.config.erasureChunkSizeBytes,
},
drives: {
paths: this.config.drivePaths.length > 0
? this.config.drivePaths
: [this.config.storageDirectory],
},
heartbeatIntervalMs: this.config.clusterHeartbeatIntervalMs,
heartbeatTimeoutMs: this.config.clusterHeartbeatTimeoutMs,
};
}
// Start smartstorage
this.smartstorageInstance = await plugins.smartstorage.SmartStorage.createAndStart(smartstorageConfig);
this.smartstorageInstance = await plugins.smartstorage.SmartStorage.createAndStart(
this.buildSmartstorageConfig(),
);
this.startedAt = Date.now();
console.log(`Storage server started on port ${this.config.objstPort}`);
// Create S3 client for management operations
const descriptor = await this.smartstorageInstance.getStorageDescriptor();
this.s3Client = new plugins.S3Client({
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
region: this.config.region,
credentials: {
accessKeyId: descriptor.accessKey,
secretAccessKey: descriptor.accessSecret,
},
forcePathStyle: true,
});
await this.refreshManagementClient();
// Load named policies
await this.policyManager.load();
@@ -159,41 +145,54 @@ export class ObjectStorageContainer {
console.log('ObjectStorage stopped.');
}
public async replaceAccessCredentials(
credentials: Array<{ accessKeyId: string; secretAccessKey: string }>,
): Promise<void> {
const nextCredentials = credentials.map((credential) => ({ ...credential }));
const previousCredentials = this.config.accessCredentials.map((credential) => ({
...credential,
}));
if (this.smartstorageInstance) {
await this.smartstorageInstance.replaceCredentials(nextCredentials);
}
this.config.accessCredentials = nextCredentials;
try {
await this.savePersistedAdminConfig();
} catch (error) {
this.config.accessCredentials = previousCredentials;
if (this.smartstorageInstance) {
await this.smartstorageInstance.replaceCredentials(previousCredentials);
await this.refreshManagementClient();
}
throw error;
}
if (this.smartstorageInstance) {
await this.refreshManagementClient();
}
}
public async listAccessCredentials(): Promise<Array<{ accessKeyId: string }>> {
if (!this.smartstorageInstance) {
return this.config.accessCredentials.map((credential) => ({
accessKeyId: credential.accessKeyId,
}));
}
return await this.smartstorageInstance.listCredentials();
}
// ── Management methods ──
public async listBuckets(): Promise<interfaces.data.IBucketInfo[]> {
const response = await this.s3Client.send(new plugins.ListBucketsCommand({}));
const buckets: interfaces.data.IBucketInfo[] = [];
for (const bucket of response.Buckets || []) {
const name = bucket.Name || '';
const creationDate = bucket.CreationDate?.getTime() || 0;
// Get object count and size for each bucket
let objectCount = 0;
let totalSizeBytes = 0;
let continuationToken: string | undefined;
do {
const listResp = await this.s3Client.send(
new plugins.ListObjectsV2Command({
Bucket: name,
ContinuationToken: continuationToken,
}),
);
for (const obj of listResp.Contents || []) {
objectCount++;
totalSizeBytes += obj.Size || 0;
}
continuationToken = listResp.IsTruncated ? listResp.NextContinuationToken : undefined;
} while (continuationToken);
buckets.push({ name, creationDate, objectCount, totalSizeBytes });
}
return buckets;
const summaries = await this.smartstorageInstance.listBucketSummaries();
return summaries.map((bucket) => ({
name: bucket.name,
creationDate: bucket.creationDate || 0,
objectCount: bucket.objectCount,
totalSizeBytes: bucket.totalSizeBytes,
}));
}
public async createBucket(bucketName: string): Promise<void> {
@@ -377,13 +376,7 @@ export class ObjectStorageContainer {
}
public async getServerStats(): Promise<interfaces.data.IServerStatus> {
const buckets = await this.listBuckets();
let totalObjectCount = 0;
let totalStorageBytes = 0;
for (const b of buckets) {
totalObjectCount += b.objectCount;
totalStorageBytes += b.totalSizeBytes;
}
const stats = await this.smartstorageInstance.getStorageStats();
return {
running: true,
@@ -391,15 +384,19 @@ export class ObjectStorageContainer {
uiPort: this.config.uiPort,
uptime: Math.floor((Date.now() - this.startedAt) / 1000),
startedAt: this.startedAt,
bucketCount: buckets.length,
totalObjectCount,
totalStorageBytes,
storageDirectory: this.config.storageDirectory,
bucketCount: stats.bucketCount,
totalObjectCount: stats.totalObjectCount,
totalStorageBytes: stats.totalStorageBytes,
storageDirectory: stats.storageDirectory,
region: this.config.region,
authEnabled: true,
};
}
public async getClusterHealth(): Promise<interfaces.data.IClusterHealth> {
return await this.smartstorageInstance.getClusterHealth();
}
public async getBucketPolicy(bucketName: string): Promise<string | null> {
try {
const response = await this.s3Client.send(
@@ -436,4 +433,186 @@ export class ObjectStorageContainer {
region: this.config.region,
};
}
public isReady(): boolean {
return Boolean(this.smartstorageInstance && this.s3Client && this.startedAt);
}
public async getOperationalHealth(): Promise<Record<string, unknown>> {
const clusterHealth = this.smartstorageInstance
? await this.getClusterHealth()
: { enabled: false };
const stats = this.smartstorageInstance
? await this.getServerStats()
: null;
const ready = this.isReady();
return {
ok: ready,
status: ready ? 'healthy' : 'starting',
startedAt: this.startedAt || null,
uptimeSeconds: this.startedAt ? Math.floor((Date.now() - this.startedAt) / 1000) : 0,
storageDirectory: this.config.storageDirectory,
stats,
cluster: clusterHealth,
};
}
public async getOperationalMetrics(): Promise<string> {
const stats = this.smartstorageInstance
? await this.smartstorageInstance.getStorageStats()
: null;
const clusterHealth = this.smartstorageInstance
? await this.smartstorageInstance.getClusterHealth()
: { enabled: false };
return [
'# HELP objectstorage_ready ObjectStorage readiness state.',
'# TYPE objectstorage_ready gauge',
`objectstorage_ready ${this.isReady() ? 1 : 0}`,
'# HELP objectstorage_buckets_total Runtime bucket count.',
'# TYPE objectstorage_buckets_total gauge',
`objectstorage_buckets_total ${stats?.bucketCount ?? 0}`,
'# HELP objectstorage_objects_total Runtime object count.',
'# TYPE objectstorage_objects_total gauge',
`objectstorage_objects_total ${stats?.totalObjectCount ?? 0}`,
'# HELP objectstorage_cluster_enabled Cluster mode enabled.',
'# TYPE objectstorage_cluster_enabled gauge',
`objectstorage_cluster_enabled ${clusterHealth.enabled ? 1 : 0}`,
'',
].join('\n');
}
private get persistedAdminConfigPath(): string {
return `${this.config.storageDirectory}/.objectstorage/admin-config.json`;
}
private async loadPersistedAdminConfig(): Promise<void> {
if (this.envAccessCredentialsProvided) {
return;
}
try {
const content = await Deno.readTextFile(this.persistedAdminConfigPath);
const persistedConfig = JSON.parse(content) as IPersistedAdminConfig;
const persistedCredentials = persistedConfig.accessCredentials;
if (!Array.isArray(persistedCredentials) || persistedCredentials.length === 0) {
return;
}
const validCredentials = persistedCredentials
.filter((credential) => credential?.accessKeyId && credential?.secretAccessKey)
.map((credential) => ({
accessKeyId: credential.accessKeyId,
secretAccessKey: credential.secretAccessKey,
}));
if (validCredentials.length === 0) {
return;
}
this.config.accessCredentials = validCredentials;
} catch (error) {
if (error instanceof Deno.errors.NotFound) {
return;
}
throw error;
}
}
private async savePersistedAdminConfig(): Promise<void> {
const dirPath = this.persistedAdminConfigPath.substring(
0,
this.persistedAdminConfigPath.lastIndexOf('/'),
);
await Deno.mkdir(dirPath, { recursive: true });
const persistedConfig: IPersistedAdminConfig = {
accessCredentials: this.config.accessCredentials,
};
await Deno.writeTextFile(
this.persistedAdminConfigPath,
JSON.stringify(persistedConfig, null, 2),
{ mode: 0o600 },
);
await this.restrictPersistedAdminConfigPermissions();
}
private async refreshManagementClient(): Promise<void> {
const descriptor = await this.smartstorageInstance.getStorageDescriptor();
this.s3Client = new plugins.S3Client({
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
region: this.config.region,
credentials: {
accessKeyId: descriptor.accessKey,
secretAccessKey: descriptor.accessSecret,
},
forcePathStyle: true,
});
}
private buildSmartstorageConfig(): any {
const smartstorageConfig: any = {
server: {
port: this.config.objstPort,
address: '0.0.0.0',
region: this.config.region,
},
storage: {
directory: this.config.storageDirectory,
},
auth: {
enabled: true,
credentials: this.config.accessCredentials,
},
};
if (this.config.clusterEnabled) {
smartstorageConfig.cluster = {
enabled: true,
nodeId: this.config.clusterNodeId || crypto.randomUUID().slice(0, 8),
quicPort: this.config.clusterQuicPort,
seedNodes: this.config.clusterSeedNodes,
erasure: {
dataShards: this.config.erasureDataShards,
parityShards: this.config.erasureParityShards,
chunkSizeBytes: this.config.erasureChunkSizeBytes,
},
drives: {
paths: this.config.drivePaths.length > 0
? this.config.drivePaths
: [this.config.storageDirectory],
},
heartbeatIntervalMs: this.config.clusterHeartbeatIntervalMs,
heartbeatTimeoutMs: this.config.clusterHeartbeatTimeoutMs,
};
}
return smartstorageConfig;
}
private assertSecureStartupConfig(): void {
const allowInsecureDefaults = Deno.env.get('OBJST_ALLOW_INSECURE_DEFAULTS') === 'true';
const usesDefaultAdminPassword = this.config.adminPassword === 'admin';
const usesDefaultAccessCredential = this.config.accessCredentials.some((credential) => {
return credential.accessKeyId === 'admin' && credential.secretAccessKey === 'admin';
});
const looksLikePersistentProductionStorage = this.config.storageDirectory === '/data';
if (
looksLikePersistentProductionStorage &&
!allowInsecureDefaults &&
(usesDefaultAdminPassword || usesDefaultAccessCredential)
) {
throw new Error(
'Refusing to start with default admin credentials on persistent /data storage. Set OBJST_ADMIN_PASSWORD and OBJST_ACCESS_KEY/OBJST_SECRET_KEY, or set OBJST_ALLOW_INSECURE_DEFAULTS=true for disposable development.',
);
}
}
private async restrictPersistedAdminConfigPermissions(): Promise<void> {
try {
await Deno.chmod(this.persistedAdminConfigPath, 0o600);
} catch {
// chmod is not available on every platform Deno supports.
}
}
}