import * as plugins from './plugins.js'; import * as paths from './paths.js'; /** * Authentication configuration */ export interface IStorageCredential { accessKeyId: string; secretAccessKey: string; bucketName?: string; region?: string; } export interface IStorageCredentialMetadata { accessKeyId: string; bucketName?: string; region?: string; } /** * Authentication configuration */ export interface IAuthConfig { enabled: boolean; credentials: IStorageCredential[]; } /** * CORS configuration */ export interface ICorsConfig { enabled: boolean; allowedOrigins?: string[]; allowedMethods?: string[]; allowedHeaders?: string[]; exposedHeaders?: string[]; maxAge?: number; allowCredentials?: boolean; } /** * Logging configuration */ export interface ILoggingConfig { level?: 'error' | 'warn' | 'info' | 'debug'; format?: 'text' | 'json'; enabled?: boolean; } /** * Request limits configuration */ export interface ILimitsConfig { maxObjectSize?: number; maxMetadataSize?: number; requestTimeout?: number; } /** * Multipart upload configuration */ export interface IMultipartConfig { expirationDays?: number; cleanupIntervalMinutes?: number; } /** * Server configuration */ export interface IServerConfig { port?: number; address?: string; silent?: boolean; region?: string; } /** * Storage configuration */ export interface IStorageConfig { directory?: string; cleanSlate?: boolean; } /** * Erasure coding configuration */ export interface IErasureConfig { dataShards?: number; parityShards?: number; chunkSizeBytes?: number; } /** * Drive configuration for multi-drive support */ export interface IDriveConfig { paths: string[]; } /** * Cluster configuration for distributed mode */ export interface IClusterConfig { enabled: boolean; nodeId?: string; quicPort?: number; seedNodes?: string[]; erasure?: IErasureConfig; drives?: IDriveConfig; heartbeatIntervalMs?: number; heartbeatTimeoutMs?: number; } /** * Complete smartstorage configuration */ export interface ISmartStorageConfig { server?: IServerConfig; storage?: IStorageConfig; auth?: IAuthConfig; cors?: ICorsConfig; logging?: ILoggingConfig; limits?: ILimitsConfig; multipart?: IMultipartConfig; cluster?: IClusterConfig; } /** * Logical bucket stats maintained by the Rust runtime. * Values are initialized from native storage on startup and updated on smartstorage mutations. */ export interface IBucketSummary { name: string; objectCount: number; totalSizeBytes: number; creationDate?: number; } /** * Filesystem-level capacity snapshot for the storage directory or configured drive path. */ export interface IStorageLocationSummary { path: string; totalBytes?: number; availableBytes?: number; usedBytes?: number; } /** * Runtime storage stats served by the Rust core without issuing S3 list calls. */ export interface IStorageStats { bucketCount: number; totalObjectCount: number; totalStorageBytes: number; buckets: IBucketSummary[]; storageDirectory: string; storageLocations?: IStorageLocationSummary[]; } export interface IBucketTenantInput { bucketName: string; accessKeyId?: string; secretAccessKey?: string; region?: string; } export interface IDeleteBucketTenantInput { bucketName: string; accessKeyId?: string; } export interface IBucketTenantMetadata { bucketName: string; accessKeyId: string; region?: string; } export interface IBucketTenantDescriptor extends plugins.tsclass.storage.IS3Descriptor { endpoint: string; port: number; region: string; bucket: string; bucketName: string; accessKeyId: string; secretAccessKey: string; accessKey: string; accessSecret: string; useSsl: boolean; ssl: boolean; env: Record; } export interface IBucketExportObject { key: string; size: number; md5: string; metadata: Record; dataHex: string; } export interface IBucketExport { format: 'smartstorage.bucket.v1'; bucketName: string; exportedAt: number; objects: IBucketExportObject[]; } export interface IExportBucketInput { bucketName: string; } export interface IImportBucketInput { bucketName: string; source: IBucketExport; } export interface ISmartStorageHealth { ok: boolean; running: boolean; storageDirectory: string; auth: { enabled: boolean; credentialCount: number; tenantCredentialCount: number; }; bucketCount: number; objectCount: number; totalBytes: number; cluster: IClusterHealth; } export interface ISmartStorageMetrics { bucketCount: number; objectCount: number; totalBytes: number; authCredentialCount: number; tenantCredentialCount: number; clusterEnabled: boolean; prometheusText: string; } /** * Known peer status from the local node's current cluster view. */ export interface IClusterPeerHealth { nodeId: string; status: 'online' | 'suspect' | 'offline'; quicAddress?: string; s3Address?: string; driveCount?: number; lastHeartbeat?: number; missedHeartbeats?: number; } /** * Local drive health as measured by smartstorage's runtime probes. */ export interface IClusterDriveHealth { index: number; path: string; status: 'online' | 'degraded' | 'offline' | 'healing'; totalBytes?: number; usedBytes?: number; availableBytes?: number; errorCount?: number; lastError?: string; lastCheck?: number; erasureSetId?: number; } export interface IClusterErasureHealth { dataShards: number; parityShards: number; chunkSizeBytes: number; totalShards: number; readQuorum: number; writeQuorum: number; erasureSetCount: number; } export interface IClusterRepairHealth { active: boolean; scanIntervalMs?: number; lastRunStartedAt?: number; lastRunCompletedAt?: number; lastDurationMs?: number; shardsChecked?: number; shardsHealed?: number; failed?: number; lastError?: string; } /** * Cluster runtime health from the Rust core. * When clustering is disabled, the response is `{ enabled: false }`. */ export interface IClusterHealth { enabled: boolean; nodeId?: string; quorumHealthy?: boolean; majorityHealthy?: boolean; peers?: IClusterPeerHealth[]; drives?: IClusterDriveHealth[]; erasure?: IClusterErasureHealth; repairs?: IClusterRepairHealth; } /** * Default configuration values */ const DEFAULT_CONFIG: ISmartStorageConfig = { server: { port: 3000, address: '0.0.0.0', silent: false, region: 'us-east-1', }, storage: { directory: paths.bucketsDir, cleanSlate: false, }, auth: { enabled: false, credentials: [ { accessKeyId: 'STORAGE', secretAccessKey: 'STORAGE', }, ], }, cors: { enabled: false, allowedOrigins: ['*'], allowedMethods: ['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'], allowedHeaders: ['*'], exposedHeaders: ['ETag', 'x-amz-request-id', 'x-amz-version-id'], maxAge: 86400, allowCredentials: false, }, logging: { level: 'info', format: 'text', enabled: true, }, limits: { maxObjectSize: 5 * 1024 * 1024 * 1024, // 5GB maxMetadataSize: 2048, requestTimeout: 300000, // 5 minutes }, multipart: { expirationDays: 7, cleanupIntervalMinutes: 60, }, }; /** * Merge user config with defaults (deep merge) */ function mergeConfig(userConfig: ISmartStorageConfig): Required { return { server: { ...DEFAULT_CONFIG.server!, ...(userConfig.server || {}), }, storage: { ...DEFAULT_CONFIG.storage!, ...(userConfig.storage || {}), }, auth: { ...DEFAULT_CONFIG.auth!, ...(userConfig.auth || {}), }, cors: { ...DEFAULT_CONFIG.cors!, ...(userConfig.cors || {}), }, logging: { ...DEFAULT_CONFIG.logging!, ...(userConfig.logging || {}), }, limits: { ...DEFAULT_CONFIG.limits!, ...(userConfig.limits || {}), }, multipart: { ...DEFAULT_CONFIG.multipart!, ...(userConfig.multipart || {}), }, ...(userConfig.cluster ? { cluster: userConfig.cluster } : {}), } as Required; } function createAccessKeyId(): string { return `SS${plugins.crypto.randomBytes(10).toString('hex').toUpperCase()}`; } function createSecretAccessKey(): string { return plugins.crypto.randomBytes(32).toString('hex'); } /** * IPC command type map for RustBridge */ type TRustStorageCommands = { start: { params: { config: Required }; result: {} }; stop: { params: {}; result: {} }; createBucket: { params: { name: string }; result: {} }; createBucketTenant: { params: { bucketName: string; accessKeyId: string; secretAccessKey: string; region?: string; }; result: IStorageCredential; }; deleteBucketTenant: { params: { bucketName: string; accessKeyId?: string }; result: {}; }; rotateBucketTenantCredentials: { params: { bucketName: string; accessKeyId: string; secretAccessKey: string; region?: string; }; result: IStorageCredential; }; listBucketTenants: { params: {}; result: IBucketTenantMetadata[] }; getBucketTenantCredential: { params: { bucketName: string }; result: IStorageCredential; }; exportBucket: { params: { bucketName: string }; result: IBucketExport }; importBucket: { params: { bucketName: string; source: IBucketExport }; result: {} }; getStorageStats: { params: {}; result: IStorageStats }; listBucketSummaries: { params: {}; result: IBucketSummary[] }; listCredentials: { params: {}; result: IStorageCredentialMetadata[] }; replaceCredentials: { params: { credentials: IStorageCredential[] }; result: {} }; getClusterHealth: { params: {}; result: IClusterHealth }; }; /** * Main SmartStorage class - production-ready S3-compatible storage server */ export class SmartStorage { // STATIC public static async createAndStart(configArg: ISmartStorageConfig = {}) { const smartStorageInstance = new SmartStorage(configArg); await smartStorageInstance.start(); return smartStorageInstance; } // INSTANCE public config: Required; private bridge: InstanceType>; private running = false; constructor(configArg: ISmartStorageConfig = {}) { this.config = mergeConfig(configArg); this.bridge = new plugins.RustBridge({ binaryName: 'ruststorage', localPaths: [ plugins.path.join(paths.packageDir, 'dist_rust', 'ruststorage'), ], readyTimeoutMs: 30000, requestTimeoutMs: 300000, }); } public async start() { const spawned = await this.bridge.spawn(); if (!spawned) { throw new Error('Failed to spawn ruststorage binary. Make sure it is compiled (pnpm build).'); } await this.bridge.sendCommand('start', { config: this.config }); this.running = true; if (!this.config.server.silent) { console.log('storage server is running'); } } public async getStorageDescriptor( optionsArg?: Partial, ): Promise { const cred = this.config.auth.credentials[0] || { accessKeyId: 'STORAGE', secretAccessKey: 'STORAGE', }; const descriptor: plugins.tsclass.storage.IS3Descriptor = { endpoint: this.config.server.address === '0.0.0.0' ? 'localhost' : this.config.server.address!, port: this.config.server.port!, useSsl: false, accessKey: cred.accessKeyId, accessSecret: cred.secretAccessKey, bucketName: '', }; return { ...descriptor, ...(optionsArg ? optionsArg : {}), }; } private getEndpoint(): string { return this.config.server.address === '0.0.0.0' ? 'localhost' : this.config.server.address!; } private buildBucketTenantDescriptor( credential: IStorageCredential, bucketNameArg: string, ): IBucketTenantDescriptor { const bucketName = credential.bucketName || bucketNameArg; const region = credential.region || this.config.server.region || 'us-east-1'; const endpoint = this.getEndpoint(); const port = this.config.server.port!; const useSsl = false; return { endpoint, port, region, bucket: bucketName, bucketName, accessKeyId: credential.accessKeyId, secretAccessKey: credential.secretAccessKey, accessKey: credential.accessKeyId, accessSecret: credential.secretAccessKey, useSsl, ssl: useSsl, env: { S3_ENDPOINT: endpoint, S3_PORT: String(port), S3_REGION: region, S3_BUCKET: bucketName, S3_ACCESS_KEY_ID: credential.accessKeyId, S3_SECRET_ACCESS_KEY: credential.secretAccessKey, S3_USE_SSL: String(useSsl), AWS_ACCESS_KEY_ID: credential.accessKeyId, AWS_SECRET_ACCESS_KEY: credential.secretAccessKey, AWS_REGION: region, }, }; } private assertTenantAuthEnabled(): void { if (!this.config.auth.enabled) { throw new Error('Bucket tenant APIs require auth.enabled=true.'); } } public async createBucket(bucketNameArg: string) { await this.bridge.sendCommand('createBucket', { name: bucketNameArg }); return { name: bucketNameArg }; } public async createBucketTenant( tenantArg: IBucketTenantInput, ): Promise { this.assertTenantAuthEnabled(); const credential = await this.bridge.sendCommand('createBucketTenant', { bucketName: tenantArg.bucketName, accessKeyId: tenantArg.accessKeyId || createAccessKeyId(), secretAccessKey: tenantArg.secretAccessKey || createSecretAccessKey(), region: tenantArg.region || this.config.server.region, }); return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName); } public async deleteBucketTenant(tenantArg: IDeleteBucketTenantInput): Promise { this.assertTenantAuthEnabled(); await this.bridge.sendCommand('deleteBucketTenant', tenantArg); } public async rotateBucketTenantCredentials( tenantArg: IBucketTenantInput, ): Promise { this.assertTenantAuthEnabled(); const credential = await this.bridge.sendCommand('rotateBucketTenantCredentials', { bucketName: tenantArg.bucketName, accessKeyId: tenantArg.accessKeyId || createAccessKeyId(), secretAccessKey: tenantArg.secretAccessKey || createSecretAccessKey(), region: tenantArg.region || this.config.server.region, }); return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName); } public async listBucketTenants(): Promise { this.assertTenantAuthEnabled(); return this.bridge.sendCommand('listBucketTenants', {}); } public async getBucketTenantDescriptor(optionsArg: { bucketName: string; }): Promise { this.assertTenantAuthEnabled(); const credential = await this.bridge.sendCommand('getBucketTenantCredential', { bucketName: optionsArg.bucketName, }); return this.buildBucketTenantDescriptor(credential, optionsArg.bucketName); } public async exportBucket(optionsArg: IExportBucketInput): Promise { return this.bridge.sendCommand('exportBucket', { bucketName: optionsArg.bucketName }); } public async importBucket(optionsArg: IImportBucketInput): Promise { await this.bridge.sendCommand('importBucket', optionsArg); } public async getStorageStats(): Promise { return this.bridge.sendCommand('getStorageStats', {}); } public async listBucketSummaries(): Promise { return this.bridge.sendCommand('listBucketSummaries', {}); } public async listCredentials(): Promise { return this.bridge.sendCommand('listCredentials', {}); } public async replaceCredentials(credentials: IStorageCredential[]): Promise { await this.bridge.sendCommand('replaceCredentials', { credentials }); this.config.auth.credentials = credentials.map((credential) => ({ ...credential })); } public async getClusterHealth(): Promise { return this.bridge.sendCommand('getClusterHealth', {}); } public async getHealth(): Promise { if (!this.running) { return { ok: false, running: false, storageDirectory: this.config.storage.directory || paths.bucketsDir, auth: { enabled: this.config.auth.enabled, credentialCount: this.config.auth.credentials.length, tenantCredentialCount: 0, }, bucketCount: 0, objectCount: 0, totalBytes: 0, cluster: { enabled: false }, }; } const [stats, credentials, tenants, cluster] = await Promise.all([ this.getStorageStats(), this.listCredentials(), this.config.auth.enabled ? this.listBucketTenants() : Promise.resolve([]), this.getClusterHealth(), ]); return { ok: true, running: true, storageDirectory: stats.storageDirectory, auth: { enabled: this.config.auth.enabled, credentialCount: credentials.length, tenantCredentialCount: tenants.length, }, bucketCount: stats.bucketCount, objectCount: stats.totalObjectCount, totalBytes: stats.totalStorageBytes, cluster, }; } public async getMetrics(): Promise { const health = await this.getHealth(); const clusterEnabled = health.cluster.enabled; return { bucketCount: health.bucketCount, objectCount: health.objectCount, totalBytes: health.totalBytes, authCredentialCount: health.auth.credentialCount, tenantCredentialCount: health.auth.tenantCredentialCount, clusterEnabled, prometheusText: [ '# HELP smartstorage_buckets_total Runtime bucket count.', '# TYPE smartstorage_buckets_total gauge', `smartstorage_buckets_total ${health.bucketCount}`, '# HELP smartstorage_objects_total Runtime object count.', '# TYPE smartstorage_objects_total gauge', `smartstorage_objects_total ${health.objectCount}`, '# HELP smartstorage_storage_bytes_total Runtime storage bytes.', '# TYPE smartstorage_storage_bytes_total gauge', `smartstorage_storage_bytes_total ${health.totalBytes}`, '# HELP smartstorage_tenant_credentials_total Scoped bucket tenant credential count.', '# TYPE smartstorage_tenant_credentials_total gauge', `smartstorage_tenant_credentials_total ${health.auth.tenantCredentialCount}`, '# HELP smartstorage_cluster_enabled Cluster mode enabled.', '# TYPE smartstorage_cluster_enabled gauge', `smartstorage_cluster_enabled ${clusterEnabled ? 1 : 0}`, ].join('\n'), }; } public async stop() { await this.bridge.sendCommand('stop', {}); this.bridge.kill(); this.running = false; } }