Files
smartstorage/ts/index.ts
T

713 lines
19 KiB
TypeScript

import * as plugins from './plugins.js';
import * as paths from './paths.js';
/**
* Authentication configuration
*/
export interface IStorageCredential {
accessKeyId: string;
secretAccessKey: string;
bucketName?: string;
region?: string;
}
export interface IStorageCredentialMetadata {
accessKeyId: string;
bucketName?: string;
region?: string;
}
/**
* Authentication configuration
*/
export interface IAuthConfig {
enabled: boolean;
credentials: IStorageCredential[];
}
/**
* CORS configuration
*/
export interface ICorsConfig {
enabled: boolean;
allowedOrigins?: string[];
allowedMethods?: string[];
allowedHeaders?: string[];
exposedHeaders?: string[];
maxAge?: number;
allowCredentials?: boolean;
}
/**
* Logging configuration
*/
export interface ILoggingConfig {
level?: 'error' | 'warn' | 'info' | 'debug';
format?: 'text' | 'json';
enabled?: boolean;
}
/**
* Request limits configuration
*/
export interface ILimitsConfig {
maxObjectSize?: number;
maxMetadataSize?: number;
requestTimeout?: number;
}
/**
* Multipart upload configuration
*/
export interface IMultipartConfig {
expirationDays?: number;
cleanupIntervalMinutes?: number;
}
/**
* Server configuration
*/
export interface IServerConfig {
port?: number;
address?: string;
silent?: boolean;
region?: string;
}
/**
* Storage configuration
*/
export interface IStorageConfig {
directory?: string;
cleanSlate?: boolean;
}
/**
* Erasure coding configuration
*/
export interface IErasureConfig {
dataShards?: number;
parityShards?: number;
chunkSizeBytes?: number;
}
/**
* Drive configuration for multi-drive support
*/
export interface IDriveConfig {
paths: string[];
}
/**
* Cluster configuration for distributed mode
*/
export interface IClusterConfig {
enabled: boolean;
nodeId?: string;
quicPort?: number;
seedNodes?: string[];
erasure?: IErasureConfig;
drives?: IDriveConfig;
heartbeatIntervalMs?: number;
heartbeatTimeoutMs?: number;
}
/**
* Complete smartstorage configuration
*/
export interface ISmartStorageConfig {
server?: IServerConfig;
storage?: IStorageConfig;
auth?: IAuthConfig;
cors?: ICorsConfig;
logging?: ILoggingConfig;
limits?: ILimitsConfig;
multipart?: IMultipartConfig;
cluster?: IClusterConfig;
}
/**
* Logical bucket stats maintained by the Rust runtime.
* Values are initialized from native storage on startup and updated on smartstorage mutations.
*/
export interface IBucketSummary {
name: string;
objectCount: number;
totalSizeBytes: number;
creationDate?: number;
}
/**
* Filesystem-level capacity snapshot for the storage directory or configured drive path.
*/
export interface IStorageLocationSummary {
path: string;
totalBytes?: number;
availableBytes?: number;
usedBytes?: number;
}
/**
* Runtime storage stats served by the Rust core without issuing S3 list calls.
*/
export interface IStorageStats {
bucketCount: number;
totalObjectCount: number;
totalStorageBytes: number;
buckets: IBucketSummary[];
storageDirectory: string;
storageLocations?: IStorageLocationSummary[];
}
export interface IBucketTenantInput {
bucketName: string;
accessKeyId?: string;
secretAccessKey?: string;
region?: string;
}
export interface IDeleteBucketTenantInput {
bucketName: string;
accessKeyId?: string;
}
export interface IBucketTenantMetadata {
bucketName: string;
accessKeyId: string;
region?: string;
}
export interface IBucketTenantDescriptor extends plugins.tsclass.storage.IS3Descriptor {
endpoint: string;
port: number;
region: string;
bucket: string;
bucketName: string;
accessKeyId: string;
secretAccessKey: string;
accessKey: string;
accessSecret: string;
useSsl: boolean;
ssl: boolean;
env: Record<string, string>;
}
export interface IBucketExportObject {
key: string;
size: number;
md5: string;
metadata: Record<string, string>;
dataHex: string;
}
export interface IBucketExport {
format: 'smartstorage.bucket.v1';
bucketName: string;
exportedAt: number;
objects: IBucketExportObject[];
}
export interface IExportBucketInput {
bucketName: string;
}
export interface IImportBucketInput {
bucketName: string;
source: IBucketExport;
}
export interface ISmartStorageHealth {
ok: boolean;
running: boolean;
storageDirectory: string;
auth: {
enabled: boolean;
credentialCount: number;
tenantCredentialCount: number;
};
bucketCount: number;
objectCount: number;
totalBytes: number;
cluster: IClusterHealth;
}
export interface ISmartStorageMetrics {
bucketCount: number;
objectCount: number;
totalBytes: number;
authCredentialCount: number;
tenantCredentialCount: number;
clusterEnabled: boolean;
prometheusText: string;
}
/**
* Known peer status from the local node's current cluster view.
*/
export interface IClusterPeerHealth {
nodeId: string;
status: 'online' | 'suspect' | 'offline';
quicAddress?: string;
s3Address?: string;
driveCount?: number;
lastHeartbeat?: number;
missedHeartbeats?: number;
}
/**
* Local drive health as measured by smartstorage's runtime probes.
*/
export interface IClusterDriveHealth {
index: number;
path: string;
status: 'online' | 'degraded' | 'offline' | 'healing';
totalBytes?: number;
usedBytes?: number;
availableBytes?: number;
errorCount?: number;
lastError?: string;
lastCheck?: number;
erasureSetId?: number;
}
export interface IClusterErasureHealth {
dataShards: number;
parityShards: number;
chunkSizeBytes: number;
totalShards: number;
readQuorum: number;
writeQuorum: number;
erasureSetCount: number;
}
export interface IClusterRepairHealth {
active: boolean;
scanIntervalMs?: number;
lastRunStartedAt?: number;
lastRunCompletedAt?: number;
lastDurationMs?: number;
shardsChecked?: number;
shardsHealed?: number;
failed?: number;
lastError?: string;
}
/**
* Cluster runtime health from the Rust core.
* When clustering is disabled, the response is `{ enabled: false }`.
*/
export interface IClusterHealth {
enabled: boolean;
nodeId?: string;
quorumHealthy?: boolean;
majorityHealthy?: boolean;
peers?: IClusterPeerHealth[];
drives?: IClusterDriveHealth[];
erasure?: IClusterErasureHealth;
repairs?: IClusterRepairHealth;
}
/**
* Default configuration values
*/
const DEFAULT_CONFIG: ISmartStorageConfig = {
server: {
port: 3000,
address: '0.0.0.0',
silent: false,
region: 'us-east-1',
},
storage: {
directory: paths.bucketsDir,
cleanSlate: false,
},
auth: {
enabled: false,
credentials: [
{
accessKeyId: 'STORAGE',
secretAccessKey: 'STORAGE',
},
],
},
cors: {
enabled: false,
allowedOrigins: ['*'],
allowedMethods: ['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'],
allowedHeaders: ['*'],
exposedHeaders: ['ETag', 'x-amz-request-id', 'x-amz-version-id'],
maxAge: 86400,
allowCredentials: false,
},
logging: {
level: 'info',
format: 'text',
enabled: true,
},
limits: {
maxObjectSize: 5 * 1024 * 1024 * 1024, // 5GB
maxMetadataSize: 2048,
requestTimeout: 300000, // 5 minutes
},
multipart: {
expirationDays: 7,
cleanupIntervalMinutes: 60,
},
};
/**
* Merge user config with defaults (deep merge)
*/
function mergeConfig(userConfig: ISmartStorageConfig): Required<ISmartStorageConfig> {
return {
server: {
...DEFAULT_CONFIG.server!,
...(userConfig.server || {}),
},
storage: {
...DEFAULT_CONFIG.storage!,
...(userConfig.storage || {}),
},
auth: {
...DEFAULT_CONFIG.auth!,
...(userConfig.auth || {}),
},
cors: {
...DEFAULT_CONFIG.cors!,
...(userConfig.cors || {}),
},
logging: {
...DEFAULT_CONFIG.logging!,
...(userConfig.logging || {}),
},
limits: {
...DEFAULT_CONFIG.limits!,
...(userConfig.limits || {}),
},
multipart: {
...DEFAULT_CONFIG.multipart!,
...(userConfig.multipart || {}),
},
...(userConfig.cluster ? { cluster: userConfig.cluster } : {}),
} as Required<ISmartStorageConfig>;
}
function createAccessKeyId(): string {
return `SS${plugins.crypto.randomBytes(10).toString('hex').toUpperCase()}`;
}
function createSecretAccessKey(): string {
return plugins.crypto.randomBytes(32).toString('hex');
}
/**
* IPC command type map for RustBridge
*/
type TRustStorageCommands = {
start: { params: { config: Required<ISmartStorageConfig> }; result: {} };
stop: { params: {}; result: {} };
createBucket: { params: { name: string }; result: {} };
createBucketTenant: {
params: {
bucketName: string;
accessKeyId: string;
secretAccessKey: string;
region?: string;
};
result: IStorageCredential;
};
deleteBucketTenant: {
params: { bucketName: string; accessKeyId?: string };
result: {};
};
rotateBucketTenantCredentials: {
params: {
bucketName: string;
accessKeyId: string;
secretAccessKey: string;
region?: string;
};
result: IStorageCredential;
};
listBucketTenants: { params: {}; result: IBucketTenantMetadata[] };
getBucketTenantCredential: {
params: { bucketName: string };
result: IStorageCredential;
};
exportBucket: { params: { bucketName: string }; result: IBucketExport };
importBucket: { params: { bucketName: string; source: IBucketExport }; result: {} };
getStorageStats: { params: {}; result: IStorageStats };
listBucketSummaries: { params: {}; result: IBucketSummary[] };
listCredentials: { params: {}; result: IStorageCredentialMetadata[] };
replaceCredentials: { params: { credentials: IStorageCredential[] }; result: {} };
getClusterHealth: { params: {}; result: IClusterHealth };
};
/**
* Main SmartStorage class - production-ready S3-compatible storage server
*/
export class SmartStorage {
// STATIC
public static async createAndStart(configArg: ISmartStorageConfig = {}) {
const smartStorageInstance = new SmartStorage(configArg);
await smartStorageInstance.start();
return smartStorageInstance;
}
// INSTANCE
public config: Required<ISmartStorageConfig>;
private bridge: InstanceType<typeof plugins.RustBridge<TRustStorageCommands>>;
private running = false;
constructor(configArg: ISmartStorageConfig = {}) {
this.config = mergeConfig(configArg);
this.bridge = new plugins.RustBridge<TRustStorageCommands>({
binaryName: 'ruststorage',
localPaths: [
plugins.path.join(paths.packageDir, 'dist_rust', 'ruststorage'),
],
readyTimeoutMs: 30000,
requestTimeoutMs: 300000,
});
}
public async start() {
const spawned = await this.bridge.spawn();
if (!spawned) {
throw new Error('Failed to spawn ruststorage binary. Make sure it is compiled (pnpm build).');
}
await this.bridge.sendCommand('start', { config: this.config });
this.running = true;
if (!this.config.server.silent) {
console.log('storage server is running');
}
}
public async getStorageDescriptor(
optionsArg?: Partial<plugins.tsclass.storage.IS3Descriptor>,
): Promise<plugins.tsclass.storage.IS3Descriptor> {
const cred = this.config.auth.credentials[0] || {
accessKeyId: 'STORAGE',
secretAccessKey: 'STORAGE',
};
const descriptor: plugins.tsclass.storage.IS3Descriptor = {
endpoint: this.config.server.address === '0.0.0.0' ? 'localhost' : this.config.server.address!,
port: this.config.server.port!,
useSsl: false,
accessKey: cred.accessKeyId,
accessSecret: cred.secretAccessKey,
bucketName: '',
};
return {
...descriptor,
...(optionsArg ? optionsArg : {}),
};
}
private getEndpoint(): string {
return this.config.server.address === '0.0.0.0' ? 'localhost' : this.config.server.address!;
}
private buildBucketTenantDescriptor(
credential: IStorageCredential,
bucketNameArg: string,
): IBucketTenantDescriptor {
const bucketName = credential.bucketName || bucketNameArg;
const region = credential.region || this.config.server.region || 'us-east-1';
const endpoint = this.getEndpoint();
const port = this.config.server.port!;
const useSsl = false;
return {
endpoint,
port,
region,
bucket: bucketName,
bucketName,
accessKeyId: credential.accessKeyId,
secretAccessKey: credential.secretAccessKey,
accessKey: credential.accessKeyId,
accessSecret: credential.secretAccessKey,
useSsl,
ssl: useSsl,
env: {
S3_ENDPOINT: endpoint,
S3_PORT: String(port),
S3_REGION: region,
S3_BUCKET: bucketName,
S3_ACCESS_KEY_ID: credential.accessKeyId,
S3_SECRET_ACCESS_KEY: credential.secretAccessKey,
S3_USE_SSL: String(useSsl),
AWS_ACCESS_KEY_ID: credential.accessKeyId,
AWS_SECRET_ACCESS_KEY: credential.secretAccessKey,
AWS_REGION: region,
},
};
}
private assertTenantAuthEnabled(): void {
if (!this.config.auth.enabled) {
throw new Error('Bucket tenant APIs require auth.enabled=true.');
}
}
public async createBucket(bucketNameArg: string) {
await this.bridge.sendCommand('createBucket', { name: bucketNameArg });
return { name: bucketNameArg };
}
public async createBucketTenant(
tenantArg: IBucketTenantInput,
): Promise<IBucketTenantDescriptor> {
this.assertTenantAuthEnabled();
const credential = await this.bridge.sendCommand('createBucketTenant', {
bucketName: tenantArg.bucketName,
accessKeyId: tenantArg.accessKeyId || createAccessKeyId(),
secretAccessKey: tenantArg.secretAccessKey || createSecretAccessKey(),
region: tenantArg.region || this.config.server.region,
});
return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName);
}
public async deleteBucketTenant(tenantArg: IDeleteBucketTenantInput): Promise<void> {
this.assertTenantAuthEnabled();
await this.bridge.sendCommand('deleteBucketTenant', tenantArg);
}
public async rotateBucketTenantCredentials(
tenantArg: IBucketTenantInput,
): Promise<IBucketTenantDescriptor> {
this.assertTenantAuthEnabled();
const credential = await this.bridge.sendCommand('rotateBucketTenantCredentials', {
bucketName: tenantArg.bucketName,
accessKeyId: tenantArg.accessKeyId || createAccessKeyId(),
secretAccessKey: tenantArg.secretAccessKey || createSecretAccessKey(),
region: tenantArg.region || this.config.server.region,
});
return this.buildBucketTenantDescriptor(credential, tenantArg.bucketName);
}
public async listBucketTenants(): Promise<IBucketTenantMetadata[]> {
this.assertTenantAuthEnabled();
return this.bridge.sendCommand('listBucketTenants', {});
}
public async getBucketTenantDescriptor(optionsArg: {
bucketName: string;
}): Promise<IBucketTenantDescriptor> {
this.assertTenantAuthEnabled();
const credential = await this.bridge.sendCommand('getBucketTenantCredential', {
bucketName: optionsArg.bucketName,
});
return this.buildBucketTenantDescriptor(credential, optionsArg.bucketName);
}
public async exportBucket(optionsArg: IExportBucketInput): Promise<IBucketExport> {
return this.bridge.sendCommand('exportBucket', { bucketName: optionsArg.bucketName });
}
public async importBucket(optionsArg: IImportBucketInput): Promise<void> {
await this.bridge.sendCommand('importBucket', optionsArg);
}
public async getStorageStats(): Promise<IStorageStats> {
return this.bridge.sendCommand('getStorageStats', {});
}
public async listBucketSummaries(): Promise<IBucketSummary[]> {
return this.bridge.sendCommand('listBucketSummaries', {});
}
public async listCredentials(): Promise<IStorageCredentialMetadata[]> {
return this.bridge.sendCommand('listCredentials', {});
}
public async replaceCredentials(credentials: IStorageCredential[]): Promise<void> {
await this.bridge.sendCommand('replaceCredentials', { credentials });
this.config.auth.credentials = credentials.map((credential) => ({ ...credential }));
}
public async getClusterHealth(): Promise<IClusterHealth> {
return this.bridge.sendCommand('getClusterHealth', {});
}
public async getHealth(): Promise<ISmartStorageHealth> {
if (!this.running) {
return {
ok: false,
running: false,
storageDirectory: this.config.storage.directory || paths.bucketsDir,
auth: {
enabled: this.config.auth.enabled,
credentialCount: this.config.auth.credentials.length,
tenantCredentialCount: 0,
},
bucketCount: 0,
objectCount: 0,
totalBytes: 0,
cluster: { enabled: false },
};
}
const [stats, credentials, tenants, cluster] = await Promise.all([
this.getStorageStats(),
this.listCredentials(),
this.config.auth.enabled ? this.listBucketTenants() : Promise.resolve([]),
this.getClusterHealth(),
]);
return {
ok: true,
running: true,
storageDirectory: stats.storageDirectory,
auth: {
enabled: this.config.auth.enabled,
credentialCount: credentials.length,
tenantCredentialCount: tenants.length,
},
bucketCount: stats.bucketCount,
objectCount: stats.totalObjectCount,
totalBytes: stats.totalStorageBytes,
cluster,
};
}
public async getMetrics(): Promise<ISmartStorageMetrics> {
const health = await this.getHealth();
const clusterEnabled = health.cluster.enabled;
return {
bucketCount: health.bucketCount,
objectCount: health.objectCount,
totalBytes: health.totalBytes,
authCredentialCount: health.auth.credentialCount,
tenantCredentialCount: health.auth.tenantCredentialCount,
clusterEnabled,
prometheusText: [
'# HELP smartstorage_buckets_total Runtime bucket count.',
'# TYPE smartstorage_buckets_total gauge',
`smartstorage_buckets_total ${health.bucketCount}`,
'# HELP smartstorage_objects_total Runtime object count.',
'# TYPE smartstorage_objects_total gauge',
`smartstorage_objects_total ${health.objectCount}`,
'# HELP smartstorage_storage_bytes_total Runtime storage bytes.',
'# TYPE smartstorage_storage_bytes_total gauge',
`smartstorage_storage_bytes_total ${health.totalBytes}`,
'# HELP smartstorage_tenant_credentials_total Scoped bucket tenant credential count.',
'# TYPE smartstorage_tenant_credentials_total gauge',
`smartstorage_tenant_credentials_total ${health.auth.tenantCredentialCount}`,
'# HELP smartstorage_cluster_enabled Cluster mode enabled.',
'# TYPE smartstorage_cluster_enabled gauge',
`smartstorage_cluster_enabled ${clusterEnabled ? 1 : 0}`,
].join('\n'),
};
}
public async stop() {
await this.bridge.sendCommand('stop', {});
this.bridge.kill();
this.running = false;
}
}