update
This commit is contained in:
266
ts/cache/classes.secrets.scan.service.ts
vendored
Normal file
266
ts/cache/classes.secrets.scan.service.ts
vendored
Normal file
@@ -0,0 +1,266 @@
|
||||
import { logger } from '../logging.ts';
|
||||
import type { ConnectionManager } from '../classes/connectionmanager.ts';
|
||||
import { CachedSecret } from './documents/classes.cached.secret.ts';
|
||||
import { TTL } from './classes.cached.document.ts';
|
||||
import type { ISecret } from '../../ts_interfaces/data/secret.ts';
|
||||
|
||||
export interface IScanResult {
|
||||
connectionsScanned: number;
|
||||
secretsFound: number;
|
||||
errors: string[];
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Centralized secrets scanning service. Fetches all secrets from all
|
||||
* connections and upserts them into the CachedSecret collection.
|
||||
*/
|
||||
export class SecretsScanService {
|
||||
public lastScanTimestamp: number = 0;
|
||||
public lastScanResult: IScanResult | null = null;
|
||||
public isScanning: boolean = false;
|
||||
|
||||
private connectionManager: ConnectionManager;
|
||||
|
||||
constructor(connectionManager: ConnectionManager) {
|
||||
this.connectionManager = connectionManager;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert a single secret into the cache. If a doc with the same composite ID
|
||||
* already exists, update it in place; otherwise insert a new one.
|
||||
*/
|
||||
private async upsertSecret(secret: ISecret): Promise<void> {
|
||||
const id = CachedSecret.buildId(secret.connectionId, secret.scope, secret.scopeId, secret.key);
|
||||
const existing = await CachedSecret.getInstance({ id });
|
||||
if (existing) {
|
||||
existing.value = secret.value;
|
||||
existing.protected = secret.protected;
|
||||
existing.masked = secret.masked;
|
||||
existing.environment = secret.environment;
|
||||
existing.scopeName = secret.scopeName;
|
||||
existing.setTTL(TTL.HOURS_24);
|
||||
await existing.save();
|
||||
} else {
|
||||
const doc = CachedSecret.fromISecret(secret);
|
||||
await doc.save();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an array of secrets to cache using upsert logic.
|
||||
* Best-effort: individual failures are silently ignored.
|
||||
*/
|
||||
async saveSecrets(secrets: ISecret[]): Promise<void> {
|
||||
for (const secret of secrets) {
|
||||
try {
|
||||
await this.upsertSecret(secret);
|
||||
} catch {
|
||||
// Best-effort caching
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Full scan: iterate all connections, fetch all projects+groups,
|
||||
* fetch all secrets per entity, upsert CachedSecret docs.
|
||||
*/
|
||||
async fullScan(): Promise<IScanResult> {
|
||||
if (this.isScanning) {
|
||||
return {
|
||||
connectionsScanned: 0,
|
||||
secretsFound: 0,
|
||||
errors: ['Scan already in progress'],
|
||||
durationMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
this.isScanning = true;
|
||||
const startTime = Date.now();
|
||||
const errors: string[] = [];
|
||||
let totalSecrets = 0;
|
||||
let connectionsScanned = 0;
|
||||
|
||||
try {
|
||||
const connections = this.connectionManager.getConnections();
|
||||
for (const conn of connections) {
|
||||
try {
|
||||
const provider = this.connectionManager.getProvider(conn.id);
|
||||
connectionsScanned++;
|
||||
|
||||
// Scan project secrets
|
||||
try {
|
||||
const projects = await provider.getProjects();
|
||||
for (let i = 0; i < projects.length; i += 5) {
|
||||
const batch = projects.slice(i, i + 5);
|
||||
const results = await Promise.allSettled(
|
||||
batch.map(async (p) => {
|
||||
const secrets = await provider.getProjectSecrets(p.id);
|
||||
return secrets.map((s) => ({
|
||||
...s,
|
||||
scope: 'project' as const,
|
||||
scopeId: p.id,
|
||||
scopeName: p.fullPath || p.name,
|
||||
connectionId: conn.id,
|
||||
}));
|
||||
}),
|
||||
);
|
||||
for (const result of results) {
|
||||
if (result.status === 'fulfilled') {
|
||||
for (const secret of result.value) {
|
||||
try {
|
||||
await this.upsertSecret(secret);
|
||||
totalSecrets++;
|
||||
} catch (err) {
|
||||
errors.push(`Save secret ${secret.key}: ${err}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
errors.push(`Fetch project secrets: ${result.reason}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(`Fetch projects for ${conn.id}: ${err}`);
|
||||
}
|
||||
|
||||
// Scan group secrets
|
||||
try {
|
||||
const groups = await provider.getGroups();
|
||||
for (let i = 0; i < groups.length; i += 5) {
|
||||
const batch = groups.slice(i, i + 5);
|
||||
const results = await Promise.allSettled(
|
||||
batch.map(async (g) => {
|
||||
const secrets = await provider.getGroupSecrets(g.id);
|
||||
return secrets.map((s) => ({
|
||||
...s,
|
||||
scope: 'group' as const,
|
||||
scopeId: g.id,
|
||||
scopeName: g.fullPath || g.name,
|
||||
connectionId: conn.id,
|
||||
}));
|
||||
}),
|
||||
);
|
||||
for (const result of results) {
|
||||
if (result.status === 'fulfilled') {
|
||||
for (const secret of result.value) {
|
||||
try {
|
||||
await this.upsertSecret(secret);
|
||||
totalSecrets++;
|
||||
} catch (err) {
|
||||
errors.push(`Save secret ${secret.key}: ${err}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
errors.push(`Fetch group secrets: ${result.reason}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(`Fetch groups for ${conn.id}: ${err}`);
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(`Connection ${conn.id}: ${err}`);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.isScanning = false;
|
||||
}
|
||||
|
||||
const result: IScanResult = {
|
||||
connectionsScanned,
|
||||
secretsFound: totalSecrets,
|
||||
errors,
|
||||
durationMs: Date.now() - startTime,
|
||||
};
|
||||
|
||||
this.lastScanTimestamp = Date.now();
|
||||
this.lastScanResult = result;
|
||||
|
||||
logger.info(
|
||||
`Secrets scan complete: ${totalSecrets} secrets from ${connectionsScanned} connections in ${result.durationMs}ms` +
|
||||
(errors.length > 0 ? ` (${errors.length} errors)` : ''),
|
||||
);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan a single entity: delete existing cached secrets for that entity,
|
||||
* fetch fresh from provider, and save to cache.
|
||||
*/
|
||||
async scanEntity(
|
||||
connectionId: string,
|
||||
scope: 'project' | 'group',
|
||||
scopeId: string,
|
||||
scopeName?: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
// Delete existing cached secrets for this entity
|
||||
const existing = await CachedSecret.getInstances({
|
||||
connectionId,
|
||||
scope,
|
||||
scopeId,
|
||||
});
|
||||
for (const doc of existing) {
|
||||
await doc.delete();
|
||||
}
|
||||
|
||||
// Fetch fresh from provider
|
||||
const provider = this.connectionManager.getProvider(connectionId);
|
||||
const secrets = scope === 'project'
|
||||
? await provider.getProjectSecrets(scopeId)
|
||||
: await provider.getGroupSecrets(scopeId);
|
||||
|
||||
// Save to cache
|
||||
for (const s of secrets) {
|
||||
const doc = CachedSecret.fromISecret({
|
||||
...s,
|
||||
scope,
|
||||
scopeId,
|
||||
scopeName: scopeName || s.scopeName || '',
|
||||
connectionId,
|
||||
});
|
||||
await doc.save();
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(`scanEntity failed for ${connectionId}/${scope}/${scopeId}: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached secrets matching the filter criteria.
|
||||
*/
|
||||
async getCachedSecrets(filter: {
|
||||
connectionId: string;
|
||||
scope: 'project' | 'group';
|
||||
scopeId?: string;
|
||||
}): Promise<ISecret[]> {
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const query: any = {
|
||||
connectionId: filter.connectionId,
|
||||
scope: filter.scope,
|
||||
};
|
||||
if (filter.scopeId) {
|
||||
query.scopeId = filter.scopeId;
|
||||
}
|
||||
const docs = await CachedSecret.getInstances(query);
|
||||
// Filter out expired docs
|
||||
const now = Date.now();
|
||||
return docs
|
||||
.filter((d) => d.expiresAt > now)
|
||||
.map((d) => d.toISecret());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if non-expired cached data exists for the given connection+scope.
|
||||
*/
|
||||
async hasCachedData(connectionId: string, scope: 'project' | 'group'): Promise<boolean> {
|
||||
const docs = await CachedSecret.getInstances({
|
||||
connectionId,
|
||||
scope,
|
||||
expiresAt: { $gt: Date.now() },
|
||||
});
|
||||
return docs.length > 0;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user