update

ts/cache/classes.secrets.scan.service.ts

@@ -0,0 +1,266 @@
+import { logger } from '../logging.ts';
+import type { ConnectionManager } from '../classes/connectionmanager.ts';
+import { CachedSecret } from './documents/classes.cached.secret.ts';
+import { TTL } from './classes.cached.document.ts';
+import type { ISecret } from '../../ts_interfaces/data/secret.ts';
+
+export interface IScanResult {
+  connectionsScanned: number;
+  secretsFound: number;
+  errors: string[];
+  durationMs: number;
+}
+
+/**
+ * Centralized secrets scanning service. Fetches all secrets from all
+ * connections and upserts them into the CachedSecret collection.
+ */
+export class SecretsScanService {
+  public lastScanTimestamp: number = 0;
+  public lastScanResult: IScanResult | null = null;
+  public isScanning: boolean = false;
+
+  private connectionManager: ConnectionManager;
+
+  constructor(connectionManager: ConnectionManager) {
+    this.connectionManager = connectionManager;
+  }
+
+  /**
+   * Upsert a single secret into the cache. If a doc with the same composite ID
+   * already exists, update it in place; otherwise insert a new one.
+   */
+  private async upsertSecret(secret: ISecret): Promise<void> {
+    const id = CachedSecret.buildId(secret.connectionId, secret.scope, secret.scopeId, secret.key);
+    const existing = await CachedSecret.getInstance({ id });
+    if (existing) {
+      existing.value = secret.value;
+      existing.protected = secret.protected;
+      existing.masked = secret.masked;
+      existing.environment = secret.environment;
+      existing.scopeName = secret.scopeName;
+      existing.setTTL(TTL.HOURS_24);
+      await existing.save();
+    } else {
+      const doc = CachedSecret.fromISecret(secret);
+      await doc.save();
+    }
+  }
+
+  /**
+   * Save an array of secrets to cache using upsert logic.
+   * Best-effort: individual failures are silently ignored.
+   */
+  async saveSecrets(secrets: ISecret[]): Promise<void> {
+    for (const secret of secrets) {
+      try {
+        await this.upsertSecret(secret);
+      } catch {
+        // Best-effort caching
+      }
+    }
+  }
+
+  /**
+   * Full scan: iterate all connections, fetch all projects+groups,
+   * fetch all secrets per entity, upsert CachedSecret docs.
+   */
+  async fullScan(): Promise<IScanResult> {
+    if (this.isScanning) {
+      return {
+        connectionsScanned: 0,
+        secretsFound: 0,
+        errors: ['Scan already in progress'],
+        durationMs: 0,
+      };
+    }
+
+    this.isScanning = true;
+    const startTime = Date.now();
+    const errors: string[] = [];
+    let totalSecrets = 0;
+    let connectionsScanned = 0;
+
+    try {
+      const connections = this.connectionManager.getConnections();
+      for (const conn of connections) {
+        try {
+          const provider = this.connectionManager.getProvider(conn.id);
+          connectionsScanned++;
+
+          // Scan project secrets
+          try {
+            const projects = await provider.getProjects();
+            for (let i = 0; i < projects.length; i += 5) {
+              const batch = projects.slice(i, i + 5);
+              const results = await Promise.allSettled(
+                batch.map(async (p) => {
+                  const secrets = await provider.getProjectSecrets(p.id);
+                  return secrets.map((s) => ({
+                    ...s,
+                    scope: 'project' as const,
+                    scopeId: p.id,
+                    scopeName: p.fullPath || p.name,
+                    connectionId: conn.id,
+                  }));
+                }),
+              );
+              for (const result of results) {
+                if (result.status === 'fulfilled') {
+                  for (const secret of result.value) {
+                    try {
+                      await this.upsertSecret(secret);
+                      totalSecrets++;
+                    } catch (err) {
+                      errors.push(`Save secret ${secret.key}: ${err}`);
+                    }
+                  }
+                } else {
+                  errors.push(`Fetch project secrets: ${result.reason}`);
+                }
+              }
+            }
+          } catch (err) {
+            errors.push(`Fetch projects for ${conn.id}: ${err}`);
+          }
+
+          // Scan group secrets
+          try {
+            const groups = await provider.getGroups();
+            for (let i = 0; i < groups.length; i += 5) {
+              const batch = groups.slice(i, i + 5);
+              const results = await Promise.allSettled(
+                batch.map(async (g) => {
+                  const secrets = await provider.getGroupSecrets(g.id);
+                  return secrets.map((s) => ({
+                    ...s,
+                    scope: 'group' as const,
+                    scopeId: g.id,
+                    scopeName: g.fullPath || g.name,
+                    connectionId: conn.id,
+                  }));
+                }),
+              );
+              for (const result of results) {
+                if (result.status === 'fulfilled') {
+                  for (const secret of result.value) {
+                    try {
+                      await this.upsertSecret(secret);
+                      totalSecrets++;
+                    } catch (err) {
+                      errors.push(`Save secret ${secret.key}: ${err}`);
+                    }
+                  }
+                } else {
+                  errors.push(`Fetch group secrets: ${result.reason}`);
+                }
+              }
+            }
+          } catch (err) {
+            errors.push(`Fetch groups for ${conn.id}: ${err}`);
+          }
+        } catch (err) {
+          errors.push(`Connection ${conn.id}: ${err}`);
+        }
+      }
+    } finally {
+      this.isScanning = false;
+    }
+
+    const result: IScanResult = {
+      connectionsScanned,
+      secretsFound: totalSecrets,
+      errors,
+      durationMs: Date.now() - startTime,
+    };
+
+    this.lastScanTimestamp = Date.now();
+    this.lastScanResult = result;
+
+    logger.info(
+      `Secrets scan complete: ${totalSecrets} secrets from ${connectionsScanned} connections in ${result.durationMs}ms` +
+        (errors.length > 0 ? ` (${errors.length} errors)` : ''),
+    );
+
+    return result;
+  }
+
+  /**
+   * Scan a single entity: delete existing cached secrets for that entity,
+   * fetch fresh from provider, and save to cache.
+   */
+  async scanEntity(
+    connectionId: string,
+    scope: 'project' | 'group',
+    scopeId: string,
+    scopeName?: string,
+  ): Promise<void> {
+    try {
+      // Delete existing cached secrets for this entity
+      const existing = await CachedSecret.getInstances({
+        connectionId,
+        scope,
+        scopeId,
+      });
+      for (const doc of existing) {
+        await doc.delete();
+      }
+
+      // Fetch fresh from provider
+      const provider = this.connectionManager.getProvider(connectionId);
+      const secrets = scope === 'project'
+        ? await provider.getProjectSecrets(scopeId)
+        : await provider.getGroupSecrets(scopeId);
+
+      // Save to cache
+      for (const s of secrets) {
+        const doc = CachedSecret.fromISecret({
+          ...s,
+          scope,
+          scopeId,
+          scopeName: scopeName || s.scopeName || '',
+          connectionId,
+        });
+        await doc.save();
+      }
+    } catch (err) {
+      logger.error(`scanEntity failed for ${connectionId}/${scope}/${scopeId}: ${err}`);
+    }
+  }
+
+  /**
+   * Get cached secrets matching the filter criteria.
+   */
+  async getCachedSecrets(filter: {
+    connectionId: string;
+    scope: 'project' | 'group';
+    scopeId?: string;
+  }): Promise<ISecret[]> {
+    // deno-lint-ignore no-explicit-any
+    const query: any = {
+      connectionId: filter.connectionId,
+      scope: filter.scope,
+    };
+    if (filter.scopeId) {
+      query.scopeId = filter.scopeId;
+    }
+    const docs = await CachedSecret.getInstances(query);
+    // Filter out expired docs
+    const now = Date.now();
+    return docs
+      .filter((d) => d.expiresAt > now)
+      .map((d) => d.toISecret());
+  }
+
+  /**
+   * Check if non-expired cached data exists for the given connection+scope.
+   */
+  async hasCachedData(connectionId: string, scope: 'project' | 'group'): Promise<boolean> {
+    const docs = await CachedSecret.getInstances({
+      connectionId,
+      scope,
+      expiresAt: { $gt: Date.now() },
+    });
+    return docs.length > 0;
+  }
+}

ts/cache/documents/classes.cached.secret.ts

@@ -0,0 +1,81 @@
+import * as plugins from '../../plugins.ts';
+import { CacheDb } from '../classes.cachedb.ts';
+import { CachedDocument, TTL } from '../classes.cached.document.ts';
+import type { ISecret } from '../../../ts_interfaces/data/secret.ts';
+
+/**
+ * Cached secret data from git providers. TTL: 24 hours.
+ */
+@plugins.smartdata.Collection(() => CacheDb.getInstance().getDb())
+export class CachedSecret extends CachedDocument<CachedSecret> {
+  @plugins.smartdata.unI()
+  public id: string = '';
+
+  @plugins.smartdata.svDb()
+  public connectionId: string = '';
+
+  @plugins.smartdata.svDb()
+  public scope: 'project' | 'group' = 'project';
+
+  @plugins.smartdata.svDb()
+  public scopeId: string = '';
+
+  @plugins.smartdata.svDb()
+  public scopeName: string = '';
+
+  @plugins.smartdata.svDb()
+  public key: string = '';
+
+  @plugins.smartdata.svDb()
+  public value: string = '';
+
+  @plugins.smartdata.svDb()
+  public protected: boolean = false;
+
+  @plugins.smartdata.svDb()
+  public masked: boolean = false;
+
+  @plugins.smartdata.svDb()
+  public environment: string = '';
+
+  constructor() {
+    super();
+    this.setTTL(TTL.HOURS_24);
+  }
+
+  /** Build the composite unique ID */
+  static buildId(connectionId: string, scope: string, scopeId: string, key: string): string {
+    return `${connectionId}:${scope}:${scopeId}:${key}`;
+  }
+
+  /** Create a CachedSecret from an ISecret */
+  static fromISecret(secret: ISecret): CachedSecret {
+    const doc = new CachedSecret();
+    doc.id = CachedSecret.buildId(secret.connectionId, secret.scope, secret.scopeId, secret.key);
+    doc.connectionId = secret.connectionId;
+    doc.scope = secret.scope;
+    doc.scopeId = secret.scopeId;
+    doc.scopeName = secret.scopeName;
+    doc.key = secret.key;
+    doc.value = secret.value;
+    doc.protected = secret.protected;
+    doc.masked = secret.masked;
+    doc.environment = secret.environment;
+    return doc;
+  }
+
+  /** Convert back to ISecret */
+  toISecret(): ISecret {
+    return {
+      connectionId: this.connectionId,
+      scope: this.scope,
+      scopeId: this.scopeId,
+      scopeName: this.scopeName,
+      key: this.key,
+      value: this.value,
+      protected: this.protected,
+      masked: this.masked,
+      environment: this.environment,
+    };
+  }
+}

ts/cache/documents/index.ts

@@ -1 +1,2 @@
 export { CachedProject } from './classes.cached.project.ts';
+export { CachedSecret } from './classes.cached.secret.ts';

ts/cache/index.ts

@@ -2,4 +2,6 @@ export { CacheDb } from './classes.cachedb.ts';
 export type { ICacheDbOptions } from './classes.cachedb.ts';
 export { CachedDocument, TTL } from './classes.cached.document.ts';
 export { CacheCleaner } from './classes.cache.cleaner.ts';
+export { SecretsScanService } from './classes.secrets.scan.service.ts';
+export type { IScanResult } from './classes.secrets.scan.service.ts';
 export * from './documents/index.ts';

ts/classes/gitopsapp.ts

@@ -3,7 +3,7 @@ import { logger } from '../logging.ts';
 import { ConnectionManager } from './connectionmanager.ts';
 import { OpsServer } from '../opsserver/index.ts';
 import { StorageManager } from '../storage/index.ts';
-import { CacheDb, CacheCleaner, CachedProject } from '../cache/index.ts';
+import { CacheDb, CacheCleaner, CachedProject, CachedSecret, SecretsScanService } from '../cache/index.ts';
 import { resolvePaths } from '../paths.ts';
 
 /**
@@ -16,6 +16,8 @@ export class GitopsApp {
   public opsServer: OpsServer;
   public cacheDb: CacheDb;
   public cacheCleaner: CacheCleaner;
+  public secretsScanService!: SecretsScanService;
+  private scanIntervalId: number | null = null;
 
   constructor() {
     const paths = resolvePaths();
@@ -32,6 +34,7 @@ export class GitopsApp {
     });
     this.cacheCleaner = new CacheCleaner(this.cacheDb);
     this.cacheCleaner.registerClass(CachedProject);
+    this.cacheCleaner.registerClass(CachedSecret);
 
     this.opsServer = new OpsServer(this);
   }
@@ -45,6 +48,20 @@ export class GitopsApp {
     // Initialize connection manager (loads saved connections)
     await this.connectionManager.init();
 
+    // Initialize secrets scan service with 24h auto-scan
+    this.secretsScanService = new SecretsScanService(this.connectionManager);
+    const SCAN_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
+    this.scanIntervalId = setInterval(() => {
+      this.secretsScanService.fullScan().catch((err) => {
+        logger.error(`Scheduled secrets scan failed: ${err}`);
+      });
+    }, SCAN_INTERVAL_MS);
+    Deno.unrefTimer(this.scanIntervalId);
+    // Fire-and-forget initial scan (doesn't block startup)
+    this.secretsScanService.fullScan().catch((err) => {
+      logger.error(`Initial secrets scan failed: ${err}`);
+    });
+
     // Start CacheCleaner
     this.cacheCleaner.start();
 
@@ -56,6 +73,10 @@ export class GitopsApp {
 
   async stop(): Promise<void> {
     logger.info('Shutting down GitOps...');
+    if (this.scanIntervalId !== null) {
+      clearInterval(this.scanIntervalId);
+      this.scanIntervalId = null;
+    }
     await this.opsServer.stop();
     this.cacheCleaner.stop();
     await this.cacheDb.stop();

ts/opsserver/classes.opsserver.ts

@@ -18,6 +18,7 @@ export class OpsServer {
   public pipelinesHandler!: handlers.PipelinesHandler;
   public logsHandler!: handlers.LogsHandler;
   public webhookHandler!: handlers.WebhookHandler;
+  public actionsHandler!: handlers.ActionsHandler;
 
   constructor(gitopsAppRef: GitopsApp) {
     this.gitopsAppRef = gitopsAppRef;
@@ -58,6 +59,7 @@ export class OpsServer {
     this.secretsHandler = new handlers.SecretsHandler(this);
     this.pipelinesHandler = new handlers.PipelinesHandler(this);
     this.logsHandler = new handlers.LogsHandler(this);
+    this.actionsHandler = new handlers.ActionsHandler(this);
 
     logger.success('OpsServer TypedRequest handlers initialized');
   }

ts/opsserver/handlers/actions.handler.ts

@@ -0,0 +1,50 @@
+import * as plugins from '../../plugins.ts';
+import type { OpsServer } from '../classes.opsserver.ts';
+import * as interfaces from '../../../ts_interfaces/index.ts';
+import { requireValidIdentity } from '../helpers/guards.ts';
+
+export class ActionsHandler {
+  public typedrouter = new plugins.typedrequest.TypedRouter();
+
+  constructor(private opsServerRef: OpsServer) {
+    this.opsServerRef.typedrouter.addTypedRouter(this.typedrouter);
+    this.registerHandlers();
+  }
+
+  private registerHandlers(): void {
+    // Force scan secrets
+    this.typedrouter.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_ForceScanSecrets>(
+        'forceScanSecrets',
+        async (dataArg) => {
+          await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
+          const scanService = this.opsServerRef.gitopsAppRef.secretsScanService;
+          const result = await scanService.fullScan();
+          return {
+            ok: true,
+            connectionsScanned: result.connectionsScanned,
+            secretsFound: result.secretsFound,
+            errors: result.errors,
+            durationMs: result.durationMs,
+          };
+        },
+      ),
+    );
+
+    // Get scan status
+    this.typedrouter.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetScanStatus>(
+        'getScanStatus',
+        async (dataArg) => {
+          await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
+          const scanService = this.opsServerRef.gitopsAppRef.secretsScanService;
+          return {
+            lastScanTimestamp: scanService.lastScanTimestamp,
+            isScanning: scanService.isScanning,
+            lastResult: scanService.lastScanResult,
+          };
+        },
+      ),
+    );
+  }
+}

ts/opsserver/handlers/index.ts

@@ -6,3 +6,4 @@ export { SecretsHandler } from './secrets.handler.ts';
 export { PipelinesHandler } from './pipelines.handler.ts';
 export { LogsHandler } from './logs.handler.ts';
 export { WebhookHandler } from './webhook.handler.ts';
+export { ActionsHandler } from './actions.handler.ts';

ts/opsserver/handlers/secrets.handler.ts

@@ -12,12 +12,25 @@ export class SecretsHandler {
   }
 
   private registerHandlers(): void {
-    // Get all secrets (bulk fetch across all entities)
+    // Get all secrets (cache-first, falls back to live fetch)
     this.typedrouter.addTypedHandler(
       new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetAllSecrets>(
         'getAllSecrets',
         async (dataArg) => {
           await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
+          const scanService = this.opsServerRef.gitopsAppRef.secretsScanService;
+
+          // Try cache first
+          const hasCached = await scanService.hasCachedData(dataArg.connectionId, dataArg.scope);
+          if (hasCached) {
+            const secrets = await scanService.getCachedSecrets({
+              connectionId: dataArg.connectionId,
+              scope: dataArg.scope,
+            });
+            return { secrets };
+          }
+
+          // Cache miss: live fetch and save to cache
           const provider = this.opsServerRef.gitopsAppRef.connectionManager.getProvider(
             dataArg.connectionId,
           );
@@ -26,13 +39,18 @@ export class SecretsHandler {
 
           if (dataArg.scope === 'project') {
             const projects = await provider.getProjects();
-            // Fetch in batches of 5 for performance
             for (let i = 0; i < projects.length; i += 5) {
               const batch = projects.slice(i, i + 5);
               const results = await Promise.allSettled(
                 batch.map(async (p) => {
                   const secrets = await provider.getProjectSecrets(p.id);
-                  return secrets.map((s) => ({ ...s, scopeName: p.fullPath || p.name }));
+                  return secrets.map((s) => ({
+                    ...s,
+                    scopeName: p.fullPath || p.name,
+                    scope: 'project' as const,
+                    scopeId: p.id,
+                    connectionId: dataArg.connectionId,
+                  }));
                 }),
               );
               for (const result of results) {
@@ -48,7 +66,13 @@ export class SecretsHandler {
               const results = await Promise.allSettled(
                 batch.map(async (g) => {
                   const secrets = await provider.getGroupSecrets(g.id);
-                  return secrets.map((s) => ({ ...s, scopeName: g.fullPath || g.name }));
+                  return secrets.map((s) => ({
+                    ...s,
+                    scopeName: g.fullPath || g.name,
+                    scope: 'group' as const,
+                    scopeId: g.id,
+                    connectionId: dataArg.connectionId,
+                  }));
                 }),
               );
               for (const result of results) {
@@ -59,23 +83,49 @@ export class SecretsHandler {
             }
           }
 
+          // Save fetched secrets to cache (fire-and-forget)
+          scanService.saveSecrets(allSecrets).catch(() => {});
+
           return { secrets: allSecrets };
         },
       ),
     );
 
-    // Get secrets
+    // Get secrets (cache-first for single entity)
     this.typedrouter.addTypedHandler(
       new plugins.typedrequest.TypedHandler<interfaces.requests.IReq_GetSecrets>(
         'getSecrets',
         async (dataArg) => {
           await requireValidIdentity(this.opsServerRef.adminHandler, dataArg);
+          const scanService = this.opsServerRef.gitopsAppRef.secretsScanService;
+
+          // Try cache first
+          const cached = await scanService.getCachedSecrets({
+            connectionId: dataArg.connectionId,
+            scope: dataArg.scope,
+            scopeId: dataArg.scopeId,
+          });
+          if (cached.length > 0) {
+            return { secrets: cached };
+          }
+
+          // Cache miss: live fetch
           const provider = this.opsServerRef.gitopsAppRef.connectionManager.getProvider(
             dataArg.connectionId,
           );
           const secrets = dataArg.scope === 'project'
             ? await provider.getProjectSecrets(dataArg.scopeId)
             : await provider.getGroupSecrets(dataArg.scopeId);
+
+          // Save to cache (fire-and-forget)
+          const fullSecrets = secrets.map((s) => ({
+            ...s,
+            scope: dataArg.scope,
+            scopeId: dataArg.scopeId,
+            connectionId: dataArg.connectionId,
+          }));
+          scanService.saveSecrets(fullSecrets).catch(() => {});
+
           return { secrets };
         },
       ),
@@ -93,6 +143,9 @@ export class SecretsHandler {
           const secret = dataArg.scope === 'project'
             ? await provider.createProjectSecret(dataArg.scopeId, dataArg.key, dataArg.value)
             : await provider.createGroupSecret(dataArg.scopeId, dataArg.key, dataArg.value);
+          // Refresh cache for this entity
+          const scanService = this.opsServerRef.gitopsAppRef.secretsScanService;
+          scanService.scanEntity(dataArg.connectionId, dataArg.scope, dataArg.scopeId).catch(() => {});
           return { secret };
         },
       ),
@@ -110,6 +163,9 @@ export class SecretsHandler {
           const secret = dataArg.scope === 'project'
             ? await provider.updateProjectSecret(dataArg.scopeId, dataArg.key, dataArg.value)
             : await provider.updateGroupSecret(dataArg.scopeId, dataArg.key, dataArg.value);
+          // Refresh cache for this entity
+          const scanService = this.opsServerRef.gitopsAppRef.secretsScanService;
+          scanService.scanEntity(dataArg.connectionId, dataArg.scope, dataArg.scopeId).catch(() => {});
           return { secret };
         },
       ),
@@ -129,6 +185,9 @@ export class SecretsHandler {
           } else {
             await provider.deleteGroupSecret(dataArg.scopeId, dataArg.key);
           }
+          // Refresh cache for this entity
+          const scanService = this.opsServerRef.gitopsAppRef.secretsScanService;
+          scanService.scanEntity(dataArg.connectionId, dataArg.scope, dataArg.scopeId).catch(() => {});
           return { ok: true };
         },
       ),