feat(task-execution): implement task cancellation handling and improve UI feedback for canceling tasks
This commit is contained in:
		| @@ -63,7 +63,7 @@ export class TaskExecution extends plugins.smartdata.SmartDataDbDoc< | |||||||
|   } |   } | ||||||
|  |  | ||||||
|   // INSTANCE |   // INSTANCE | ||||||
|   @plugins.smartdata.svDb() |   @plugins.smartdata.unI() | ||||||
|   public id: string; |   public id: string; | ||||||
|  |  | ||||||
|   @plugins.smartdata.svDb() |   @plugins.smartdata.svDb() | ||||||
|   | |||||||
| @@ -22,6 +22,7 @@ export class CloudlyTaskManager { | |||||||
|   private taskRegistry = new Map<string, plugins.taskbuffer.Task>(); |   private taskRegistry = new Map<string, plugins.taskbuffer.Task>(); | ||||||
|   private taskInfo = new Map<string, ITaskInfo>(); |   private taskInfo = new Map<string, ITaskInfo>(); | ||||||
|   private currentExecutions = new Map<string, TaskExecution>(); |   private currentExecutions = new Map<string, TaskExecution>(); | ||||||
|  |   private cancellationRequests = new Set<string>(); | ||||||
|    |    | ||||||
|   // Database connection helper |   // Database connection helper | ||||||
|   get db() { |   get db() { | ||||||
| @@ -107,9 +108,14 @@ export class CloudlyTaskManager { | |||||||
|       // Execute the task |       // Execute the task | ||||||
|       const result = await task.trigger(); |       const result = await task.trigger(); | ||||||
|        |        | ||||||
|  |       // If a cancellation was requested during execution, don't mark as completed | ||||||
|  |       if (execution.data.status === 'cancelled' || this.cancellationRequests.has(execution.id)) { | ||||||
|  |         await execution.addLog('Task cancelled during execution', 'warning'); | ||||||
|  |       } else { | ||||||
|         // Task completed successfully |         // Task completed successfully | ||||||
|         await execution.complete(result); |         await execution.complete(result); | ||||||
|         await execution.addLog(`Task completed successfully`, 'success'); |         await execution.addLog(`Task completed successfully`, 'success'); | ||||||
|  |       } | ||||||
|        |        | ||||||
|       // Update last run time |       // Update last run time | ||||||
|       if (info) { |       if (info) { | ||||||
| @@ -117,13 +123,19 @@ export class CloudlyTaskManager { | |||||||
|       } |       } | ||||||
|        |        | ||||||
|     } catch (error) { |     } catch (error) { | ||||||
|  |       // If already cancelled, don't mark as failed | ||||||
|  |       if (execution.data.status === 'cancelled' || this.cancellationRequests.has(execution.id)) { | ||||||
|  |         await execution.addLog('Task was cancelled', 'warning'); | ||||||
|  |       } else { | ||||||
|         // Task failed |         // Task failed | ||||||
|       await execution.fail(error); |         await execution.fail(error as any); | ||||||
|       await execution.addLog(`Task failed: ${error.message}`, 'error'); |         await execution.addLog(`Task failed: ${(error as any).message}`, 'error'); | ||||||
|       logger.log('error', `Task ${taskName} failed: ${error.message}`); |         logger.log('error', `Task ${taskName} failed: ${(error as any).message}`); | ||||||
|  |       } | ||||||
|     } finally { |     } finally { | ||||||
|       // Clean up current execution |       // Clean up current execution | ||||||
|       this.currentExecutions.delete(taskName); |       this.currentExecutions.delete(taskName); | ||||||
|  |       this.cancellationRequests.delete(execution.id); | ||||||
|     } |     } | ||||||
|      |      | ||||||
|     return execution; |     return execution; | ||||||
| @@ -168,12 +180,19 @@ export class CloudlyTaskManager { | |||||||
|      |      | ||||||
|     await execution.cancel(); |     await execution.cancel(); | ||||||
|     await execution.addLog('Task cancelled by user', 'warning'); |     await execution.addLog('Task cancelled by user', 'warning'); | ||||||
|      |     // mark cancellation request so running task functions can react cooperatively | ||||||
|     // TODO: Implement actual task cancellation in taskbuffer |     this.cancellationRequests.add(execution.id); | ||||||
|      |      | ||||||
|     return true; |     return true; | ||||||
|   } |   } | ||||||
|  |  | ||||||
|  |   /** | ||||||
|  |    * Check if cancellation is requested for an execution | ||||||
|  |    */ | ||||||
|  |   public isCancellationRequested(executionId: string): boolean { | ||||||
|  |     return this.cancellationRequests.has(executionId); | ||||||
|  |   } | ||||||
|  |  | ||||||
|   /** |   /** | ||||||
|    * Get all registered tasks |    * Get all registered tasks | ||||||
|    */ |    */ | ||||||
|   | |||||||
| @@ -16,6 +16,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|        |        | ||||||
|       try { |       try { | ||||||
|         await execution?.addLog('Starting DNS synchronization...', 'info'); |         await execution?.addLog('Starting DNS synchronization...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |           await execution.addLog('Cancellation requested. Aborting DNS sync...', 'warning'); | ||||||
|  |           return; | ||||||
|  |         } | ||||||
|          |          | ||||||
|         // Get all DNS entries marked as external |         // Get all DNS entries marked as external | ||||||
|         const dnsEntries = await dnsManager.CDnsEntry.getInstances({ |         const dnsEntries = await dnsManager.CDnsEntry.getInstances({ | ||||||
| @@ -29,6 +33,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|         let failedCount = 0; |         let failedCount = 0; | ||||||
|          |          | ||||||
|         for (const entry of dnsEntries) { |         for (const entry of dnsEntries) { | ||||||
|  |           if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |             await execution.addLog('Cancellation requested. Stopping loop...', 'warning'); | ||||||
|  |             break; | ||||||
|  |           } | ||||||
|           try { |           try { | ||||||
|             // TODO: Implement actual sync with external DNS provider |             // TODO: Implement actual sync with external DNS provider | ||||||
|             await execution?.addLog(`Syncing DNS entry: ${entry.data.name}.${entry.data.zone}`, 'info'); |             await execution?.addLog(`Syncing DNS entry: ${entry.data.name}.${entry.data.zone}`, 'info'); | ||||||
| @@ -66,6 +74,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|        |        | ||||||
|       try { |       try { | ||||||
|         await execution?.addLog('Checking certificates for renewal...', 'info'); |         await execution?.addLog('Checking certificates for renewal...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |           await execution.addLog('Cancellation requested. Aborting certificate renewal...', 'warning'); | ||||||
|  |           return; | ||||||
|  |         } | ||||||
|          |          | ||||||
|         // Get all domains |         // Get all domains | ||||||
|         const domains = await taskManager.cloudlyRef.domainManager.CDomain.getInstances({}); |         const domains = await taskManager.cloudlyRef.domainManager.CDomain.getInstances({}); | ||||||
| @@ -75,6 +87,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|         let upToDateCount = 0; |         let upToDateCount = 0; | ||||||
|          |          | ||||||
|         for (const domain of domains) { |         for (const domain of domains) { | ||||||
|  |           if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |             await execution.addLog('Cancellation requested. Stopping loop...', 'warning'); | ||||||
|  |             break; | ||||||
|  |           } | ||||||
|           // TODO: Check certificate expiry and renew if needed |           // TODO: Check certificate expiry and renew if needed | ||||||
|           await execution?.addLog(`Checking certificate for ${domain.data.name}`, 'info'); |           await execution?.addLog(`Checking certificate for ${domain.data.name}`, 'info'); | ||||||
|            |            | ||||||
| @@ -117,17 +133,23 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|        |        | ||||||
|       try { |       try { | ||||||
|         await execution?.addLog('Starting cleanup tasks...', 'info'); |         await execution?.addLog('Starting cleanup tasks...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |           await execution.addLog('Cancellation requested. Aborting cleanup...', 'warning'); | ||||||
|  |           return; | ||||||
|  |         } | ||||||
|          |          | ||||||
|         // Clean up old task executions |         // Clean up old task executions | ||||||
|         await execution?.addLog('Cleaning old task executions...', 'info'); |         await execution?.addLog('Cleaning old task executions...', 'info'); | ||||||
|         const deletedExecutions = await taskManager.CTaskExecution.cleanupOldExecutions(30); |         const deletedExecutions = await taskManager.CTaskExecution.cleanupOldExecutions(30); | ||||||
|         await execution?.setMetric('deletedExecutions', deletedExecutions); |         await execution?.setMetric('deletedExecutions', deletedExecutions); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) return; | ||||||
|          |          | ||||||
|         // TODO: Clean up old logs |         // TODO: Clean up old logs | ||||||
|         await execution?.addLog('Cleaning old logs...', 'info'); |         await execution?.addLog('Cleaning old logs...', 'info'); | ||||||
|         // Placeholder |         // Placeholder | ||||||
|         const deletedLogs = 0; |         const deletedLogs = 0; | ||||||
|         await execution?.setMetric('deletedLogs', deletedLogs); |         await execution?.setMetric('deletedLogs', deletedLogs); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) return; | ||||||
|          |          | ||||||
|         // TODO: Clean up Docker images |         // TODO: Clean up Docker images | ||||||
|         await execution?.addLog('Cleaning unused Docker images...', 'info'); |         await execution?.addLog('Cleaning unused Docker images...', 'info'); | ||||||
| @@ -164,6 +186,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|        |        | ||||||
|       try { |       try { | ||||||
|         await execution?.addLog('Starting health checks...', 'info'); |         await execution?.addLog('Starting health checks...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |           await execution.addLog('Cancellation requested. Aborting health checks...', 'warning'); | ||||||
|  |           return; | ||||||
|  |         } | ||||||
|          |          | ||||||
|         // Check all deployments |         // Check all deployments | ||||||
|         const deployments = await taskManager.cloudlyRef.deploymentManager.getAllDeployments(); |         const deployments = await taskManager.cloudlyRef.deploymentManager.getAllDeployments(); | ||||||
| @@ -174,6 +200,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|         const issues = []; |         const issues = []; | ||||||
|          |          | ||||||
|         for (const deployment of deployments) { |         for (const deployment of deployments) { | ||||||
|  |           if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |             await execution.addLog('Cancellation requested. Stopping loop...', 'warning'); | ||||||
|  |             break; | ||||||
|  |           } | ||||||
|           if (deployment.status === 'running') { |           if (deployment.status === 'running') { | ||||||
|             // TODO: Actual health check logic |             // TODO: Actual health check logic | ||||||
|             const isHealthy = Math.random() > 0.1; // 90% healthy for demo |             const isHealthy = Math.random() > 0.1; // 90% healthy for demo | ||||||
| @@ -225,6 +255,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|        |        | ||||||
|       try { |       try { | ||||||
|         await execution?.addLog('Generating resource usage report...', 'info'); |         await execution?.addLog('Generating resource usage report...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |           await execution.addLog('Cancellation requested. Aborting report...', 'warning'); | ||||||
|  |           return; | ||||||
|  |         } | ||||||
|          |          | ||||||
|         // Get all nodes |         // Get all nodes | ||||||
|         const nodes = await taskManager.cloudlyRef.nodeManager.CClusterNode.getInstances({}); |         const nodes = await taskManager.cloudlyRef.nodeManager.CClusterNode.getInstances({}); | ||||||
| @@ -238,6 +272,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|         }; |         }; | ||||||
|          |          | ||||||
|         for (const node of nodes) { |         for (const node of nodes) { | ||||||
|  |           if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |             await execution.addLog('Cancellation requested. Stopping loop...', 'warning'); | ||||||
|  |             break; | ||||||
|  |           } | ||||||
|           // TODO: Get actual resource usage |           // TODO: Get actual resource usage | ||||||
|           const nodeUsage = { |           const nodeUsage = { | ||||||
|             nodeId: node.id, |             nodeId: node.id, | ||||||
| @@ -289,10 +327,16 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|        |        | ||||||
|       try { |       try { | ||||||
|         await execution?.addLog('Starting database maintenance...', 'info'); |         await execution?.addLog('Starting database maintenance...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |           await execution.addLog('Cancellation requested. Aborting DB maintenance...', 'warning'); | ||||||
|  |           return; | ||||||
|  |         } | ||||||
|          |          | ||||||
|         // TODO: Implement actual database maintenance |         // TODO: Implement actual database maintenance | ||||||
|         await execution?.addLog('Analyzing indexes...', 'info'); |         await execution?.addLog('Analyzing indexes...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) return; | ||||||
|         await execution?.addLog('Compacting collections...', 'info'); |         await execution?.addLog('Compacting collections...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) return; | ||||||
|         await execution?.addLog('Updating statistics...', 'info'); |         await execution?.addLog('Updating statistics...', 'info'); | ||||||
|          |          | ||||||
|         await execution?.setMetric('collectionsOptimized', 5); // Placeholder |         await execution?.setMetric('collectionsOptimized', 5); // Placeholder | ||||||
| @@ -323,6 +367,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|        |        | ||||||
|       try { |       try { | ||||||
|         await execution?.addLog('Starting security scan...', 'info'); |         await execution?.addLog('Starting security scan...', 'info'); | ||||||
|  |         if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |           await execution.addLog('Cancellation requested. Aborting security scan...', 'warning'); | ||||||
|  |           return; | ||||||
|  |         } | ||||||
|          |          | ||||||
|         const vulnerabilities = []; |         const vulnerabilities = []; | ||||||
|          |          | ||||||
| @@ -335,6 +383,10 @@ export function createPredefinedTasks(taskManager: CloudlyTaskManager) { | |||||||
|         const images = await taskManager.cloudlyRef.imageManager.CImage.getInstances({}); |         const images = await taskManager.cloudlyRef.imageManager.CImage.getInstances({}); | ||||||
|          |          | ||||||
|         for (const image of images) { |         for (const image of images) { | ||||||
|  |           if (execution && (taskManager.isCancellationRequested(execution.id) || execution.data.status === 'cancelled')) { | ||||||
|  |             await execution.addLog('Cancellation requested. Stopping loop...', 'warning'); | ||||||
|  |             break; | ||||||
|  |           } | ||||||
|           // TODO: Check if image is outdated |           // TODO: Check if image is outdated | ||||||
|           const isOutdated = Math.random() > 0.7; // 30% outdated for demo |           const isOutdated = Math.random() > 0.7; // 30% outdated for demo | ||||||
|            |            | ||||||
|   | |||||||
| @@ -37,6 +37,8 @@ export class CloudlyViewTasks extends DeesElement { | |||||||
|   private autoRefresh: boolean = true; |   private autoRefresh: boolean = true; | ||||||
|  |  | ||||||
|   private _refreshHandle: any = null; |   private _refreshHandle: any = null; | ||||||
|  |   @state() | ||||||
|  |   private canceling: Record<string, boolean> = {}; | ||||||
|  |  | ||||||
|   constructor() { |   constructor() { | ||||||
|     super(); |     super(); | ||||||
| @@ -492,6 +494,8 @@ export class CloudlyViewTasks extends DeesElement { | |||||||
|       const running = executions[0]; |       const running = executions[0]; | ||||||
|       if (!running) return; |       if (!running) return; | ||||||
|  |  | ||||||
|  |       this.canceling = { ...this.canceling, [running.id]: true }; | ||||||
|  |       try { | ||||||
|         await appstate.dataState.dispatchAction( |         await appstate.dataState.dispatchAction( | ||||||
|           appstate.taskActions.cancelTask, { executionId: running.id } |           appstate.taskActions.cancelTask, { executionId: running.id } | ||||||
|         ); |         ); | ||||||
| @@ -499,7 +503,10 @@ export class CloudlyViewTasks extends DeesElement { | |||||||
|           message: `Cancelled ${taskName}`, |           message: `Cancelled ${taskName}`, | ||||||
|           type: 'success', |           type: 'success', | ||||||
|         }); |         }); | ||||||
|  |       } finally { | ||||||
|  |         this.canceling = { ...this.canceling, [running.id]: false }; | ||||||
|         await this.loadExecutionsWithFilter(); |         await this.loadExecutionsWithFilter(); | ||||||
|  |       } | ||||||
|     } catch (err) { |     } catch (err) { | ||||||
|       console.error('Failed to cancel task:', err); |       console.error('Failed to cancel task:', err); | ||||||
|       plugins.deesCatalog.DeesToast.createAndShow({ |       plugins.deesCatalog.DeesToast.createAndShow({ | ||||||
| @@ -642,7 +649,12 @@ export class CloudlyViewTasks extends DeesElement { | |||||||
|             ${lastExecution ? html`<span class="status-badge status-${lastExecution.data.status}">${lastExecution.data.status}</span>` : html`<span class="status-badge" style="background:#2e2e2e;color:#ddd;border:1px solid #3a3a3a;">idle</span>`} |             ${lastExecution ? html`<span class="status-badge status-${lastExecution.data.status}">${lastExecution.data.status}</span>` : html`<span class="status-badge" style="background:#2e2e2e;color:#ddd;border:1px solid #3a3a3a;">idle</span>`} | ||||||
|             ${isRunning ? html` |             ${isRunning ? html` | ||||||
|               <dees-spinner style="--size: 18px"></dees-spinner> |               <dees-spinner style="--size: 18px"></dees-spinner> | ||||||
|               <dees-button .text=${'Cancel'} .type=${'secondary'} @click=${() => this.cancelTaskFor(task.name)}></dees-button> |               <dees-button | ||||||
|  |                 .text=${this.canceling[lastExecution!.id] ? 'Cancelling…' : 'Cancel'} | ||||||
|  |                 .type=${'secondary'} | ||||||
|  |                 .disabled=${!!this.canceling[lastExecution!.id]} | ||||||
|  |                 @click=${() => this.cancelTaskFor(task.name)} | ||||||
|  |               ></dees-button> | ||||||
|             ` : html` |             ` : html` | ||||||
|               <dees-button .text=${'Run'} .type=${'primary'} .disabled=${!task.enabled} @click=${() => this.triggerTask(task.name)}></dees-button> |               <dees-button .text=${'Run'} .type=${'primary'} .disabled=${!task.enabled} @click=${() => this.triggerTask(task.name)}></dees-button> | ||||||
|             `} |             `} | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user