Improve remote runtime cache logging

applications/electron-shell/ts/main.ts

@@ -75,9 +75,9 @@ class GitZoneIdeElectronShell {
       const serverVersion = plugins.electron.app.getVersion();
       progress('Staging remote runtime payload.');
       const runtime = await createLocalEphemeralRuntime(serverVersion);
-      progress(`Runtime hash ${runtime.contentHash.slice(0, 12)} staged for ${runtime.remoteRoot}.`);
+      progress(`Runtime hash ${runtime.contentHash.slice(0, 12)} staged: ${runtime.fileCount} files, ${formatBytes(runtime.totalBytes)} unpacked.`);
       try {
-        progress('Checking remote runtime cache.');
+        progress(`Checking remote /tmp cache at ${runtime.remoteRoot}.`);
         const cacheCheckCommand = plugins.ideServerInstaller.createRemoteEphemeralRuntimeCacheCheckCommand({
           runtimeRoot: runtime.remoteRoot,
           runtimeSha256: runtime.contentHash,
@@ -87,17 +87,31 @@ class GitZoneIdeElectronShell {
           batchMode: input.batchMode ?? true,
         });
         if (cacheCheckResult.exitCode === 0) {
-          progress('Remote runtime cache hit; skipping upload.');
+          progress(`Remote runtime hash matches; skipping copy for ${runtime.contentHash.slice(0, 12)}.`);
         } else {
-          progress('Remote runtime cache miss; uploading payload.');
+          progress(`Remote runtime cache miss; copying ${formatBytes(runtime.totalBytes)} to ${runtime.remoteRoot}.`);
+          const reportUploadProgress = createUploadProgressReporter(progress, runtime.totalBytes);
           const uploadResult = await plugins.ideSsh.uploadDirectoryToRemote(target, runtime.localRoot, runtime.remoteRoot, {
             timeoutMs: 300000,
             batchMode: input.batchMode ?? true,
+            onProgress: (uploadProgress) => reportUploadProgress(uploadProgress.bytesUploaded),
           });
           if (uploadResult.exitCode !== 0) {
             throw new Error(uploadResult.stderr || `Remote runtime upload failed with ${uploadResult.exitCode}`);
           }
-          progress('Remote runtime upload complete.');
+          progress('Remote runtime files copied; writing cache marker.');
+          const markCommand = plugins.ideServerInstaller.createRemoteEphemeralRuntimeMarkCommand({
+            runtimeRoot: runtime.remoteRoot,
+            runtimeSha256: runtime.contentHash,
+          });
+          const markResult = await plugins.ideSsh.runSshCommand(target, markCommand, {
+            timeoutMs: 30000,
+            batchMode: input.batchMode ?? true,
+          });
+          if (markResult.exitCode !== 0) {
+            throw new Error(markResult.stderr || `Remote runtime marker write failed with ${markResult.exitCode}`);
+          }
+          progress(`Remote runtime upload complete; cache marker ${runtime.contentHash.slice(0, 12)} stored.`);
         }
 
         progress('Starting remote Theia runtime.');
@@ -267,6 +281,25 @@ const createProgressEmitter = (webContents: { isDestroyed(): boolean; send(chann
   };
 };
 
+const createUploadProgressReporter = (progress: (message: string) => void, unpackedBytes: number) => {
+  const startedAt = Date.now();
+  let lastReportedAt = 0;
+  let lastReportedBytes = 0;
+
+  return (bytesUploaded: number) => {
+    const now = Date.now();
+    if (bytesUploaded - lastReportedBytes < 4 * 1024 * 1024 && now - lastReportedAt < 2000) {
+      return;
+    }
+
+    lastReportedAt = now;
+    lastReportedBytes = bytesUploaded;
+    const elapsedSeconds = Math.max((now - startedAt) / 1000, 0.001);
+    const uploadRate = bytesUploaded / elapsedSeconds;
+    progress(`Copying runtime: ${formatBytes(bytesUploaded)} compressed sent at ${formatBytes(uploadRate)}/s (${formatBytes(unpackedBytes)} unpacked).`);
+  };
+};
+
 const createLocalEphemeralRuntime = async (serverVersion: string) => {
   const stageId = `gitzone-ide-stage-${sanitizeRuntimePart(serverVersion)}-${Date.now()}-${plugins.crypto.randomBytes(4).toString('hex')}`;
   const localRoot = path.join(os.tmpdir(), stageId);
@@ -283,11 +316,10 @@ const createLocalEphemeralRuntime = async (serverVersion: string) => {
   await fs.copyFile(nodeBinary, targetNodeBinary);
   await fs.chmod(targetNodeBinary, 0o755);
   await copyNodeSharedLibraries(nodeBinary, path.join(localRoot, 'node', 'lib'));
-  const contentHash = await hashLocalRuntimeDirectory(localRoot);
+  const runtimeHash = await hashLocalRuntimeDirectory(localRoot);
-  await fs.writeFile(path.join(localRoot, runtimeMarkerFileName), `${contentHash}\n`, 'utf8');
+  const remoteRoot = `/tmp/gitzone-ide-${sanitizeRuntimePart(serverVersion)}-${runtimeHash.contentHash}`;
-  const remoteRoot = `/tmp/gitzone-ide-${sanitizeRuntimePart(serverVersion)}-${contentHash}`;
 
-  return { localRoot, remoteRoot, contentHash };
+  return { localRoot, remoteRoot, ...runtimeHash };
 };
 
 const copyNodeSharedLibraries = async (nodeBinary: string, targetDirectory: string) => {
@@ -311,6 +343,8 @@ const copyNodeSharedLibraries = async (nodeBinary: string, targetDirectory: stri
 const hashLocalRuntimeDirectory = async (rootDirectory: string) => {
   const hash = plugins.crypto.createHash('sha256');
   const filePaths = await listLocalRuntimeFiles(rootDirectory);
+  let fileCount = 0;
+  let totalBytes = 0;
 
   for (const filePath of filePaths) {
     const relativePath = path.relative(rootDirectory, filePath).split(path.sep).join('/');
@@ -321,6 +355,8 @@ const hashLocalRuntimeDirectory = async (rootDirectory: string) => {
     const stats = await fs.lstat(filePath);
     if (stats.isSymbolicLink()) {
       const linkTarget = await fs.readlink(filePath);
+      fileCount++;
+      totalBytes += Buffer.byteLength(linkTarget);
       hash.update(`link\0${relativePath}\0${linkTarget}\0`);
       continue;
     }
@@ -329,12 +365,14 @@ const hashLocalRuntimeDirectory = async (rootDirectory: string) => {
       continue;
     }
 
+    fileCount++;
+    totalBytes += stats.size;
     hash.update(`file\0${relativePath}\0${stats.mode & 0o111 ? 'x' : '-'}\0${stats.size}\0`);
     await updateHashFromFile(hash, filePath);
     hash.update('\0');
   }
 
-  return hash.digest('hex');
+  return { contentHash: hash.digest('hex'), fileCount, totalBytes };
 };
 
 const listLocalRuntimeFiles = async (rootDirectory: string) => {
@@ -385,6 +423,17 @@ const resolveLocalNodeBinary = async () => {
 
 const sanitizeRuntimePart = (value: string) => value.replace(/[^a-zA-Z0-9._-]/g, '-');
 
+const formatBytes = (bytes: number) => {
+  const units = ['B', 'KiB', 'MiB', 'GiB'];
+  let value = bytes;
+  let unitIndex = 0;
+  while (value >= 1024 && unitIndex < units.length - 1) {
+    value /= 1024;
+    unitIndex++;
+  }
+  return `${value.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
+};
+
 const waitForHttpUrl = async (url: string, timeoutMs: number) => {
   const startedAt = Date.now();
   let lastError: unknown;

packages/server-installer/ts/index.ts

@@ -53,6 +53,13 @@ export interface IRemoteEphemeralRuntimeCacheCheckOptions {
   nodePath?: string;
 }
 
+export interface IRemoteEphemeralRuntimeMarkOptions {
+  runtimeRoot: string;
+  runtimeSha256: string;
+  markerFileName?: string;
+  nodePath?: string;
+}
+
 export const defaultIdeDataRoot = '~/.git.zone/ide';
 export const defaultInstallRoot = '~/.git.zone/ide/server';
 export const remoteEphemeralRuntimeMarkerFileName = '.gitzone-runtime-sha256';
@@ -209,6 +216,23 @@ export const createRemoteEphemeralRuntimeCacheCheckCommand = (options: IRemoteEp
   ].join('\n');
 };
 
+export const createRemoteEphemeralRuntimeMarkCommand = (options: IRemoteEphemeralRuntimeMarkOptions) => {
+  const markerFileName = options.markerFileName ?? remoteEphemeralRuntimeMarkerFileName;
+  const markerPath = joinRemotePath(options.runtimeRoot, markerFileName);
+  const markerTempPath = `${markerPath}.tmp`;
+  const nodePath = options.nodePath ?? joinRemotePath(options.runtimeRoot, 'node/bin/node');
+  const backendPath = joinRemotePath(options.runtimeRoot, 'applications/remote-theia/lib/backend/main.js');
+
+  return [
+    'set -euo pipefail',
+    `test -x ${quoteRemotePath(nodePath)}`,
+    `test -f ${quoteRemotePath(backendPath)}`,
+    `printf '%s\n' ${quoteShellArg(options.runtimeSha256)} > ${quoteRemotePath(markerTempPath)}`,
+    `mv ${quoteRemotePath(markerTempPath)} ${quoteRemotePath(markerPath)}`,
+    `printf 'runtimeCache=stored\n'`,
+  ].join('\n');
+};
+
 export const createRemoteHealthCommand = (serverVersion: string, installRoot = defaultInstallRoot) => {
   const plan = createRemoteServerInstallPlan({
     serverVersion,

packages/ssh/ts/index.ts

@@ -36,6 +36,11 @@ export interface ISshTunnelOptions extends ISshRunOptions {
 
 export interface ISshUploadOptions extends ISshRunOptions {
   cleanRemote?: boolean;
+  onProgress?: (progress: ISshUploadProgress) => void;
+}
+
+export interface ISshUploadProgress {
+  bytesUploaded: number;
 }
 
 export interface ISshTunnelHandle {
@@ -367,6 +372,7 @@ export const uploadDirectoryToRemote = async (
     const stdout: Buffer[] = [];
     const stderr: Buffer[] = [];
     let tarStderr = '';
+    let bytesUploaded = 0;
     let settled = false;
     const timeout = options.timeoutMs
       ? setTimeout(() => {
@@ -386,7 +392,15 @@ export const uploadDirectoryToRemote = async (
       resolve(result);
     };
 
-    tar.stdout.pipe(ssh.stdin);
+    const progressStream = new plugins.stream.Transform({
+      transform(chunk: Buffer, _encoding, callback) {
+        bytesUploaded += chunk.length;
+        options.onProgress?.({ bytesUploaded });
+        callback(undefined, chunk);
+      },
+    });
+
+    tar.stdout.pipe(progressStream).pipe(ssh.stdin);
     tar.stderr.on('data', (chunk: Buffer) => {
       tarStderr += chunk.toString('utf8');
     });

packages/ssh/ts/plugins.ts

@@ -4,5 +4,6 @@ import * as fsSync from 'node:fs';
 import * as net from 'node:net';
 import * as os from 'node:os';
 import * as path from 'node:path';
+import * as stream from 'node:stream';
 
-export { childProcess, fs, fsSync, net, os, path };
+export { childProcess, fs, fsSync, net, os, path, stream };

test/test.installer.node.ts

@@ -3,6 +3,7 @@ import {
   createRemoteEphemeralBootstrapCommand,
   createRemoteEphemeralReadinessCommand,
   createRemoteEphemeralRuntimeCacheCheckCommand,
+  createRemoteEphemeralRuntimeMarkCommand,
   createRemoteBootstrapCommand,
   createRemoteInstallCommand,
   createRemoteServerInstallPlan,
@@ -113,4 +114,16 @@ tap.test('should render ephemeral runtime cache check command', async () => {
   expect(cacheCheckCommand).toInclude('runtimeCache=hit');
 });
 
+tap.test('should render ephemeral runtime mark command', async () => {
+  const markCommand = createRemoteEphemeralRuntimeMarkCommand({
+    runtimeRoot: '/tmp/gitzone-ide-0.1.0-deadbeef',
+    runtimeSha256: 'deadbeef',
+  });
+
+  expect(markCommand).toInclude('.gitzone-runtime-sha256.tmp');
+  expect(markCommand).toInclude("printf '%s");
+  expect(markCommand).toInclude("'deadbeef'");
+  expect(markCommand).toInclude('runtimeCache=stored');
+});
+
 export default tap.start();