From 7de42e10dfb1ea5097a04d0acea24e7fd8466da9 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Sun, 10 May 2026 22:56:29 +0000 Subject: [PATCH] Improve remote runtime cache logging --- applications/electron-shell/ts/main.ts | 69 ++++++++++++++++++++++---- packages/server-installer/ts/index.ts | 24 +++++++++ packages/ssh/ts/index.ts | 16 +++++- packages/ssh/ts/plugins.ts | 3 +- test/test.installer.node.ts | 13 +++++ 5 files changed, 113 insertions(+), 12 deletions(-) diff --git a/applications/electron-shell/ts/main.ts b/applications/electron-shell/ts/main.ts index 935463c..221efda 100644 --- a/applications/electron-shell/ts/main.ts +++ b/applications/electron-shell/ts/main.ts @@ -75,9 +75,9 @@ class GitZoneIdeElectronShell { const serverVersion = plugins.electron.app.getVersion(); progress('Staging remote runtime payload.'); const runtime = await createLocalEphemeralRuntime(serverVersion); - progress(`Runtime hash ${runtime.contentHash.slice(0, 12)} staged for ${runtime.remoteRoot}.`); + progress(`Runtime hash ${runtime.contentHash.slice(0, 12)} staged: ${runtime.fileCount} files, ${formatBytes(runtime.totalBytes)} unpacked.`); try { - progress('Checking remote runtime cache.'); + progress(`Checking remote /tmp cache at ${runtime.remoteRoot}.`); const cacheCheckCommand = plugins.ideServerInstaller.createRemoteEphemeralRuntimeCacheCheckCommand({ runtimeRoot: runtime.remoteRoot, runtimeSha256: runtime.contentHash, @@ -87,17 +87,31 @@ class GitZoneIdeElectronShell { batchMode: input.batchMode ?? true, }); if (cacheCheckResult.exitCode === 0) { - progress('Remote runtime cache hit; skipping upload.'); + progress(`Remote runtime hash matches; skipping copy for ${runtime.contentHash.slice(0, 12)}.`); } else { - progress('Remote runtime cache miss; uploading payload.'); + progress(`Remote runtime cache miss; copying ${formatBytes(runtime.totalBytes)} to ${runtime.remoteRoot}.`); + const reportUploadProgress = createUploadProgressReporter(progress, runtime.totalBytes); const uploadResult = await plugins.ideSsh.uploadDirectoryToRemote(target, runtime.localRoot, runtime.remoteRoot, { timeoutMs: 300000, batchMode: input.batchMode ?? true, + onProgress: (uploadProgress) => reportUploadProgress(uploadProgress.bytesUploaded), }); if (uploadResult.exitCode !== 0) { throw new Error(uploadResult.stderr || `Remote runtime upload failed with ${uploadResult.exitCode}`); } - progress('Remote runtime upload complete.'); + progress('Remote runtime files copied; writing cache marker.'); + const markCommand = plugins.ideServerInstaller.createRemoteEphemeralRuntimeMarkCommand({ + runtimeRoot: runtime.remoteRoot, + runtimeSha256: runtime.contentHash, + }); + const markResult = await plugins.ideSsh.runSshCommand(target, markCommand, { + timeoutMs: 30000, + batchMode: input.batchMode ?? true, + }); + if (markResult.exitCode !== 0) { + throw new Error(markResult.stderr || `Remote runtime marker write failed with ${markResult.exitCode}`); + } + progress(`Remote runtime upload complete; cache marker ${runtime.contentHash.slice(0, 12)} stored.`); } progress('Starting remote Theia runtime.'); @@ -267,6 +281,25 @@ const createProgressEmitter = (webContents: { isDestroyed(): boolean; send(chann }; }; +const createUploadProgressReporter = (progress: (message: string) => void, unpackedBytes: number) => { + const startedAt = Date.now(); + let lastReportedAt = 0; + let lastReportedBytes = 0; + + return (bytesUploaded: number) => { + const now = Date.now(); + if (bytesUploaded - lastReportedBytes < 4 * 1024 * 1024 && now - lastReportedAt < 2000) { + return; + } + + lastReportedAt = now; + lastReportedBytes = bytesUploaded; + const elapsedSeconds = Math.max((now - startedAt) / 1000, 0.001); + const uploadRate = bytesUploaded / elapsedSeconds; + progress(`Copying runtime: ${formatBytes(bytesUploaded)} compressed sent at ${formatBytes(uploadRate)}/s (${formatBytes(unpackedBytes)} unpacked).`); + }; +}; + const createLocalEphemeralRuntime = async (serverVersion: string) => { const stageId = `gitzone-ide-stage-${sanitizeRuntimePart(serverVersion)}-${Date.now()}-${plugins.crypto.randomBytes(4).toString('hex')}`; const localRoot = path.join(os.tmpdir(), stageId); @@ -283,11 +316,10 @@ const createLocalEphemeralRuntime = async (serverVersion: string) => { await fs.copyFile(nodeBinary, targetNodeBinary); await fs.chmod(targetNodeBinary, 0o755); await copyNodeSharedLibraries(nodeBinary, path.join(localRoot, 'node', 'lib')); - const contentHash = await hashLocalRuntimeDirectory(localRoot); - await fs.writeFile(path.join(localRoot, runtimeMarkerFileName), `${contentHash}\n`, 'utf8'); - const remoteRoot = `/tmp/gitzone-ide-${sanitizeRuntimePart(serverVersion)}-${contentHash}`; + const runtimeHash = await hashLocalRuntimeDirectory(localRoot); + const remoteRoot = `/tmp/gitzone-ide-${sanitizeRuntimePart(serverVersion)}-${runtimeHash.contentHash}`; - return { localRoot, remoteRoot, contentHash }; + return { localRoot, remoteRoot, ...runtimeHash }; }; const copyNodeSharedLibraries = async (nodeBinary: string, targetDirectory: string) => { @@ -311,6 +343,8 @@ const copyNodeSharedLibraries = async (nodeBinary: string, targetDirectory: stri const hashLocalRuntimeDirectory = async (rootDirectory: string) => { const hash = plugins.crypto.createHash('sha256'); const filePaths = await listLocalRuntimeFiles(rootDirectory); + let fileCount = 0; + let totalBytes = 0; for (const filePath of filePaths) { const relativePath = path.relative(rootDirectory, filePath).split(path.sep).join('/'); @@ -321,6 +355,8 @@ const hashLocalRuntimeDirectory = async (rootDirectory: string) => { const stats = await fs.lstat(filePath); if (stats.isSymbolicLink()) { const linkTarget = await fs.readlink(filePath); + fileCount++; + totalBytes += Buffer.byteLength(linkTarget); hash.update(`link\0${relativePath}\0${linkTarget}\0`); continue; } @@ -329,12 +365,14 @@ const hashLocalRuntimeDirectory = async (rootDirectory: string) => { continue; } + fileCount++; + totalBytes += stats.size; hash.update(`file\0${relativePath}\0${stats.mode & 0o111 ? 'x' : '-'}\0${stats.size}\0`); await updateHashFromFile(hash, filePath); hash.update('\0'); } - return hash.digest('hex'); + return { contentHash: hash.digest('hex'), fileCount, totalBytes }; }; const listLocalRuntimeFiles = async (rootDirectory: string) => { @@ -385,6 +423,17 @@ const resolveLocalNodeBinary = async () => { const sanitizeRuntimePart = (value: string) => value.replace(/[^a-zA-Z0-9._-]/g, '-'); +const formatBytes = (bytes: number) => { + const units = ['B', 'KiB', 'MiB', 'GiB']; + let value = bytes; + let unitIndex = 0; + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024; + unitIndex++; + } + return `${value.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`; +}; + const waitForHttpUrl = async (url: string, timeoutMs: number) => { const startedAt = Date.now(); let lastError: unknown; diff --git a/packages/server-installer/ts/index.ts b/packages/server-installer/ts/index.ts index b9b5ff4..b0d4cdc 100644 --- a/packages/server-installer/ts/index.ts +++ b/packages/server-installer/ts/index.ts @@ -53,6 +53,13 @@ export interface IRemoteEphemeralRuntimeCacheCheckOptions { nodePath?: string; } +export interface IRemoteEphemeralRuntimeMarkOptions { + runtimeRoot: string; + runtimeSha256: string; + markerFileName?: string; + nodePath?: string; +} + export const defaultIdeDataRoot = '~/.git.zone/ide'; export const defaultInstallRoot = '~/.git.zone/ide/server'; export const remoteEphemeralRuntimeMarkerFileName = '.gitzone-runtime-sha256'; @@ -209,6 +216,23 @@ export const createRemoteEphemeralRuntimeCacheCheckCommand = (options: IRemoteEp ].join('\n'); }; +export const createRemoteEphemeralRuntimeMarkCommand = (options: IRemoteEphemeralRuntimeMarkOptions) => { + const markerFileName = options.markerFileName ?? remoteEphemeralRuntimeMarkerFileName; + const markerPath = joinRemotePath(options.runtimeRoot, markerFileName); + const markerTempPath = `${markerPath}.tmp`; + const nodePath = options.nodePath ?? joinRemotePath(options.runtimeRoot, 'node/bin/node'); + const backendPath = joinRemotePath(options.runtimeRoot, 'applications/remote-theia/lib/backend/main.js'); + + return [ + 'set -euo pipefail', + `test -x ${quoteRemotePath(nodePath)}`, + `test -f ${quoteRemotePath(backendPath)}`, + `printf '%s\n' ${quoteShellArg(options.runtimeSha256)} > ${quoteRemotePath(markerTempPath)}`, + `mv ${quoteRemotePath(markerTempPath)} ${quoteRemotePath(markerPath)}`, + `printf 'runtimeCache=stored\n'`, + ].join('\n'); +}; + export const createRemoteHealthCommand = (serverVersion: string, installRoot = defaultInstallRoot) => { const plan = createRemoteServerInstallPlan({ serverVersion, diff --git a/packages/ssh/ts/index.ts b/packages/ssh/ts/index.ts index 13e007f..b98a595 100644 --- a/packages/ssh/ts/index.ts +++ b/packages/ssh/ts/index.ts @@ -36,6 +36,11 @@ export interface ISshTunnelOptions extends ISshRunOptions { export interface ISshUploadOptions extends ISshRunOptions { cleanRemote?: boolean; + onProgress?: (progress: ISshUploadProgress) => void; +} + +export interface ISshUploadProgress { + bytesUploaded: number; } export interface ISshTunnelHandle { @@ -367,6 +372,7 @@ export const uploadDirectoryToRemote = async ( const stdout: Buffer[] = []; const stderr: Buffer[] = []; let tarStderr = ''; + let bytesUploaded = 0; let settled = false; const timeout = options.timeoutMs ? setTimeout(() => { @@ -386,7 +392,15 @@ export const uploadDirectoryToRemote = async ( resolve(result); }; - tar.stdout.pipe(ssh.stdin); + const progressStream = new plugins.stream.Transform({ + transform(chunk: Buffer, _encoding, callback) { + bytesUploaded += chunk.length; + options.onProgress?.({ bytesUploaded }); + callback(undefined, chunk); + }, + }); + + tar.stdout.pipe(progressStream).pipe(ssh.stdin); tar.stderr.on('data', (chunk: Buffer) => { tarStderr += chunk.toString('utf8'); }); diff --git a/packages/ssh/ts/plugins.ts b/packages/ssh/ts/plugins.ts index 1e124d0..b8c1b6f 100644 --- a/packages/ssh/ts/plugins.ts +++ b/packages/ssh/ts/plugins.ts @@ -4,5 +4,6 @@ import * as fsSync from 'node:fs'; import * as net from 'node:net'; import * as os from 'node:os'; import * as path from 'node:path'; +import * as stream from 'node:stream'; -export { childProcess, fs, fsSync, net, os, path }; +export { childProcess, fs, fsSync, net, os, path, stream }; diff --git a/test/test.installer.node.ts b/test/test.installer.node.ts index 25e2169..3e23c0e 100644 --- a/test/test.installer.node.ts +++ b/test/test.installer.node.ts @@ -3,6 +3,7 @@ import { createRemoteEphemeralBootstrapCommand, createRemoteEphemeralReadinessCommand, createRemoteEphemeralRuntimeCacheCheckCommand, + createRemoteEphemeralRuntimeMarkCommand, createRemoteBootstrapCommand, createRemoteInstallCommand, createRemoteServerInstallPlan, @@ -113,4 +114,16 @@ tap.test('should render ephemeral runtime cache check command', async () => { expect(cacheCheckCommand).toInclude('runtimeCache=hit'); }); +tap.test('should render ephemeral runtime mark command', async () => { + const markCommand = createRemoteEphemeralRuntimeMarkCommand({ + runtimeRoot: '/tmp/gitzone-ide-0.1.0-deadbeef', + runtimeSha256: 'deadbeef', + }); + + expect(markCommand).toInclude('.gitzone-runtime-sha256.tmp'); + expect(markCommand).toInclude("printf '%s"); + expect(markCommand).toInclude("'deadbeef'"); + expect(markCommand).toInclude('runtimeCache=stored'); +}); + export default tap.start();