git.zone/ide
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve remote runtime cache logging

Browse Source
This commit is contained in:
Jürgen Kunz
2026-05-10 22:56:29 +00:00
parent 61f6d37960
commit 7de42e10df
5 changed files with 113 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
applications/electron-shell/ts/main.ts
+59 -10
View File
@@ -75,9 +75,9 @@ class GitZoneIdeElectronShell {
const serverVersion = plugins.electron.app.getVersion();
progress('Staging remote runtime payload.');
const runtime = await createLocalEphemeralRuntime(serverVersion);
progress(`Runtime hash ${runtime.contentHash.slice(0, 12)} staged for ${runtime.remoteRoot}.`);
progress(`Runtime hash ${runtime.contentHash.slice(0, 12)} staged: ${runtime.fileCount} files, ${formatBytes(runtime.totalBytes)} unpacked.`);
try {
progress('Checking remote runtime cache.');
progress(`Checking remote /tmp cache at ${runtime.remoteRoot}.`);
const cacheCheckCommand = plugins.ideServerInstaller.createRemoteEphemeralRuntimeCacheCheckCommand({
runtimeRoot: runtime.remoteRoot,
runtimeSha256: runtime.contentHash,
@@ -87,17 +87,31 @@ class GitZoneIdeElectronShell {
batchMode: input.batchMode ?? true,
});
if (cacheCheckResult.exitCode === 0) {
progress('Remote runtime cache hit; skipping upload.');
progress(`Remote runtime hash matches; skipping copy for ${runtime.contentHash.slice(0, 12)}.`);
} else {
progress('Remote runtime cache miss; uploading payload.');
progress(`Remote runtime cache miss; copying ${formatBytes(runtime.totalBytes)} to ${runtime.remoteRoot}.`);
const reportUploadProgress = createUploadProgressReporter(progress, runtime.totalBytes);
const uploadResult = await plugins.ideSsh.uploadDirectoryToRemote(target, runtime.localRoot, runtime.remoteRoot, {
timeoutMs: 300000,
batchMode: input.batchMode ?? true,
onProgress: (uploadProgress) => reportUploadProgress(uploadProgress.bytesUploaded),
});
if (uploadResult.exitCode !== 0) {
throw new Error(uploadResult.stderr || `Remote runtime upload failed with ${uploadResult.exitCode}`);
}
progress('Remote runtime upload complete.');
progress('Remote runtime files copied; writing cache marker.');
const markCommand = plugins.ideServerInstaller.createRemoteEphemeralRuntimeMarkCommand({
runtimeRoot: runtime.remoteRoot,
runtimeSha256: runtime.contentHash,
});
const markResult = await plugins.ideSsh.runSshCommand(target, markCommand, {
timeoutMs: 30000,
batchMode: input.batchMode ?? true,
});
if (markResult.exitCode !== 0) {
throw new Error(markResult.stderr || `Remote runtime marker write failed with ${markResult.exitCode}`);
}
progress(`Remote runtime upload complete; cache marker ${runtime.contentHash.slice(0, 12)} stored.`);
}
progress('Starting remote Theia runtime.');
@@ -267,6 +281,25 @@ const createProgressEmitter = (webContents: { isDestroyed(): boolean; send(chann
};
};
const createUploadProgressReporter = (progress: (message: string) => void, unpackedBytes: number) => {
const startedAt = Date.now();
let lastReportedAt = 0;
let lastReportedBytes = 0;
return (bytesUploaded: number) => {
const now = Date.now();
if (bytesUploaded - lastReportedBytes < 4 * 1024 * 1024 && now - lastReportedAt < 2000) {
return;
}
lastReportedAt = now;
lastReportedBytes = bytesUploaded;
const elapsedSeconds = Math.max((now - startedAt) / 1000, 0.001);
const uploadRate = bytesUploaded / elapsedSeconds;
progress(`Copying runtime: ${formatBytes(bytesUploaded)} compressed sent at ${formatBytes(uploadRate)}/s (${formatBytes(unpackedBytes)} unpacked).`);
};
};
const createLocalEphemeralRuntime = async (serverVersion: string) => {
const stageId = `gitzone-ide-stage-${sanitizeRuntimePart(serverVersion)}-${Date.now()}-${plugins.crypto.randomBytes(4).toString('hex')}`;
const localRoot = path.join(os.tmpdir(), stageId);
@@ -283,11 +316,10 @@ const createLocalEphemeralRuntime = async (serverVersion: string) => {
await fs.copyFile(nodeBinary, targetNodeBinary);
await fs.chmod(targetNodeBinary, 0o755);
await copyNodeSharedLibraries(nodeBinary, path.join(localRoot, 'node', 'lib'));
const contentHash = await hashLocalRuntimeDirectory(localRoot);
await fs.writeFile(path.join(localRoot, runtimeMarkerFileName), `${contentHash}\n`, 'utf8');
const remoteRoot = `/tmp/gitzone-ide-${sanitizeRuntimePart(serverVersion)}-${contentHash}`;
const runtimeHash = await hashLocalRuntimeDirectory(localRoot);
const remoteRoot = `/tmp/gitzone-ide-${sanitizeRuntimePart(serverVersion)}-${runtimeHash.contentHash}`;
return { localRoot, remoteRoot, contentHash };
return { localRoot, remoteRoot, ...runtimeHash };
};
const copyNodeSharedLibraries = async (nodeBinary: string, targetDirectory: string) => {
@@ -311,6 +343,8 @@ const copyNodeSharedLibraries = async (nodeBinary: string, targetDirectory: stri
const hashLocalRuntimeDirectory = async (rootDirectory: string) => {
const hash = plugins.crypto.createHash('sha256');
const filePaths = await listLocalRuntimeFiles(rootDirectory);
let fileCount = 0;
let totalBytes = 0;
for (const filePath of filePaths) {
const relativePath = path.relative(rootDirectory, filePath).split(path.sep).join('/');
@@ -321,6 +355,8 @@ const hashLocalRuntimeDirectory = async (rootDirectory: string) => {
const stats = await fs.lstat(filePath);
if (stats.isSymbolicLink()) {
const linkTarget = await fs.readlink(filePath);
fileCount++;
totalBytes += Buffer.byteLength(linkTarget);
hash.update(`link\0${relativePath}\0${linkTarget}\0`);
continue;
}
@@ -329,12 +365,14 @@ const hashLocalRuntimeDirectory = async (rootDirectory: string) => {
continue;
}
fileCount++;
totalBytes += stats.size;
hash.update(`file\0${relativePath}\0${stats.mode & 0o111 ? 'x' : '-'}\0${stats.size}\0`);
await updateHashFromFile(hash, filePath);
hash.update('\0');
}
return hash.digest('hex');
return { contentHash: hash.digest('hex'), fileCount, totalBytes };
};
const listLocalRuntimeFiles = async (rootDirectory: string) => {
@@ -385,6 +423,17 @@ const resolveLocalNodeBinary = async () => {
const sanitizeRuntimePart = (value: string) => value.replace(/[^a-zA-Z0-9._-]/g, '-');
const formatBytes = (bytes: number) => {
const units = ['B', 'KiB', 'MiB', 'GiB'];
let value = bytes;
let unitIndex = 0;
while (value >= 1024 && unitIndex < units.length - 1) {
value /= 1024;
unitIndex++;
}
return `${value.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
};
const waitForHttpUrl = async (url: string, timeoutMs: number) => {
const startedAt = Date.now();
let lastError: unknown;
packages/server-installer/ts/index.ts
+24
View File
@@ -53,6 +53,13 @@ export interface IRemoteEphemeralRuntimeCacheCheckOptions {
nodePath?: string;
}
export interface IRemoteEphemeralRuntimeMarkOptions {
runtimeRoot: string;
runtimeSha256: string;
markerFileName?: string;
nodePath?: string;
}
export const defaultIdeDataRoot = '~/.git.zone/ide';
export const defaultInstallRoot = '~/.git.zone/ide/server';
export const remoteEphemeralRuntimeMarkerFileName = '.gitzone-runtime-sha256';
@@ -209,6 +216,23 @@ export const createRemoteEphemeralRuntimeCacheCheckCommand = (options: IRemoteEp
].join('\n');
};
export const createRemoteEphemeralRuntimeMarkCommand = (options: IRemoteEphemeralRuntimeMarkOptions) => {
const markerFileName = options.markerFileName ?? remoteEphemeralRuntimeMarkerFileName;
const markerPath = joinRemotePath(options.runtimeRoot, markerFileName);
const markerTempPath = `${markerPath}.tmp`;
const nodePath = options.nodePath ?? joinRemotePath(options.runtimeRoot, 'node/bin/node');
const backendPath = joinRemotePath(options.runtimeRoot, 'applications/remote-theia/lib/backend/main.js');
return [
'set -euo pipefail',
`test -x ${quoteRemotePath(nodePath)}`,
`test -f ${quoteRemotePath(backendPath)}`,
`printf '%s\n' ${quoteShellArg(options.runtimeSha256)} > ${quoteRemotePath(markerTempPath)}`,
`mv ${quoteRemotePath(markerTempPath)} ${quoteRemotePath(markerPath)}`,
`printf 'runtimeCache=stored\n'`,
].join('\n');
};
export const createRemoteHealthCommand = (serverVersion: string, installRoot = defaultInstallRoot) => {
const plan = createRemoteServerInstallPlan({
serverVersion,
packages/ssh/ts/index.ts
+15 -1
View File
@@ -36,6 +36,11 @@ export interface ISshTunnelOptions extends ISshRunOptions {
export interface ISshUploadOptions extends ISshRunOptions {
cleanRemote?: boolean;
onProgress?: (progress: ISshUploadProgress) => void;
}
export interface ISshUploadProgress {
bytesUploaded: number;
}
export interface ISshTunnelHandle {
@@ -367,6 +372,7 @@ export const uploadDirectoryToRemote = async (
const stdout: Buffer[] = [];
const stderr: Buffer[] = [];
let tarStderr = '';
let bytesUploaded = 0;
let settled = false;
const timeout = options.timeoutMs
? setTimeout(() => {
@@ -386,7 +392,15 @@ export const uploadDirectoryToRemote = async (
resolve(result);
};
tar.stdout.pipe(ssh.stdin);
const progressStream = new plugins.stream.Transform({
transform(chunk: Buffer, _encoding, callback) {
bytesUploaded += chunk.length;
options.onProgress?.({ bytesUploaded });
callback(undefined, chunk);
},
});
tar.stdout.pipe(progressStream).pipe(ssh.stdin);
tar.stderr.on('data', (chunk: Buffer) => {
tarStderr += chunk.toString('utf8');
});
packages/ssh/ts/plugins.ts
+2 -1
View File
@@ -4,5 +4,6 @@ import * as fsSync from 'node:fs';
import * as net from 'node:net';
import * as os from 'node:os';
import * as path from 'node:path';
import * as stream from 'node:stream';
export { childProcess, fs, fsSync, net, os, path };
export { childProcess, fs, fsSync, net, os, path, stream };
test/test.installer.node.ts
+13
View File
@@ -3,6 +3,7 @@ import {
createRemoteEphemeralBootstrapCommand,
createRemoteEphemeralReadinessCommand,
createRemoteEphemeralRuntimeCacheCheckCommand,
createRemoteEphemeralRuntimeMarkCommand,
createRemoteBootstrapCommand,
createRemoteInstallCommand,
createRemoteServerInstallPlan,
@@ -113,4 +114,16 @@ tap.test('should render ephemeral runtime cache check command', async () => {
expect(cacheCheckCommand).toInclude('runtimeCache=hit');
});
tap.test('should render ephemeral runtime mark command', async () => {
const markCommand = createRemoteEphemeralRuntimeMarkCommand({
runtimeRoot: '/tmp/gitzone-ide-0.1.0-deadbeef',
runtimeSha256: 'deadbeef',
});
expect(markCommand).toInclude('.gitzone-runtime-sha256.tmp');
expect(markCommand).toInclude("printf '%s");
expect(markCommand).toInclude("'deadbeef'");
expect(markCommand).toInclude('runtimeCache=stored');
});
export default tap.start();