From 79d48b08751f2b0330b073aeb6d9023cf6b45674 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Thu, 5 Mar 2026 15:55:42 +0000 Subject: [PATCH] fix(compiler): defer unpacking until after all compilations and remove diagnostic filesystem syncs to avoid XFS metadata visibility issues --- changelog.md | 8 ++ ts/00_commitinfo_data.ts | 2 +- ts/mod_compiler/classes.tscompiler.ts | 142 +++----------------------- ts/mod_unpack/classes.tsunpacker.ts | 21 +--- 4 files changed, 27 insertions(+), 146 deletions(-) diff --git a/changelog.md b/changelog.md index d85a326..21e2e91 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 2026-03-05 - 4.2.3 - fix(compiler) +defer unpacking until after all compilations and remove diagnostic filesystem syncs to avoid XFS metadata visibility issues + +- Queue pending unpack operations during compilation and run them after all compile tasks complete to avoid modifying output directories while other compilations are writing. +- Remove TypeScript sys interception, execSync('sync') calls, and per-unpack fs.fsyncSync usage that attempted to work around XFS delayed metadata commits; rely on performing all unpacks after compilation instead. +- Clean up noisy diagnostic code (external 'ls' comparisons, readdir snapshots) and simplify logging of unpack results. +- Remove unused imports (fs and child_process.execSync) from the compiler module. + ## 2026-03-05 - 4.2.2 - fix(compiler) force global filesystem sync to flush XFS delayed logging and add diagnostics comparing Node's readdirSync with system ls to detect directory entry inconsistencies diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index df38eb4..954108c 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@git.zone/tsbuild', - version: '4.2.2', + version: '4.2.3', description: 'A tool for compiling TypeScript files using the latest nightly features, offering flexible APIs and a CLI for streamlined development.' } diff --git a/ts/mod_compiler/classes.tscompiler.ts b/ts/mod_compiler/classes.tscompiler.ts index 716c876..9ef550c 100644 --- a/ts/mod_compiler/classes.tscompiler.ts +++ b/ts/mod_compiler/classes.tscompiler.ts @@ -1,7 +1,5 @@ import type { CompilerOptions, Diagnostic, Program } from 'typescript'; import typescript from 'typescript'; -import * as fs from 'fs'; -import { execSync } from 'child_process'; import * as smartdelay from '@push.rocks/smartdelay'; import * as smartpromise from '@push.rocks/smartpromise'; import * as smartpath from '@push.rocks/smartpath'; @@ -326,6 +324,12 @@ export class TsCompiler { console.log(''); } + // Collect unpack tasks to perform AFTER all compilations complete. + // This prevents filesystem metadata corruption on XFS where heavy write + // activity during subsequent compilations can make freshly-renamed entries + // in previously-unpacked directories invisible or lost. + const pendingUnpacks: Array<{ pattern: string; destDir: string }> = []; + for (const pattern of Object.keys(globPatterns)) { const destPath = globPatterns[pattern]; if (!pattern || !destPath) continue; @@ -339,31 +343,13 @@ export class TsCompiler { // Get destination directory as absolute path const destDir = smartpath.transform.toAbsolute(destPath, this.cwd) as string; - // Diagnostic helper - const diagSnap = (label: string) => { - if (!isQuiet && !isJson) { - for (const prevDir of successfulOutputDirs) { - try { - const entries = fs.readdirSync(prevDir); - const dirs = entries.filter(e => { try { return fs.statSync(prevDir + '/' + e).isDirectory(); } catch { return false; } }); - const shortDir = prevDir.replace(this.cwd + '/', ''); - console.log(` ๐Ÿ“‹ [${label}] ${shortDir}: ${entries.length} entries, ${dirs.length} dirs [${entries.sort().join(', ')}]`); - } catch { - console.log(` ๐Ÿ“‹ [${label}] ${prevDir.replace(this.cwd + '/', '')}: MISSING!`); - } - } - } - }; - // Clear the destination directory before compilation if it exists - diagSnap('pre-clear'); if (await FsHelpers.directoryExists(destDir)) { if (!isQuiet && !isJson) { console.log(`๐Ÿงน Clearing output directory: ${destPath}`); } await FsHelpers.removeDirectory(destDir); } - diagSnap('post-clear'); // Update compiler options with the output directory const options: CompilerOptions = { @@ -381,107 +367,22 @@ export class TsCompiler { fileCount: absoluteFiles.length, }; - // Diagnostic: intercept TypeScript sys operations during compilation to detect - // any unexpected writes/deletes in previously compiled output directories - const watchedDirs = successfulOutputDirs.filter(d => d !== destDir); - let interceptedOps: string[] = []; - const origSysWriteFile = typescript.sys.writeFile; - const origSysDeleteFile = typescript.sys.deleteFile; - const origSysCreateDir = typescript.sys.createDirectory; - if (watchedDirs.length > 0 && !isQuiet && !isJson) { - typescript.sys.writeFile = (p: string, data: string, writeBom?: boolean) => { - if (watchedDirs.some(d => p.startsWith(d + '/'))) { - interceptedOps.push(`sys.writeFile: ${p}`); - } - return origSysWriteFile.call(typescript.sys, p, data, writeBom); - }; - if (origSysDeleteFile) { - typescript.sys.deleteFile = (p: string) => { - if (watchedDirs.some(d => p.startsWith(d + '/'))) { - interceptedOps.push(`sys.deleteFile: ${p}`); - } - return origSysDeleteFile!.call(typescript.sys, p); - }; - } - typescript.sys.createDirectory = (p: string) => { - if (watchedDirs.some(d => p.startsWith(d + '/'))) { - interceptedOps.push(`sys.createDirectory: ${p}`); - } - return origSysCreateDir.call(typescript.sys, p); - }; - } - const result = await this.compileFiles(absoluteFiles, options, taskInfo); emittedFiles.push(...result.emittedFiles); errorSummaries.push(result.errorSummary); - // Restore original sys methods and report any intercepted operations - if (watchedDirs.length > 0 && !isQuiet && !isJson) { - typescript.sys.writeFile = origSysWriteFile; - typescript.sys.deleteFile = origSysDeleteFile; - typescript.sys.createDirectory = origSysCreateDir; - if (interceptedOps.length > 0) { - console.log(` โš ๏ธ [diag] ${interceptedOps.length} TypeScript sys ops on previous output dirs:`); - for (const op of interceptedOps.slice(0, 30)) { - console.log(` ${op.replace(this.cwd + '/', '')}`); - } - if (interceptedOps.length > 30) { - console.log(` ... and ${interceptedOps.length - 30} more`); - } - } else { - console.log(` โ„น๏ธ [diag] No TypeScript sys ops on previous output dirs during this compilation`); - } - } - diagSnap('post-compile'); - - // Diagnostic: log emitted files that went to unexpected directories - if (!isQuiet && !isJson && result.emittedFiles.length > 0) { - const unexpectedFiles = result.emittedFiles.filter(f => !f.startsWith(destDir + '/') && !f.startsWith(destDir + '\\')); - if (unexpectedFiles.length > 0) { - console.log(` โš ๏ธ [diag] ${unexpectedFiles.length} files emitted OUTSIDE ${destPath}:`); - for (const f of unexpectedFiles.slice(0, 20)) { - console.log(` ${f.replace(this.cwd + '/', '')}`); - } - if (unexpectedFiles.length > 20) { - console.log(` ... and ${unexpectedFiles.length - 20} more`); - } - } - } - - // Perform unpack if compilation succeeded + // Queue unpack for after all compilations (don't modify output dirs between compilations) if (result.errorSummary.totalErrors === 0) { - await performUnpack(pattern, destDir, this.cwd); + pendingUnpacks.push({ pattern, destDir }); successfulOutputDirs.push(destDir); } + } - // Force global filesystem sync to flush XFS delayed logging (CIL) - // before the next compilation step. Per-directory fsync was insufficient; - // XFS defers parent B+tree metadata commits which can make sibling - // directory entries invisible to readdirSync after heavy write activity. - execSync('sync'); - - // Diagnostic: compare readdirSync vs external ls to detect Node.js caching - if (!isQuiet && !isJson) { - for (const prevDir of successfulOutputDirs) { - try { - const nodeEntries = fs.readdirSync(prevDir); - const nodeDirs = nodeEntries.filter(e => { try { return fs.statSync(prevDir + '/' + e).isDirectory(); } catch { return false; } }); - const lsOutput = execSync(`ls -1 "${prevDir}"`, { encoding: 'utf8' }).trim(); - const lsEntries = lsOutput ? lsOutput.split('\n') : []; - const shortDir = prevDir.replace(this.cwd + '/', ''); - if (nodeEntries.length !== lsEntries.length) { - console.log(` โš ๏ธ [diag] ${shortDir}: readdirSync=${nodeEntries.length}, ls=${lsEntries.length} (MISMATCH!)`); - console.log(` readdirSync: [${nodeEntries.sort().join(', ')}]`); - console.log(` ls: [${lsEntries.sort().join(', ')}]`); - } else { - console.log(` ๐Ÿ“‹ [diag] ${shortDir}: ${nodeEntries.length} entries, ${nodeDirs.length} dirs โœ“`); - } - } catch { - console.log(` ๐Ÿ“‹ [diag] ${prevDir.replace(this.cwd + '/', '')}: MISSING!`); - } - } - } - + // Perform all unpacks after all compilations are done. + // This ensures no output directory is modified while subsequent compilations + // are performing heavy filesystem writes to sibling directories. + for (const { pattern, destDir } of pendingUnpacks) { + await performUnpack(pattern, destDir, this.cwd); } // Rewrite import paths in all output directories to handle cross-module references @@ -496,21 +397,6 @@ export class TsCompiler { if (totalRewritten > 0 && !isQuiet && !isJson) { console.log(` ๐Ÿ”„ Rewrote import paths in ${totalRewritten} file${totalRewritten !== 1 ? 's' : ''}`); } - - // Diagnostic: log output directory states after path rewriting - if (!isQuiet && !isJson) { - for (const dir of successfulOutputDirs) { - try { - const entries = fs.readdirSync(dir); - const dirs = entries.filter(e => { - try { return fs.statSync(dir + '/' + e).isDirectory(); } catch { return false; } - }); - console.log(` ๐Ÿ“‹ [diag-post-rewrite] ${dir.replace(this.cwd + '/', '')}: ${entries.length} entries, ${dirs.length} dirs`); - } catch { - console.log(` ๐Ÿ“‹ [diag-post-rewrite] ${dir.replace(this.cwd + '/', '')}: MISSING!`); - } - } - } } // Merge all error summaries diff --git a/ts/mod_unpack/classes.tsunpacker.ts b/ts/mod_unpack/classes.tsunpacker.ts index 7094ef1..950810c 100644 --- a/ts/mod_unpack/classes.tsunpacker.ts +++ b/ts/mod_unpack/classes.tsunpacker.ts @@ -90,9 +90,9 @@ export class TsUnpacker { * 2. Moving contents of the nested source folder up to the dest dir * 3. Removing the now-empty nested source folder * - * Uses synchronous fs operations to avoid race conditions with - * async readdir returning partial/stale results under signal pressure - * or XFS metadata lag (observed in process-group environments like gitzone). + * Uses synchronous fs operations for reliability. + * Called after all compilations are complete (not between compilations) + * to avoid filesystem metadata issues on XFS. * * Returns true if unpacking was performed, false if skipped. */ @@ -107,17 +107,6 @@ export class TsUnpacker { const nestedPath = this.getNestedPath(); - // Force XFS to flush pending directory metadata before reading. - // XFS delayed logging (CIL) can defer metadata commits, causing - // readdirSync/opendirSync to return incomplete results immediately - // after TypeScript's emit() creates files via writeFileSync. - const destFd = fs.openSync(this.destDir, 'r'); - fs.fsyncSync(destFd); - fs.closeSync(destFd); - const nestedFd = fs.openSync(nestedPath, 'r'); - fs.fsyncSync(nestedFd); - fs.closeSync(nestedFd); - // Step 1: Remove sibling entries (everything in dest except the source folder) const destEntries = fs.readdirSync(this.destDir); for (const entry of destEntries) { @@ -138,9 +127,7 @@ export class TsUnpacker { // Step 3: Remove the now-empty nested directory fs.rmdirSync(nestedPath); - // Diagnostic: verify final state - const finalEntries = fs.readdirSync(this.destDir); - console.log(` ๐Ÿ“ฆ Unpacked ${this.sourceFolderName}: moved ${nestedEntries.length} entries, final: ${finalEntries.length} entries`); + console.log(` ๐Ÿ“ฆ Unpacked ${this.sourceFolderName}: ${nestedEntries.length} entries`); return true; }