push.rocks/smartfile
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(fs): Improve fs and stream handling, enhance SmartFile/StreamFile, update tests and CI configs

Browse Source
This commit is contained in:
Jürgen Kunz
2025-08-18 00:13:03 +00:00
parent 9b0d89b9ef
commit cd147ca38e
25 changed files with 3003 additions and 1221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
ts/fs.ts
View File

@@ -74,21 +74,45 @@ export const isFile = (pathArg): boolean => {
/**
* copies a file or directory from A to B on the local disk
*/
export const copy = async (fromArg: string, toArg: string, optionsArg?: plugins.fsExtra.CopyOptions & { replaceTargetDir?: boolean }): Promise<void> => {
if (optionsArg?.replaceTargetDir && isDirectory(fromArg) && isDirectory(toArg)) {
export const copy = async (
fromArg: string,
toArg: string,
optionsArg?: plugins.fsExtra.CopyOptions & { replaceTargetDir?: boolean },
): Promise<void> => {
if (
optionsArg?.replaceTargetDir &&
isDirectory(fromArg) &&
isDirectory(toArg)
) {
await remove(toArg);
}
return await plugins.fsExtra.copy(fromArg, toArg, optionsArg as plugins.fsExtra.CopyOptions);
return await plugins.fsExtra.copy(
fromArg,
toArg,
optionsArg as plugins.fsExtra.CopyOptions,
);
};
/**
* copies a file or directory SYNCHRONOUSLY from A to B on the local disk
*/
export const copySync = (fromArg: string, toArg: string, optionsArg?: plugins.fsExtra.CopyOptionsSync & { replaceTargetDir?: boolean }): void => {
if (optionsArg?.replaceTargetDir && isDirectory(fromArg) && isDirectory(toArg)) {
export const copySync = (
fromArg: string,
toArg: string,
optionsArg?: plugins.fsExtra.CopyOptionsSync & { replaceTargetDir?: boolean },
): void => {
if (
optionsArg?.replaceTargetDir &&
isDirectory(fromArg) &&
isDirectory(toArg)
) {
removeSync(toArg);
}
return plugins.fsExtra.copySync(fromArg, toArg, optionsArg as plugins.fsExtra.CopyOptionsSync);
return plugins.fsExtra.copySync(
fromArg,
toArg,
optionsArg as plugins.fsExtra.CopyOptionsSync,
);
};
/**
@@ -130,7 +154,10 @@ export const ensureEmptyDirSync = (dirPathArg: string) => {
* @returns Promise<void>
* @exec ASYNC
*/
export const ensureFile = async (filePathArg, initFileStringArg): Promise<void> => {
export const ensureFile = async (
filePathArg,
initFileStringArg,
): Promise<void> => {
ensureFileSync(filePathArg, initFileStringArg);
};
@@ -141,7 +168,10 @@ export const ensureFile = async (filePathArg, initFileStringArg): Promise<void>
* @returns Promise<void>
* @exec SYNC
*/
export const ensureFileSync = (filePathArg: string, initFileStringArg: string): void => {
export const ensureFileSync = (
filePathArg: string,
initFileStringArg: string,
): void => {
if (fileExistsSync(filePathArg)) {
return null;
} else {
@@ -197,7 +227,9 @@ export const removeManySync = (filePathArrayArg: string[]): void => {
export const toObjectSync = (filePathArg, fileTypeArg?) => {
const fileString = plugins.fsExtra.readFileSync(filePathArg, 'utf8');
let fileType;
fileTypeArg ? (fileType = fileTypeArg) : (fileType = interpreter.filetype(filePathArg));
fileTypeArg
? (fileType = fileTypeArg)
: (fileType = interpreter.filetype(filePathArg));
try {
return interpreter.objectFile(fileString, fileType);
} catch (err) {
@@ -211,7 +243,10 @@ export const toObjectSync = (filePathArg, fileTypeArg?) => {
*/
export const toStringSync = (filePath: string): string => {
const encoding = plugins.smartmime.getEncodingForPathSync(filePath);
let fileString: string | Buffer = plugins.fsExtra.readFileSync(filePath, encoding);
let fileString: string | Buffer = plugins.fsExtra.readFileSync(
filePath,
encoding,
);
if (Buffer.isBuffer(fileString)) {
fileString = fileString.toString('binary');
}
@@ -238,7 +273,10 @@ export const toReadStream = (filePath: string): plugins.fs.ReadStream => {
return plugins.fsExtra.createReadStream(filePath);
};
export const fileTreeToHash = async (dirPathArg: string, miniMatchFilter: string) => {
export const fileTreeToHash = async (
dirPathArg: string,
miniMatchFilter: string,
) => {
const fileTreeObject = await fileTreeToObject(dirPathArg, miniMatchFilter);
let combinedString = '';
for (const smartfile of fileTreeObject) {
@@ -253,7 +291,10 @@ export const fileTreeToHash = async (dirPathArg: string, miniMatchFilter: string
* @param dirPathArg the directory to start from
* @param miniMatchFilter a minimatch filter of what files to include
*/
export const fileTreeToObject = async (dirPathArg: string, miniMatchFilter: string) => {
export const fileTreeToObject = async (
dirPathArg: string,
miniMatchFilter: string,
) => {
// handle absolute miniMatchFilter
let dirPath: string;
if (plugins.path.isAbsolute(miniMatchFilter)) {
@@ -280,7 +321,7 @@ export const fileTreeToObject = async (dirPathArg: string, miniMatchFilter: stri
contentBuffer: fileBuffer,
base: dirPath,
path: filePath,
})
}),
);
}
return smartfileArray;
@@ -290,7 +331,10 @@ export const fileTreeToObject = async (dirPathArg: string, miniMatchFilter: stri
* lists Folders in a directory on local disk
* @returns Promise with an array that contains the folder names
*/
export const listFolders = async (pathArg: string, regexFilter?: RegExp): Promise<string[]> => {
export const listFolders = async (
pathArg: string,
regexFilter?: RegExp,
): Promise<string[]> => {
return listFoldersSync(pathArg, regexFilter);
};
@@ -298,9 +342,14 @@ export const listFolders = async (pathArg: string, regexFilter?: RegExp): Promis
* lists Folders SYNCHRONOUSLY in a directory on local disk
* @returns an array with the folder names as strings
*/
export const listFoldersSync = (pathArg: string, regexFilter?: RegExp): string[] => {
export const listFoldersSync = (
pathArg: string,
regexFilter?: RegExp,
): string[] => {
let folderArray = plugins.fsExtra.readdirSync(pathArg).filter((file) => {
return plugins.fsExtra.statSync(plugins.path.join(pathArg, file)).isDirectory();
return plugins.fsExtra
.statSync(plugins.path.join(pathArg, file))
.isDirectory();
});
if (regexFilter) {
folderArray = folderArray.filter((fileItem) => {
@@ -314,7 +363,10 @@ export const listFoldersSync = (pathArg: string, regexFilter?: RegExp): string[]
* lists Files in a directory on local disk
* @returns Promise
*/
export const listFiles = async (pathArg: string, regexFilter?: RegExp): Promise<string[]> => {
export const listFiles = async (
pathArg: string,
regexFilter?: RegExp,
): Promise<string[]> => {
return listFilesSync(pathArg, regexFilter);
};
@@ -322,7 +374,10 @@ export const listFiles = async (pathArg: string, regexFilter?: RegExp): Promise<
* lists Files SYNCHRONOUSLY in a directory on local disk
* @returns an array with the folder names as strings
*/
export const listFilesSync = (pathArg: string, regexFilter?: RegExp): string[] => {
export const listFilesSync = (
pathArg: string,
regexFilter?: RegExp,
): string[] => {
let fileArray = plugins.fsExtra.readdirSync(pathArg).filter((file) => {
return plugins.fsExtra.statSync(plugins.path.join(pathArg, file)).isFile();
});
@@ -338,7 +393,10 @@ export const listFilesSync = (pathArg: string, regexFilter?: RegExp): string[] =
* lists all items (folders AND files) in a directory on local disk
* @returns Promise<string[]>
*/
export const listAllItems = async (pathArg: string, regexFilter?: RegExp): Promise<string[]> => {
export const listAllItems = async (
pathArg: string,
regexFilter?: RegExp,
): Promise<string[]> => {
return listAllItemsSync(pathArg, regexFilter);
};
@@ -347,7 +405,10 @@ export const listAllItems = async (pathArg: string, regexFilter?: RegExp): Promi
* @returns an array with the folder names as strings
* @executes SYNC
*/
export const listAllItemsSync = (pathArg: string, regexFilter?: RegExp): string[] => {
export const listAllItemsSync = (
pathArg: string,
regexFilter?: RegExp,
): string[] => {
let allItmesArray = plugins.fsExtra.readdirSync(pathArg).filter((file) => {
return plugins.fsExtra.statSync(plugins.path.join(pathArg, file)).isFile();
});
@@ -367,7 +428,7 @@ export const listAllItemsSync = (pathArg: string, regexFilter?: RegExp): string[
export const listFileTree = async (
dirPathArg: string,
miniMatchFilter: string,
absolutePathsBool: boolean = false
absolutePathsBool: boolean = false,
): Promise<string[]> => {
// handle absolute miniMatchFilter
let dirPath: string;
@@ -400,11 +461,11 @@ export const listFileTree = async (
const allFiles = new Set<string>();
for (const pattern of patterns) {
const files = await plugins.glob.glob(pattern, options);
files.forEach(file => allFiles.add(file));
files.forEach((file) => allFiles.add(file));
}
let fileList = Array.from(allFiles).sort();
if (absolutePathsBool) {
fileList = fileList.map((filePath) => {
return plugins.path.resolve(plugins.path.join(dirPath, filePath));
@@ -429,7 +490,7 @@ export const listFileTree = async (
*/
export const waitForFileToBeReady = async (
fileOrDirPathArg: string,
timeoutMs: number = 60000
timeoutMs: number = 60000,
): Promise<void> => {
const startTime = Date.now();
@@ -442,7 +503,7 @@ export const waitForFileToBeReady = async (
while (true) {
try {
await plugins.smartpromise.fromCallback((cb) =>
plugins.fs.access(pathToCheck, plugins.fs.constants.F_OK, cb)
plugins.fs.access(pathToCheck, plugins.fs.constants.F_OK, cb),
);
return;
} catch (err: any) {
@@ -463,19 +524,23 @@ export const waitForFileToBeReady = async (
* @param filePathArg The path of the file to check.
* @returns A promise that resolves once the file stops changing.
*/
const waitForSingleFileToBeStable = async (filePathArg: string): Promise<void> => {
const waitForSingleFileToBeStable = async (
filePathArg: string,
): Promise<void> => {
let lastFileSize = -1;
let fileIsStable = false;
// We'll create a helper for repeated stats-checking logic
const checkFileStability = async () => {
try {
const stats = await plugins.smartpromise.fromCallback<plugins.fs.Stats>((cb) =>
plugins.fs.stat(filePathArg, cb)
const stats = await plugins.smartpromise.fromCallback<plugins.fs.Stats>(
(cb) => plugins.fs.stat(filePathArg, cb),
);
if (stats.isDirectory()) {
// If it unexpectedly turns out to be a directory here, throw
throw new Error(`Expected a file but found a directory: ${filePathArg}`);
throw new Error(
`Expected a file but found a directory: ${filePathArg}`,
);
}
if (stats.size === lastFileSize) {
fileIsStable = true;
@@ -495,17 +560,23 @@ export const waitForFileToBeReady = async (
await ensurePathExists(filePathArg);
// Set up a watcher on the file itself
const fileWatcher = plugins.fs.watch(filePathArg, { persistent: true }, async () => {
if (!fileIsStable) {
await checkFileStability();
}
});
const fileWatcher = plugins.fs.watch(
filePathArg,
{ persistent: true },
async () => {
if (!fileIsStable) {
await checkFileStability();
}
},
);
try {
// Poll until stable or timeout
while (!fileIsStable) {
if (Date.now() - startTime > timeoutMs) {
throw new Error(`Timeout waiting for file to stabilize: ${filePathArg}`);
throw new Error(
`Timeout waiting for file to stabilize: ${filePathArg}`,
);
}
await checkFileStability();
if (!fileIsStable) {
@@ -526,7 +597,7 @@ export const waitForFileToBeReady = async (
try {
await ensurePathExists(fileOrDirPathArg);
return await plugins.smartpromise.fromCallback<plugins.fs.Stats>((cb) =>
plugins.fs.stat(fileOrDirPathArg, cb)
plugins.fs.stat(fileOrDirPathArg, cb),
);
} catch (err) {
// If there's an error (including timeout), just rethrow
@@ -546,14 +617,15 @@ export const waitForFileToBeReady = async (
// Helper to find the first file in the directory if it exists
const getFirstFileInDirectory = async (): Promise<string | null> => {
const entries = await plugins.smartpromise.fromCallback<string[]>((cb) =>
plugins.fs.readdir(dirPath, cb)
plugins.fs.readdir(dirPath, cb),
);
// We only want actual files, not subdirectories
for (const entry of entries) {
const entryPath = plugins.path.join(dirPath, entry);
const entryStats = await plugins.smartpromise.fromCallback<plugins.fs.Stats>((cb) =>
plugins.fs.stat(entryPath, cb)
);
const entryStats =
await plugins.smartpromise.fromCallback<plugins.fs.Stats>((cb) =>
plugins.fs.stat(entryPath, cb),
);
if (entryStats.isFile()) {
return entryPath;
}
@@ -570,7 +642,9 @@ export const waitForFileToBeReady = async (
// We'll poll for the existence of a file in that directory
while (!firstFilePath) {
if (Date.now() - startTime > timeoutMs) {
throw new Error(`Timeout waiting for a file to appear in directory: ${dirPath}`);
throw new Error(
`Timeout waiting for a file to appear in directory: ${dirPath}`,
);
}
firstFilePath = await getFirstFileInDirectory();
if (!firstFilePath) {
@@ -597,7 +671,7 @@ export let toFs = async (
filePathArg: string,
optionsArg: {
respectRelative?: boolean;
} = {}
} = {},
) => {
const done = plugins.smartpromise.defer();
@@ -627,7 +701,12 @@ export let toFs = async (
throw new Error('fileContent is neither string nor Smartfile');
}
await ensureDir(plugins.path.parse(filePath).dir);
plugins.fsExtra.writeFile(filePath, fileContent, { encoding: fileEncoding }, done.resolve);
plugins.fsExtra.writeFile(
filePath,
fileContent,
{ encoding: fileEncoding },
done.resolve,
);
return await done.promise;
};