push.rocks/smartfile
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(fs): Improve fs and stream handling, enhance SmartFile/StreamFile, update tests and CI configs

Browse Source
This commit is contained in:
Jürgen Kunz
2025-08-18 00:13:03 +00:00
parent 9b0d89b9ef
commit cd147ca38e
25 changed files with 3003 additions and 1221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartfile',
version: '11.2.5',
version: '11.2.6',
description: 'Provides comprehensive tools for efficient file management in Node.js using TypeScript, including handling streams, virtual directories, and various file operations.'
}

60
ts/classes.smartfile.ts
View File

@@ -20,7 +20,10 @@ export class SmartFile extends plugins.smartjson.Smartjson {
* creates a Smartfile from a filePath
* @param filePath
*/
public static async fromFilePath(filePath: string, baseArg: string = process.cwd()) {
public static async fromFilePath(
filePath: string,
baseArg: string = process.cwd(),
) {
filePath = plugins.path.resolve(filePath);
const fileBuffer = fs.toBufferSync(filePath);
const smartfile = new SmartFile({
@@ -34,7 +37,7 @@ export class SmartFile extends plugins.smartjson.Smartjson {
public static async fromBuffer(
filePath: string,
contentBufferArg: Buffer,
baseArg: string = process.cwd()
baseArg: string = process.cwd(),
) {
const smartfile = new SmartFile({
contentBuffer: contentBufferArg,
@@ -49,7 +52,7 @@ export class SmartFile extends plugins.smartjson.Smartjson {
filePath: string,
contentStringArg: string,
encodingArg: 'utf8' | 'binary',
baseArg = process.cwd()
baseArg = process.cwd(),
) {
const smartfile = new SmartFile({
contentBuffer: Buffer.from(contentStringArg, encodingArg),
@@ -73,7 +76,7 @@ export class SmartFile extends plugins.smartjson.Smartjson {
public static async fromStream(
stream: plugins.stream.Readable,
filePath: string,
baseArg: string = process.cwd()
baseArg: string = process.cwd(),
): Promise<SmartFile> {
return new Promise<SmartFile>((resolve, reject) => {
const chunks: Buffer[] = [];
@@ -92,8 +95,12 @@ export class SmartFile extends plugins.smartjson.Smartjson {
}
public static async fromUrl(urlArg: string) {
const response = await plugins.smartrequest.getBinary(urlArg);
const smartfile = await SmartFile.fromBuffer(urlArg, response.body);
const response = await plugins.smartrequest.SmartRequest.create()
.url(urlArg)
.accept('binary')
.get();
const buffer = Buffer.from(await response.arrayBuffer());
const smartfile = await SmartFile.fromBuffer(urlArg, buffer);
return smartfile;
}
@@ -159,7 +166,10 @@ export class SmartFile extends plugins.smartjson.Smartjson {
* set contents from string
* @param contentString
*/
public setContentsFromString(contentString: string, encodingArg: 'utf8' | 'binary' = 'utf8') {
public setContentsFromString(
contentString: string,
encodingArg: 'utf8' | 'binary' = 'utf8',
) {
this.contents = Buffer.from(contentString, encodingArg);
}
@@ -169,7 +179,10 @@ export class SmartFile extends plugins.smartjson.Smartjson {
* - no argument write to exactly where the file was picked up
*/
public async write() {
let writePath = plugins.smartpath.transform.makeAbsolute(this.path, this.base);
let writePath = plugins.smartpath.transform.makeAbsolute(
this.path,
this.base,
);
console.log(`writing to ${writePath}`);
await memory.toFs(this.contentBuffer, writePath);
}
@@ -202,7 +215,9 @@ export class SmartFile extends plugins.smartjson.Smartjson {
* read file from disk
*/
public async read() {
this.contentBuffer = await fs.toBuffer(plugins.path.join(this.base, this.path));
this.contentBuffer = await fs.toBuffer(
plugins.path.join(this.base, this.path),
);
}
/**
@@ -220,7 +235,10 @@ export class SmartFile extends plugins.smartjson.Smartjson {
* @param writeToDisk (optional) If true, also renames the file on the disk.
* @returns The updated file path after renaming.
*/
public async rename(newName: string, writeToDisk: boolean = false): Promise<string> {
public async rename(
newName: string,
writeToDisk: boolean = false,
): Promise<string> {
// Validate the new name
if (!newName || typeof newName !== 'string') {
throw new Error('Invalid new name provided.');
@@ -238,8 +256,14 @@ export class SmartFile extends plugins.smartjson.Smartjson {
// Optionally write the renamed file to disk
if (writeToDisk) {
const oldAbsolutePath = plugins.smartpath.transform.makeAbsolute(oldFilePath, this.base);
const newAbsolutePath = plugins.smartpath.transform.makeAbsolute(newFilePath, this.base);
const oldAbsolutePath = plugins.smartpath.transform.makeAbsolute(
oldFilePath,
this.base,
);
const newAbsolutePath = plugins.smartpath.transform.makeAbsolute(
newFilePath,
this.base,
);
// Rename the file on disk
await plugins.fsExtra.rename(oldAbsolutePath, newAbsolutePath);
@@ -310,8 +334,12 @@ export class SmartFile extends plugins.smartjson.Smartjson {
public async getHash(typeArg: 'path' | 'content' | 'all' = 'all') {
const pathHash = await plugins.smarthash.sha256FromString(this.path);
const contentHash = await plugins.smarthash.sha256FromBuffer(this.contentBuffer);
const combinedHash = await plugins.smarthash.sha256FromString(pathHash + contentHash);
const contentHash = await plugins.smarthash.sha256FromBuffer(
this.contentBuffer,
);
const combinedHash = await plugins.smarthash.sha256FromString(
pathHash + contentHash,
);
switch (typeArg) {
case 'path':
return pathHash;
@@ -329,7 +357,9 @@ export class SmartFile extends plugins.smartjson.Smartjson {
this.path = this.path.replace(new RegExp(oldFileName + '$'), fileNameArg);
}
public async editContentAsString(editFuncArg: (fileStringArg: string) => Promise<string>) {
public async editContentAsString(
editFuncArg: (fileStringArg: string) => Promise<string>,
) {
const newFileString = await editFuncArg(this.contentBuffer.toString());
this.contentBuffer = Buffer.from(newFileString);
}

49
ts/classes.streamfile.ts
View File

@@ -3,7 +3,7 @@ import * as smartfileFs from './fs.js';
import * as smartfileFsStream from './fsstream.js';
import { Readable } from 'stream';
type TStreamSource = (streamFile: StreamFile) => Promise<Readable>;
type TStreamSource = (streamFile: StreamFile) => Promise<Readable | ReadableStream>;
/**
* The StreamFile class represents a file as a stream.
@@ -13,28 +13,41 @@ export class StreamFile {
// STATIC
public static async fromPath(filePath: string): Promise<StreamFile> {
const streamSource: TStreamSource = async (streamFileArg) => smartfileFsStream.createReadStream(filePath);
const streamSource: TStreamSource = async (streamFileArg) =>
smartfileFsStream.createReadStream(filePath);
const streamFile = new StreamFile(streamSource, filePath);
streamFile.multiUse = true;
streamFile.byteLengthComputeFunction = async () => {
const stats = await smartfileFs.stat(filePath);
return stats.size;
}
};
return streamFile;
}
public static async fromUrl(url: string): Promise<StreamFile> {
const streamSource: TStreamSource = async (streamFileArg) => plugins.smartrequest.getStream(url); // Replace with actual plugin method
const streamSource: TStreamSource = async (streamFileArg) => {
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.get();
return response.stream();
};
const streamFile = new StreamFile(streamSource);
streamFile.multiUse = true;
streamFile.byteLengthComputeFunction = async () => {
const response = await plugins.smartrequest.getBinary(url); // TODO: switch to future .getBinaryByteLength()
return response.body.length;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.accept('binary')
.get();
const buffer = Buffer.from(await response.arrayBuffer());
return buffer.length;
};
return streamFile;
}
public static fromBuffer(buffer: Buffer, relativeFilePath?: string): StreamFile {
public static fromBuffer(
buffer: Buffer,
relativeFilePath?: string,
): StreamFile {
const streamSource: TStreamSource = async (streamFileArg) => {
const stream = new Readable();
stream.push(buffer);
@@ -54,7 +67,11 @@ export class StreamFile {
* @param multiUse If true, the stream can be read multiple times, caching its content.
* @returns A StreamFile instance.
*/
public static fromStream(stream: Readable, relativeFilePath?: string, multiUse: boolean = false): StreamFile {
public static fromStream(
stream: Readable,
relativeFilePath?: string,
multiUse: boolean = false,
): StreamFile {
const streamSource: TStreamSource = (streamFileArg) => {
if (streamFileArg.multiUse) {
// If multi-use is enabled and we have cached content, create a new readable stream from the buffer
@@ -86,7 +103,6 @@ export class StreamFile {
return streamFile;
}
// INSTANCE
relativeFilePath?: string;
private streamSource: TStreamSource;
@@ -115,7 +131,16 @@ export class StreamFile {
* Creates a new readable stream from the source.
*/
public async createReadStream(): Promise<Readable> {
return this.streamSource(this);
const stream = await this.streamSource(this);
// Check if it's a Web ReadableStream and convert to Node.js Readable
if (stream && typeof (stream as any).getReader === 'function') {
// This is a Web ReadableStream, convert it to Node.js Readable
return Readable.fromWeb(stream as any);
}
// It's already a Node.js Readable stream
return stream as Readable;
}
/**
@@ -171,4 +196,4 @@ export class StreamFile {
return null;
}
}
}
}

27
ts/classes.virtualdirectory.ts
View File

@@ -2,33 +2,33 @@ import { SmartFile } from './classes.smartfile.js';
import * as plugins from './plugins.js';
import * as fs from './fs.js';
export interface IVirtualDirectoryConstructorOptions {
mode: ''
mode: '';
}
/**
* a virtual directory exposes a fs api
*/
export class VirtualDirectory {
consstructor(options = {}) {
}
consstructor(options = {}) {}
// STATIC
public static async fromFsDirPath(pathArg: string): Promise<VirtualDirectory> {
public static async fromFsDirPath(
pathArg: string,
): Promise<VirtualDirectory> {
const newVirtualDir = new VirtualDirectory();
newVirtualDir.addSmartfiles(await fs.fileTreeToObject(pathArg, '**/*'));
return newVirtualDir;
}
public static async fromVirtualDirTransferableObject(
virtualDirTransferableObjectArg: plugins.smartfileInterfaces.VirtualDirTransferableObject
virtualDirTransferableObjectArg: plugins.smartfileInterfaces.VirtualDirTransferableObject,
): Promise<VirtualDirectory> {
const newVirtualDir = new VirtualDirectory();
for (const fileArg of virtualDirTransferableObjectArg.files) {
newVirtualDir.addSmartfiles([SmartFile.enfoldFromJson(fileArg) as SmartFile]);
newVirtualDir.addSmartfiles([
SmartFile.enfoldFromJson(fileArg) as SmartFile,
]);
}
return newVirtualDir;
}
@@ -52,7 +52,9 @@ export class VirtualDirectory {
public async toVirtualDirTransferableObject(): Promise<plugins.smartfileInterfaces.VirtualDirTransferableObject> {
return {
files: this.smartfileArray.map((smartfileArg) => smartfileArg.foldToJson()),
files: this.smartfileArray.map((smartfileArg) =>
smartfileArg.foldToJson(),
),
};
}
@@ -78,7 +80,10 @@ export class VirtualDirectory {
return newVirtualDir;
}
public async addVirtualDirectory(virtualDir: VirtualDirectory, newRoot: string): Promise<void> {
public async addVirtualDirectory(
virtualDir: VirtualDirectory,
newRoot: string,
): Promise<void> {
for (const file of virtualDir.smartfileArray) {
file.path = plugins.path.join(newRoot, file.path);
}

165
ts/fs.ts
View File

@@ -74,21 +74,45 @@ export const isFile = (pathArg): boolean => {
/**
* copies a file or directory from A to B on the local disk
*/
export const copy = async (fromArg: string, toArg: string, optionsArg?: plugins.fsExtra.CopyOptions & { replaceTargetDir?: boolean }): Promise<void> => {
if (optionsArg?.replaceTargetDir && isDirectory(fromArg) && isDirectory(toArg)) {
export const copy = async (
fromArg: string,
toArg: string,
optionsArg?: plugins.fsExtra.CopyOptions & { replaceTargetDir?: boolean },
): Promise<void> => {
if (
optionsArg?.replaceTargetDir &&
isDirectory(fromArg) &&
isDirectory(toArg)
) {
await remove(toArg);
}
return await plugins.fsExtra.copy(fromArg, toArg, optionsArg as plugins.fsExtra.CopyOptions);
return await plugins.fsExtra.copy(
fromArg,
toArg,
optionsArg as plugins.fsExtra.CopyOptions,
);
};
/**
* copies a file or directory SYNCHRONOUSLY from A to B on the local disk
*/
export const copySync = (fromArg: string, toArg: string, optionsArg?: plugins.fsExtra.CopyOptionsSync & { replaceTargetDir?: boolean }): void => {
if (optionsArg?.replaceTargetDir && isDirectory(fromArg) && isDirectory(toArg)) {
export const copySync = (
fromArg: string,
toArg: string,
optionsArg?: plugins.fsExtra.CopyOptionsSync & { replaceTargetDir?: boolean },
): void => {
if (
optionsArg?.replaceTargetDir &&
isDirectory(fromArg) &&
isDirectory(toArg)
) {
removeSync(toArg);
}
return plugins.fsExtra.copySync(fromArg, toArg, optionsArg as plugins.fsExtra.CopyOptionsSync);
return plugins.fsExtra.copySync(
fromArg,
toArg,
optionsArg as plugins.fsExtra.CopyOptionsSync,
);
};
/**
@@ -130,7 +154,10 @@ export const ensureEmptyDirSync = (dirPathArg: string) => {
* @returns Promise<void>
* @exec ASYNC
*/
export const ensureFile = async (filePathArg, initFileStringArg): Promise<void> => {
export const ensureFile = async (
filePathArg,
initFileStringArg,
): Promise<void> => {
ensureFileSync(filePathArg, initFileStringArg);
};
@@ -141,7 +168,10 @@ export const ensureFile = async (filePathArg, initFileStringArg): Promise<void>
* @returns Promise<void>
* @exec SYNC
*/
export const ensureFileSync = (filePathArg: string, initFileStringArg: string): void => {
export const ensureFileSync = (
filePathArg: string,
initFileStringArg: string,
): void => {
if (fileExistsSync(filePathArg)) {
return null;
} else {
@@ -197,7 +227,9 @@ export const removeManySync = (filePathArrayArg: string[]): void => {
export const toObjectSync = (filePathArg, fileTypeArg?) => {
const fileString = plugins.fsExtra.readFileSync(filePathArg, 'utf8');
let fileType;
fileTypeArg ? (fileType = fileTypeArg) : (fileType = interpreter.filetype(filePathArg));
fileTypeArg
? (fileType = fileTypeArg)
: (fileType = interpreter.filetype(filePathArg));
try {
return interpreter.objectFile(fileString, fileType);
} catch (err) {
@@ -211,7 +243,10 @@ export const toObjectSync = (filePathArg, fileTypeArg?) => {
*/
export const toStringSync = (filePath: string): string => {
const encoding = plugins.smartmime.getEncodingForPathSync(filePath);
let fileString: string | Buffer = plugins.fsExtra.readFileSync(filePath, encoding);
let fileString: string | Buffer = plugins.fsExtra.readFileSync(
filePath,
encoding,
);
if (Buffer.isBuffer(fileString)) {
fileString = fileString.toString('binary');
}
@@ -238,7 +273,10 @@ export const toReadStream = (filePath: string): plugins.fs.ReadStream => {
return plugins.fsExtra.createReadStream(filePath);
};
export const fileTreeToHash = async (dirPathArg: string, miniMatchFilter: string) => {
export const fileTreeToHash = async (
dirPathArg: string,
miniMatchFilter: string,
) => {
const fileTreeObject = await fileTreeToObject(dirPathArg, miniMatchFilter);
let combinedString = '';
for (const smartfile of fileTreeObject) {
@@ -253,7 +291,10 @@ export const fileTreeToHash = async (dirPathArg: string, miniMatchFilter: string
* @param dirPathArg the directory to start from
* @param miniMatchFilter a minimatch filter of what files to include
*/
export const fileTreeToObject = async (dirPathArg: string, miniMatchFilter: string) => {
export const fileTreeToObject = async (
dirPathArg: string,
miniMatchFilter: string,
) => {
// handle absolute miniMatchFilter
let dirPath: string;
if (plugins.path.isAbsolute(miniMatchFilter)) {
@@ -280,7 +321,7 @@ export const fileTreeToObject = async (dirPathArg: string, miniMatchFilter: stri
contentBuffer: fileBuffer,
base: dirPath,
path: filePath,
})
}),
);
}
return smartfileArray;
@@ -290,7 +331,10 @@ export const fileTreeToObject = async (dirPathArg: string, miniMatchFilter: stri
* lists Folders in a directory on local disk
* @returns Promise with an array that contains the folder names
*/
export const listFolders = async (pathArg: string, regexFilter?: RegExp): Promise<string[]> => {
export const listFolders = async (
pathArg: string,
regexFilter?: RegExp,
): Promise<string[]> => {
return listFoldersSync(pathArg, regexFilter);
};
@@ -298,9 +342,14 @@ export const listFolders = async (pathArg: string, regexFilter?: RegExp): Promis
* lists Folders SYNCHRONOUSLY in a directory on local disk
* @returns an array with the folder names as strings
*/
export const listFoldersSync = (pathArg: string, regexFilter?: RegExp): string[] => {
export const listFoldersSync = (
pathArg: string,
regexFilter?: RegExp,
): string[] => {
let folderArray = plugins.fsExtra.readdirSync(pathArg).filter((file) => {
return plugins.fsExtra.statSync(plugins.path.join(pathArg, file)).isDirectory();
return plugins.fsExtra
.statSync(plugins.path.join(pathArg, file))
.isDirectory();
});
if (regexFilter) {
folderArray = folderArray.filter((fileItem) => {
@@ -314,7 +363,10 @@ export const listFoldersSync = (pathArg: string, regexFilter?: RegExp): string[]
* lists Files in a directory on local disk
* @returns Promise
*/
export const listFiles = async (pathArg: string, regexFilter?: RegExp): Promise<string[]> => {
export const listFiles = async (
pathArg: string,
regexFilter?: RegExp,
): Promise<string[]> => {
return listFilesSync(pathArg, regexFilter);
};
@@ -322,7 +374,10 @@ export const listFiles = async (pathArg: string, regexFilter?: RegExp): Promise<
* lists Files SYNCHRONOUSLY in a directory on local disk
* @returns an array with the folder names as strings
*/
export const listFilesSync = (pathArg: string, regexFilter?: RegExp): string[] => {
export const listFilesSync = (
pathArg: string,
regexFilter?: RegExp,
): string[] => {
let fileArray = plugins.fsExtra.readdirSync(pathArg).filter((file) => {
return plugins.fsExtra.statSync(plugins.path.join(pathArg, file)).isFile();
});
@@ -338,7 +393,10 @@ export const listFilesSync = (pathArg: string, regexFilter?: RegExp): string[] =
* lists all items (folders AND files) in a directory on local disk
* @returns Promise<string[]>
*/
export const listAllItems = async (pathArg: string, regexFilter?: RegExp): Promise<string[]> => {
export const listAllItems = async (
pathArg: string,
regexFilter?: RegExp,
): Promise<string[]> => {
return listAllItemsSync(pathArg, regexFilter);
};
@@ -347,7 +405,10 @@ export const listAllItems = async (pathArg: string, regexFilter?: RegExp): Promi
* @returns an array with the folder names as strings
* @executes SYNC
*/
export const listAllItemsSync = (pathArg: string, regexFilter?: RegExp): string[] => {
export const listAllItemsSync = (
pathArg: string,
regexFilter?: RegExp,
): string[] => {
let allItmesArray = plugins.fsExtra.readdirSync(pathArg).filter((file) => {
return plugins.fsExtra.statSync(plugins.path.join(pathArg, file)).isFile();
});
@@ -367,7 +428,7 @@ export const listAllItemsSync = (pathArg: string, regexFilter?: RegExp): string[
export const listFileTree = async (
dirPathArg: string,
miniMatchFilter: string,
absolutePathsBool: boolean = false
absolutePathsBool: boolean = false,
): Promise<string[]> => {
// handle absolute miniMatchFilter
let dirPath: string;
@@ -400,11 +461,11 @@ export const listFileTree = async (
const allFiles = new Set<string>();
for (const pattern of patterns) {
const files = await plugins.glob.glob(pattern, options);
files.forEach(file => allFiles.add(file));
files.forEach((file) => allFiles.add(file));
}
let fileList = Array.from(allFiles).sort();
if (absolutePathsBool) {
fileList = fileList.map((filePath) => {
return plugins.path.resolve(plugins.path.join(dirPath, filePath));
@@ -429,7 +490,7 @@ export const listFileTree = async (
*/
export const waitForFileToBeReady = async (
fileOrDirPathArg: string,
timeoutMs: number = 60000
timeoutMs: number = 60000,
): Promise<void> => {
const startTime = Date.now();
@@ -442,7 +503,7 @@ export const waitForFileToBeReady = async (
while (true) {
try {
await plugins.smartpromise.fromCallback((cb) =>
plugins.fs.access(pathToCheck, plugins.fs.constants.F_OK, cb)
plugins.fs.access(pathToCheck, plugins.fs.constants.F_OK, cb),
);
return;
} catch (err: any) {
@@ -463,19 +524,23 @@ export const waitForFileToBeReady = async (
* @param filePathArg The path of the file to check.
* @returns A promise that resolves once the file stops changing.
*/
const waitForSingleFileToBeStable = async (filePathArg: string): Promise<void> => {
const waitForSingleFileToBeStable = async (
filePathArg: string,
): Promise<void> => {
let lastFileSize = -1;
let fileIsStable = false;
// We'll create a helper for repeated stats-checking logic
const checkFileStability = async () => {
try {
const stats = await plugins.smartpromise.fromCallback<plugins.fs.Stats>((cb) =>
plugins.fs.stat(filePathArg, cb)
const stats = await plugins.smartpromise.fromCallback<plugins.fs.Stats>(
(cb) => plugins.fs.stat(filePathArg, cb),
);
if (stats.isDirectory()) {
// If it unexpectedly turns out to be a directory here, throw
throw new Error(`Expected a file but found a directory: ${filePathArg}`);
throw new Error(
`Expected a file but found a directory: ${filePathArg}`,
);
}
if (stats.size === lastFileSize) {
fileIsStable = true;
@@ -495,17 +560,23 @@ export const waitForFileToBeReady = async (
await ensurePathExists(filePathArg);
// Set up a watcher on the file itself
const fileWatcher = plugins.fs.watch(filePathArg, { persistent: true }, async () => {
if (!fileIsStable) {
await checkFileStability();
}
});
const fileWatcher = plugins.fs.watch(
filePathArg,
{ persistent: true },
async () => {
if (!fileIsStable) {
await checkFileStability();
}
},
);
try {
// Poll until stable or timeout
while (!fileIsStable) {
if (Date.now() - startTime > timeoutMs) {
throw new Error(`Timeout waiting for file to stabilize: ${filePathArg}`);
throw new Error(
`Timeout waiting for file to stabilize: ${filePathArg}`,
);
}
await checkFileStability();
if (!fileIsStable) {
@@ -526,7 +597,7 @@ export const waitForFileToBeReady = async (
try {
await ensurePathExists(fileOrDirPathArg);
return await plugins.smartpromise.fromCallback<plugins.fs.Stats>((cb) =>
plugins.fs.stat(fileOrDirPathArg, cb)
plugins.fs.stat(fileOrDirPathArg, cb),
);
} catch (err) {
// If there's an error (including timeout), just rethrow
@@ -546,14 +617,15 @@ export const waitForFileToBeReady = async (
// Helper to find the first file in the directory if it exists
const getFirstFileInDirectory = async (): Promise<string | null> => {
const entries = await plugins.smartpromise.fromCallback<string[]>((cb) =>
plugins.fs.readdir(dirPath, cb)
plugins.fs.readdir(dirPath, cb),
);
// We only want actual files, not subdirectories
for (const entry of entries) {
const entryPath = plugins.path.join(dirPath, entry);
const entryStats = await plugins.smartpromise.fromCallback<plugins.fs.Stats>((cb) =>
plugins.fs.stat(entryPath, cb)
);
const entryStats =
await plugins.smartpromise.fromCallback<plugins.fs.Stats>((cb) =>
plugins.fs.stat(entryPath, cb),
);
if (entryStats.isFile()) {
return entryPath;
}
@@ -570,7 +642,9 @@ export const waitForFileToBeReady = async (
// We'll poll for the existence of a file in that directory
while (!firstFilePath) {
if (Date.now() - startTime > timeoutMs) {
throw new Error(`Timeout waiting for a file to appear in directory: ${dirPath}`);
throw new Error(
`Timeout waiting for a file to appear in directory: ${dirPath}`,
);
}
firstFilePath = await getFirstFileInDirectory();
if (!firstFilePath) {
@@ -597,7 +671,7 @@ export let toFs = async (
filePathArg: string,
optionsArg: {
respectRelative?: boolean;
} = {}
} = {},
) => {
const done = plugins.smartpromise.defer();
@@ -627,7 +701,12 @@ export let toFs = async (
throw new Error('fileContent is neither string nor Smartfile');
}
await ensureDir(plugins.path.parse(filePath).dir);
plugins.fsExtra.writeFile(filePath, fileContent, { encoding: fileEncoding }, done.resolve);
plugins.fsExtra.writeFile(
filePath,
fileContent,
{ encoding: fileEncoding },
done.resolve,
);
return await done.promise;
};

80
ts/fsstream.ts
View File

@@ -13,17 +13,17 @@ export const createWriteStream = (pathArg: string) => {
export const processFile = async (
filePath: string,
asyncFunc: (fileStream: plugins.stream.Readable) => Promise<void>
asyncFunc: (fileStream: plugins.stream.Readable) => Promise<void>,
): Promise<void> => {
return new Promise((resolve, reject) => {
const fileStream = createReadStream(filePath);
asyncFunc(fileStream).then(resolve).catch(reject);
});
}
};
export const processDirectory = async (
directoryPath: string,
asyncFunc: (fileStream: plugins.stream.Readable) => Promise<void>
asyncFunc: (fileStream: plugins.stream.Readable) => Promise<void>,
): Promise<void> => {
const files = plugins.fs.readdirSync(directoryPath, { withFileTypes: true });
@@ -41,12 +41,15 @@ export const processDirectory = async (
/**
* Checks if a file is ready to be streamed (exists and is not empty).
*/
export const isFileReadyForStreaming = async (filePathArg: string): Promise<boolean> => {
export const isFileReadyForStreaming = async (
filePathArg: string,
): Promise<boolean> => {
try {
const stats = await plugins.fs.promises.stat(filePathArg);
return stats.size > 0;
} catch (error) {
if (error.code === 'ENOENT') { // File does not exist
if (error.code === 'ENOENT') {
// File does not exist
return false;
}
throw error; // Rethrow other unexpected errors
@@ -56,7 +59,9 @@ export const isFileReadyForStreaming = async (filePathArg: string): Promise<bool
/**
* Waits for a file to be ready for streaming (exists and is not empty).
*/
export const waitForFileToBeReadyForStreaming = (filePathArg: string): Promise<void> => {
export const waitForFileToBeReadyForStreaming = (
filePathArg: string,
): Promise<void> => {
return new Promise((resolve, reject) => {
// Normalize and resolve the file path
const filePath = plugins.path.resolve(filePathArg);
@@ -80,11 +85,15 @@ export const waitForFileToBeReadyForStreaming = (filePathArg: string): Promise<v
};
// Set up file watcher
const watcher = plugins.fs.watch(filePath, { persistent: false }, (eventType) => {
if (eventType === 'change' || eventType === 'rename') {
checkFile(resolve, reject);
}
});
const watcher = plugins.fs.watch(
filePath,
{ persistent: false },
(eventType) => {
if (eventType === 'change' || eventType === 'rename') {
checkFile(resolve, reject);
}
},
);
// Check file immediately in case it's already ready
checkFile(resolve, reject);
@@ -105,7 +114,11 @@ export class SmartReadStream extends plugins.stream.Readable {
private endDelay: number;
private reading: boolean = false;
constructor(filePath: string, endDelay = 60000, opts?: plugins.stream.ReadableOptions) {
constructor(
filePath: string,
endDelay = 60000,
opts?: plugins.stream.ReadableOptions,
) {
super(opts);
this.filePath = filePath;
this.endDelay = endDelay;
@@ -165,26 +178,33 @@ export class SmartReadStream extends plugins.stream.Readable {
this.emit('error', err);
return;
}
plugins.fs.read(fd, buffer, 0, chunkSize, this.lastReadSize, (err, bytesRead, buffer) => {
if (err) {
this.emit('error', err);
return;
}
if (bytesRead > 0) {
this.lastReadSize += bytesRead;
this.push(buffer.slice(0, bytesRead)); // Push the data onto the stream
} else {
this.reading = false; // No more data to read for now
this.resetEndTimeout();
}
plugins.fs.close(fd, (err) => {
plugins.fs.read(
fd,
buffer,
0,
chunkSize,
this.lastReadSize,
(err, bytesRead, buffer) => {
if (err) {
this.emit('error', err);
return;
}
});
});
if (bytesRead > 0) {
this.lastReadSize += bytesRead;
this.push(buffer.slice(0, bytesRead)); // Push the data onto the stream
} else {
this.reading = false; // No more data to read for now
this.resetEndTimeout();
}
plugins.fs.close(fd, (err) => {
if (err) {
this.emit('error', err);
}
});
},
);
});
}
@@ -192,4 +212,4 @@ export class SmartReadStream extends plugins.stream.Readable {
this.cleanup();
callback(error);
}
}
}

14
ts/memory.ts
View File

@@ -27,7 +27,7 @@ export interface IToFsOptions {
export let toFs = async (
fileContentArg: string | Buffer | SmartFile | StreamFile,
filePathArg: string,
optionsArg: IToFsOptions = {}
optionsArg: IToFsOptions = {},
) => {
const done = plugins.smartpromise.defer();
@@ -57,7 +57,12 @@ export let toFs = async (
throw new Error('fileContent is neither string nor Smartfile');
}
await smartfileFs.ensureDir(plugins.path.parse(filePath).dir);
plugins.fsExtra.writeFile(filePath, fileContent, { encoding: fileEncoding }, done.resolve);
plugins.fsExtra.writeFile(
filePath,
fileContent,
{ encoding: fileEncoding },
done.resolve,
);
return await done.promise;
};
@@ -84,7 +89,10 @@ export const toFsSync = (fileArg: string, filePathArg: string) => {
plugins.fsExtra.writeFileSync(filePath, fileString, { encoding: 'utf8' });
};
export let smartfileArrayToFs = async (smartfileArrayArg: SmartFile[], dirArg: string) => {
export let smartfileArrayToFs = async (
smartfileArrayArg: SmartFile[],
dirArg: string,
) => {
await smartfileFs.ensureDir(dirArg);
for (const smartfile of smartfileArrayArg) {
await toFs(smartfile, dirArg, {