11.0.5

npmextra.json

@@ -7,7 +7,7 @@
   "gitzone": {
     "projectType": "npm",
     "module": {
-      "githost": "gitlab.com",
+      "githost": "code.foss.global",
       "gitscope": "push.rocks",
       "gitrepo": "smartfile",
       "description": "smart ways to work with files in nodejs",

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@push.rocks/smartfile",
   "private": false,
-  "version": "11.0.0",
+  "version": "11.0.5",
   "description": "offers smart ways to work with files in nodejs",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -26,7 +26,7 @@
   },
   "homepage": "https://gitlab.com/push.rocks/smartfile#readme",
   "dependencies": {
-    "@push.rocks/lik": "^6.0.5",
+    "@push.rocks/lik": "^6.0.12",
     "@push.rocks/smartdelay": "^3.0.5",
     "@push.rocks/smartfile-interfaces": "^1.0.7",
     "@push.rocks/smarthash": "^3.0.4",
@@ -34,11 +34,11 @@
     "@push.rocks/smartmime": "^1.0.5",
     "@push.rocks/smartpath": "^5.0.11",
     "@push.rocks/smartpromise": "^4.0.2",
-    "@push.rocks/smartrequest": "^2.0.20",
-    "@push.rocks/smartstream": "^3.0.7",
-    "@types/fs-extra": "^11.0.3",
+    "@push.rocks/smartrequest": "^2.0.21",
+    "@push.rocks/smartstream": "^3.0.30",
+    "@types/fs-extra": "^11.0.4",
     "@types/glob": "^8.1.0",
-    "@types/js-yaml": "^4.0.8",
+    "@types/js-yaml": "^4.0.9",
     "fs-extra": "^11.1.1",
     "glob": "^10.3.10",
     "js-yaml": "^4.1.0"
@@ -46,9 +46,9 @@
   "devDependencies": {
     "@git.zone/tsbuild": "^2.1.70",
     "@git.zone/tsrun": "^1.2.46",
-    "@git.zone/tstest": "^1.0.81",
+    "@git.zone/tstest": "^1.0.84",
     "@push.rocks/tapbundle": "^5.0.15",
-    "@types/node": "^20.8.10"
+    "@types/node": "^20.10.0"
   },
   "files": [
     "ts/**/*",

readme.md

@@ -1,49 +1,126 @@
-# @push.rocks/smartfile
-smart ways to work with files in nodejs
+# SmartFile
+> SmartFile offers smart ways to work with files in nodejs.
 
-## Availabililty and Links
-* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/smartfile)
-* [gitlab.com (source)](https://gitlab.com/push.rocks/smartfile)
-* [github.com (source mirror)](https://github.com/push.rocks/smartfile)
-* [docs (typedoc)](https://push.rocks.gitlab.io/smartfile/)
+## Install
+To install SmartFile, use npm or Yarn as follows:
 
-## Status for master
-
-Status Category | Status Badge
--- | --
-GitLab Pipelines | [![pipeline status](https://gitlab.com/push.rocks/smartfile/badges/master/pipeline.svg)](https://lossless.cloud)
-GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/push.rocks/smartfile/badges/master/coverage.svg)](https://lossless.cloud)
-npm | [![npm downloads per month](https://badgen.net/npm/dy/@push.rocks/smartfile)](https://lossless.cloud)
-Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/push.rocks/smartfile)](https://lossless.cloud)
-TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
-node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
-Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
-PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@push.rocks/smartfile)](https://lossless.cloud)
-PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@push.rocks/smartfile)](https://lossless.cloud)
-BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@push.rocks/smartfile)](https://lossless.cloud)
+```
+npm install @push.rocks/smartfile --save
+```
+Or:
+```
+yarn add @push.rocks/smartfile
+```
 
 ## Usage
 
-smartfile is an approach of being one tool to handle files in diverse environments.
+SmartFile is a comprehensive toolkit for file manipulation in Node.js. It provides functionalities for working with the filesystem, in-memory operations, streaming, and handling virtual directories. Below, you will find examples showcasing how to utilize these functionalities effectively.
 
-### Smartfile Sections
+### Basic File Operations
 
-smartfile thinks in sections:
+For reading and writing files, SmartFile provides synchronous and asynchronous methods. Here’s how you can use them:
 
-| section     | description                                                                  |
-| ----------- | ---------------------------------------------------------------------------- |
-| fs          | (object) gets data from fs to somewhere                                      |
-| memory      | gets data from memory to somewhere                                           |
-| remote      | gets data from remote locations to somewhere                                 |
-| interpreter | (object) handles yaml and json                                               |
-| smartfile   | (class) a virtual representation of a file, alternative to vinyl file format |
+#### Async Write to File
 
-## Contribution
+```typescript
+import { memory } from '@push.rocks/smartfile';
 
-We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
+const myData: string = 'Hello, SmartFile!';
 
-For further information read the linked docs at the top of this readme.
+// Writing string data to a file asynchronously
+memory.toFs(myData, './data/hello.txt');
+```
 
-## Legal
-> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
-| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
+#### Sync Write to File
+
+```typescript
+import { memory } from '@push.rocks/smartfile';
+
+const myData: string = 'Hello, World!';
+
+// Writing string data to a file synchronously
+memory.toFsSync(myData, './data/helloSync.txt');
+```
+
+### Working with Streams
+
+Streaming files to and from the filesystem is made easy with SmartFile. Here’s an example:
+
+#### Creating Read and Write Streams
+
+```typescript
+import { fsStream } from '@push.rocks/smartfile';
+import * as fs from 'fs';
+
+// Creating a read stream
+const readStream = fsStream.createReadStream('./data/readme.txt');
+
+// Creating a write stream
+const writeStream = fsStream.createWriteStream('./data/copy.txt');
+
+// Piping the read stream to the write stream
+readStream.pipe(writeStream);
+```
+
+### Dealing with Virtual Directories
+
+Virtual directories allow you to group and manipulate files as if they were in a filesystem structure without actually writing them to disk.
+
+```typescript
+import { VirtualDirectory } from '@push.rocks/smartfile';
+
+(async () => {
+  // Creating a virtual directory from the file system
+  const virtualDir = await VirtualDirectory.fromFsDirPath('./myDirectory');
+
+  // Adding files from another virtual directory
+  const anotherVirtualDir = await VirtualDirectory.fromFsDirPath('./anotherDirectory');
+  await virtualDir.addVirtualDirectory(anotherVirtualDir, 'merged');
+
+  // Saving the virtual directory to disk
+  await virtualDir.saveToDisk('./outputDirectory');
+})();
+```
+
+### Advanced File Manipulation
+
+SmartFile also allows for more advanced file manipulation techniques through the `SmartFile` class.
+
+```typescript
+import { SmartFile } from '@push.rocks/smartfile';
+
+(async () => {
+  // Create a SmartFile instance from a file path
+  const smartFile = await SmartFile.fromFilePath('./data/example.txt');
+
+  // Edit the file content
+  await smartFile.editContentAsString(async (currentContent: string) => {
+    return currentContent.toUpperCase();
+  });
+
+  // Write the changes back to disk
+  await smartFile.write();
+})();
+```
+
+### Conversion and Interpretation
+
+You can easily convert file contents to objects or interpret file types for further processing:
+
+```typescript
+import { memory } from '@push.rocks/smartfile';
+
+(async () => {
+  const fileString: string = await fs.promises.readFile('./data/example.json', 'utf8');
+  const fileObject = memory.toObject(fileString, 'json');
+
+  console.log(fileObject);
+  // Proceed with the object...
+})();
+```
+
+SmartFile simplifies handling files in a Node.js environment, providing a concise, promise-based API for various file operations, stream handling, and in-memory file manipulation. Whether you're dealing with physical files on the disk, manipulating file streams, or managing virtual files and directories, SmartFile has got you covered.
+
+## Information on Licensing
+
+SmartFile is licensed under the MIT License. This permissive license is short and to the point. It lets people do anything they want with your code as long as they provide attribution back to you and don’t hold you liable.

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartfile',
-  version: '11.0.0',
+  version: '11.0.5',
   description: 'offers smart ways to work with files in nodejs'
 }

ts/classes.smartfile.ts

@@ -91,6 +91,12 @@ export class SmartFile extends plugins.smartjson.Smartjson {
     });
   }
 
+  public static async fromUrl (urlArg: string) {
+    const response = await plugins.smartrequest.getBinary(urlArg);
+    const smartfile = await SmartFile.fromBuffer(urlArg, response.body);
+    return smartfile;
+  }
+
   // ========
   // INSTANCE
   // ========

ts/fs.ts

@@ -389,27 +389,58 @@ export const listFileTree = async (
 };
 
 /**
- * checks wether a file is ready for processing
+ * Watches for file stability before resolving the promise.
  */
-export const waitForFileToBeReady = async (filePathArg: string): Promise<void> => {
-  if (!plugins.path.isAbsolute(filePathArg)) {
-    filePathArg = plugins.path.resolve(filePathArg);
-  }
-  const limitedArray = new plugins.lik.LimitedArray<number>(3);
-  let fileReady = false;
-  while (!fileReady) {
-    const stats = await plugins.fsExtra.stat(filePathArg);
-    limitedArray.addOne(stats.size);
-    if (
-      limitedArray.array.length < 3 ||
-      !(
-        limitedArray.array[0] === limitedArray.array[1] &&
-        limitedArray.array[1] === limitedArray.array[2]
-      )
-    ) {
-      await plugins.smartdelay.delayFor(5000);
-    } else {
-      fileReady = true;
+export const waitForFileToBeReady = (filePathArg: string): Promise<void> => {
+  return new Promise(async (resolve, reject) => {
+    let lastFileSize = -1;
+    let fileIsStable = false;
+
+    const checkFileStability = async () => {
+      let currentFileSize: number;
+      const deferred = plugins.smartpromise.defer();
+      plugins.fs.stat(filePathArg, (err, stats) => {
+        if (err) {
+          fileIsStable = true;
+          watcher.close();
+          reject(err);
+          return;
+        }
+        currentFileSize = stats.size;
+        deferred.resolve();
+      });
+      await deferred.promise;
+      if (currentFileSize === lastFileSize) {
+        fileIsStable = true;
+        await plugins.smartdelay.delayFor(100);
+        resolve();
+      }
+      lastFileSize = currentFileSize;
+    };
+
+    const watcher = plugins.fs.watch(filePathArg, (eventType, filename) => {
+      if (eventType === 'change') {
+        checkFileStability();
+      }
+    });
+
+    watcher.on('error', (error) => {
+      watcher.close();
+      reject(error);
+    });
+
+    while (!fileIsStable) {
+      await checkFileStability();
+      if (!fileIsStable) {
+        await plugins.smartdelay.delayFor(5000);
+      }
     }
-  }
+    watcher.close();
+  });
 };
+
+
+
+
+
+

ts/fsstream.ts

@@ -37,3 +37,159 @@ export const processDirectory = async (
     }
   }
 };
+
+/**
+ * Checks if a file is ready to be streamed (exists and is not empty).
+ */
+export const isFileReadyForStreaming = async (filePathArg: string): Promise<boolean> => {
+  try {
+    const stats = await plugins.fs.promises.stat(filePathArg);
+    return stats.size > 0;
+  } catch (error) {
+    if (error.code === 'ENOENT') { // File does not exist
+      return false;
+    }
+    throw error; // Rethrow other unexpected errors
+  }
+};
+
+/**
+ * Waits for a file to be ready for streaming (exists and is not empty).
+ */
+export const waitForFileToBeReadyForStreaming = (filePathArg: string): Promise<void> => {
+  return new Promise((resolve, reject) => {
+    // Normalize and resolve the file path
+    const filePath = plugins.path.resolve(filePathArg);
+
+    // Function to check file stats
+    const checkFile = (resolve: () => void, reject: (reason: any) => void) => {
+      plugins.fs.stat(filePath, (err, stats) => {
+        if (err) {
+          if (err.code === 'ENOENT') {
+            // File not found, wait and try again
+            return;
+          }
+          // Some other error occurred
+          return reject(err);
+        }
+        if (stats.size > 0) {
+          // File exists and is not empty, resolve the promise
+          resolve();
+        }
+      });
+    };
+
+    // Set up file watcher
+    const watcher = plugins.fs.watch(filePath, { persistent: false }, (eventType) => {
+      if (eventType === 'change' || eventType === 'rename') {
+        checkFile(resolve, reject);
+      }
+    });
+
+    // Check file immediately in case it's already ready
+    checkFile(resolve, reject);
+
+    // Error handling
+    watcher.on('error', (error) => {
+      watcher.close();
+      reject(error);
+    });
+  });
+};
+
+class SmartReadStream extends plugins.stream.Readable {
+  private watcher: plugins.fs.FSWatcher | null = null;
+  private lastReadSize: number = 0;
+  private endTimeout: NodeJS.Timeout | null = null;
+  private filePath: string;
+  private endDelay: number;
+  private reading: boolean = false;
+
+  constructor(filePath: string, endDelay = 60000, opts?: plugins.stream.ReadableOptions) {
+    super(opts);
+    this.filePath = filePath;
+    this.endDelay = endDelay;
+  }
+
+  private startWatching(): void {
+    this.watcher = plugins.fs.watch(this.filePath, (eventType) => {
+      if (eventType === 'change') {
+        this.resetEndTimeout();
+      }
+    });
+
+    this.watcher.on('error', (error) => {
+      this.cleanup();
+      this.emit('error', error);
+    });
+  }
+
+  private resetEndTimeout(): void {
+    if (this.endTimeout) clearTimeout(this.endTimeout);
+    this.endTimeout = setTimeout(() => this.checkForEnd(), this.endDelay);
+  }
+
+  private checkForEnd(): void {
+    plugins.fs.stat(this.filePath, (err, stats) => {
+      if (err) {
+        this.emit('error', err);
+        return;
+      }
+
+      if (this.lastReadSize === stats.size) {
+        this.push(null); // Signal the end of the stream
+        this.cleanup();
+      } else {
+        this.lastReadSize = stats.size;
+        this.resetEndTimeout();
+        if (!this.reading) {
+          // We only want to continue reading if we were previously waiting for more data
+          this.reading = true;
+          this._read(10000); // Try to read more data
+        }
+      }
+    });
+  }
+
+  private cleanup(): void {
+    if (this.endTimeout) clearTimeout(this.endTimeout);
+    if (this.watcher) this.watcher.close();
+  }
+
+  _read(size: number): void {
+    this.reading = true;
+    const chunkSize = Math.min(size, 16384); // Read in chunks of 16KB
+    const buffer = Buffer.alloc(chunkSize);
+    plugins.fs.open(this.filePath, 'r', (err, fd) => {
+      if (err) {
+        this.emit('error', err);
+        return;
+      }
+      plugins.fs.read(fd, buffer, 0, chunkSize, this.lastReadSize, (err, bytesRead, buffer) => {
+        if (err) {
+          this.emit('error', err);
+          return;
+        }
+
+        if (bytesRead > 0) {
+          this.lastReadSize += bytesRead;
+          this.push(buffer.slice(0, bytesRead)); // Push the data onto the stream
+        } else {
+          this.reading = false; // No more data to read for now
+          this.resetEndTimeout();
+        }
+
+        plugins.fs.close(fd, (err) => {
+          if (err) {
+            this.emit('error', err);
+          }
+        });
+      });
+    });
+  }
+
+  _destroy(error: Error | null, callback: (error: Error | null) => void): void {
+    this.cleanup();
+    callback(error);
+  }
+}