Compare commits

..

15 Commits

Author SHA1 Message Date
f1daec6f44 11.0.21 2024-06-23 23:55:55 +02:00
6698a583e9 fix(dependencies): Update dependencies to latest versions 2024-06-23 23:55:54 +02:00
bf52f01365 11.0.20 2024-06-07 17:13:08 +02:00
14adec5ba3 fix(core): update 2024-06-07 17:13:07 +02:00
fb2880c995 11.0.19 2024-06-07 12:24:54 +02:00
0e0ae7e42f fix(core): update 2024-06-07 12:24:53 +02:00
1391dbddbc 11.0.18 2024-06-06 23:33:36 +02:00
477736da82 fix(core): update 2024-06-06 23:33:35 +02:00
26a67d9662 11.0.17 2024-06-06 22:29:06 +02:00
14771fab27 fix(core): update 2024-06-06 22:29:06 +02:00
b0ae383622 update description 2024-05-29 14:13:08 +02:00
921f1cbb66 11.0.16 2024-05-28 12:43:47 +02:00
a9b373f5f8 fix(core): update 2024-05-28 12:43:46 +02:00
f0d48cc763 11.0.15 2024-05-17 17:49:57 +02:00
255cc844ad fix(core): update 2024-05-17 17:49:57 +02:00
10 changed files with 5313 additions and 2865 deletions

276
changelog.md Normal file
View File

@ -0,0 +1,276 @@
# Changelog
## 2024-06-23 - 11.0.21 - fix(dependencies)
Update dependencies to latest versions
- Updated @push.rocks/smartpromise to ^4.0.4
- Updated @push.rocks/smartstream to ^3.0.44
- Updated glob to ^10.4.2
- Updated @types/node to ^20.14.8
## 2024-06-07 - 11.0.20 - Changelog
11.0.20
## 2024-06-07 - 11.0.19 - fix(core): update
11.0.19
- fix(core): update
## 2024-06-07 - 11.0.18 - fix(core): update
11.0.18
- fix(core): update
## 2024-06-06 - 11.0.17 - fix(core): update
11.0.17
- fix(core): update
## 2024-06-06 - 11.0.16 - fix(core): update
11.0.16
- fix(core): update
## 2024-05-29 - 11.0.16 - update description
11.0.16
- update description
## 2024-05-17 - 11.0.15 - fix(core): update
11.0.15
- fix(core): update
## 2024-04-14 - 11.0.14 - update tsconfig
11.0.14
- update tsconfig
## 2024-04-12 - 11.0.13 - fix(core): update
11.0.13
- fix(core): update
## 2024-04-12 - 11.0.12 - fix(core): update
11.0.12
- fix(core): update
## 2024-04-12 - 11.0.11 - fix(core): update
11.0.11
- fix(core): update
## 2024-04-03 - 11.0.10 - fix(core): update
11.0.10
- fix(core): update
## 2024-04-03 - 11.0.9 - fix(core): update
11.0.9
- fix(core): update
## 2024-04-02 - 11.0.8 - fix(core): update
11.0.8
- fix(core): update
## 2024-04-02 - 11.0.7 - fix(core): update
11.0.7
- fix(core): update
## 2024-04-02 - 11.0.6 - fix(core): update
11.0.6
- fix(core): update
## 2024-04-01 - 11.0.5 - update npmextra.json
11.0.5
- update npmextra.json: githost
## 2024-04-01 - 11.0.4 - fix(core): update
11.0.4
- fix(core): update
## 2023-11-24 - 11.0.3 - fix(core): update
11.0.3
- fix(core): update
## 2023-11-07 - 11.0.2 - fix(core): update
11.0.2
- fix(core): update
## 2023-11-07 - 11.0.1 - fix(core): update
11.0.1
- fix(core): update
## 2023-11-06 - 11.0.0 - fix(core): update
11.0.0
- fix(core): update
## 2023-11-06 - 10.0.40 - BREAKING CHANGE(core): update
10.0.40
- BREAKING CHANGE(core): update
## 2023-11-04 - 10.0.39 - fix(core): update
10.0.39
- fix(core): update
## 2023-11-04 - 10.0.38 - fix(core): update
10.0.38
- fix(core): update
## 2023-11-04 - 10.0.37 - fix(core): update
10.0.37
- fix(core): update
## 2023-11-03 - 10.0.36 - fix(core): update
10.0.36
- fix(core): update
## 2023-11-03 - 10.0.35 - fix(core): update
10.0.35
- fix(core): update
## 2023-11-03 - 10.0.34 - fix(core): update
10.0.34
- fix(core): update
## 2023-11-03 - 10.0.33 - fix(core): update
10.0.33
- fix(core): update
## 2023-10-12 - 10.0.32 - fix(core): update
10.0.32
- fix(core): update
## 2023-09-22 - 10.0.31 - fix(core): update
10.0.31
- fix(core): update
## 2023-08-31 - 10.0.30 - fix(core): update
10.0.30
- fix(core): update
## 2023-08-23 - 10.0.29 - fix(core): update
10.0.29
- fix(core): update
## 2023-07-12 - 10.0.28 - fix(core): update
10.0.28
- fix(core): update
## 2023-07-10 - 10.0.27 - fix(core): update
10.0.27
- fix(core): update
## 2023-07-10 - 10.0.26 - fix(core): update
10.0.26
- fix(core): update
## 2023-07-08 - 10.0.25 - fix(core): update
10.0.25
- fix(core): update
## 2023-06-25 - 10.0.24 to 10.0.14 - Series of Fixes
10.0.24 to 10.0.14
- Series of fixes in the core module
## 2023-01-09 - 10.0.13 to 10.0.6 - Series of Fixes
10.0.13 to 10.0.6
- Series of fixes in the core module
## 2022-09-05 - 10.0.5 to 10.0.3 - Series of Fixes
10.0.5 to 10.0.3
- Series of fixes in the core module
## 2022-06-07 - 10.0.2 to 10.0.1 - Series of Fixes
10.0.2 to 10.0.1
- Series of fixes in the core module
## 2022-06-07 - 9.0.7 - BREAKING CHANGE(core): switch to esm
9.0.7
- BREAKING CHANGE(core): switch to esm
## 2022-03-11 - 9.0.6 to 9.0.2 - Series of Fixes
9.0.6 to 9.0.2
- Series of fixes in the core module
## 2021-12-01 - 9.0.1 - fix(core): update
9.0.1
- fix(core): update
## 2021-12-01 - 9.0.0 - fix(absolute pathing)
9.0.0
- add functions for easily getting absolute paths
## 2021-11-30 - 8.0.11 - BREAKING CHANGE(relative pathing)
8.0.11
- improved relative pathing
## 2020-08-10 - 8.0.10 to 7.0.12 - Series of Fixes and Updates
8.0.10 to 7.0.12
- Series of fixes in the core module
- BREAKING CHANGE(Smartfile class): switch to a Buffer-only approach
## 2019-02-17 - 7.0.0 - fix(core): update dependencies
7.0.0
- fix(core): update dependencies
## 2019-01-27 - 6.0.12 - BREAKING CHANGE(smartfile.fs.fileExists)
6.0.12
- now returns a Promise<boolean>
## 2018-08-19 - 6.0.11 to 6.0.6 - Series of Fixes
6.0.11 to 6.0.6
- Series of fixes in core and dependencies
## 2018-07-03 - 6.0.5 to 5.0.0 - Series of Fixes
6.0.5 to 5.0.0
- Series of fixes in core and dependencies
## 2018-02-16 - 4.2.28 - BREAKING CHANGE(scope)
4.2.28
- switch to pushrocks scope

View File

@ -10,11 +10,11 @@
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartfile",
"description": "provides a robust suite of tools for managing files in Node.js using TypeScript.",
"description": "Provides comprehensive tools for efficient file management in Node.js using TypeScript, including handling streams, virtual directories, and various file operations.",
"npmPackagename": "@push.rocks/smartfile",
"license": "MIT",
"keywords": [
"files management",
"file management",
"TypeScript",
"Node.js",
"file operations",
@ -23,7 +23,15 @@
"virtual directory",
"filesystem utilities",
"memory-efficient file handling",
"custom file operations"
"custom file operations",
"write files",
"read files",
"copy files",
"delete files",
"list directories",
"handle large files",
"virtual filesystems",
"buffer operations"
]
}
},

View File

@ -1,8 +1,8 @@
{
"name": "@push.rocks/smartfile",
"private": false,
"version": "11.0.14",
"description": "provides a robust suite of tools for managing files in Node.js using TypeScript.",
"version": "11.0.21",
"description": "Provides comprehensive tools for efficient file management in Node.js using TypeScript, including handling streams, virtual directories, and various file operations.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
@ -13,10 +13,10 @@
},
"repository": {
"type": "git",
"url": "git+https://gitlab.com/push.rocks/smartfile.git"
"url": "https://code.foss.global/push.rocks/smartfile.git"
},
"keywords": [
"files management",
"file management",
"TypeScript",
"Node.js",
"file operations",
@ -25,38 +25,46 @@
"virtual directory",
"filesystem utilities",
"memory-efficient file handling",
"custom file operations"
"custom file operations",
"write files",
"read files",
"copy files",
"delete files",
"list directories",
"handle large files",
"virtual filesystems",
"buffer operations"
],
"author": "Lossless GmbH <hello@lossless.com> (https://lossless.com)",
"license": "MIT",
"bugs": {
"url": "https://gitlab.com/push.rocks/smartfile/issues"
},
"homepage": "https://gitlab.com/push.rocks/smartfile#readme",
"homepage": "https://code.foss.global/push.rocks/smartfile",
"dependencies": {
"@push.rocks/lik": "^6.0.14",
"@push.rocks/lik": "^6.0.15",
"@push.rocks/smartdelay": "^3.0.5",
"@push.rocks/smartfile-interfaces": "^1.0.7",
"@push.rocks/smarthash": "^3.0.4",
"@push.rocks/smartjson": "^5.0.16",
"@push.rocks/smartmime": "^1.0.5",
"@push.rocks/smartpath": "^5.0.13",
"@push.rocks/smartpromise": "^4.0.2",
"@push.rocks/smartrequest": "^2.0.21",
"@push.rocks/smartstream": "^3.0.34",
"@push.rocks/smartjson": "^5.0.20",
"@push.rocks/smartmime": "^2.0.2",
"@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpromise": "^4.0.4",
"@push.rocks/smartrequest": "^2.0.22",
"@push.rocks/smartstream": "^3.0.44",
"@types/fs-extra": "^11.0.4",
"@types/glob": "^8.1.0",
"@types/js-yaml": "^4.0.9",
"fs-extra": "^11.2.0",
"glob": "^10.3.12",
"glob": "^10.4.2",
"js-yaml": "^4.1.0"
},
"devDependencies": {
"@git.zone/tsbuild": "^2.1.72",
"@git.zone/tsbuild": "^2.1.80",
"@git.zone/tsrun": "^1.2.46",
"@git.zone/tstest": "^1.0.88",
"@push.rocks/tapbundle": "^5.0.22",
"@types/node": "^20.12.7"
"@git.zone/tstest": "^1.0.90",
"@push.rocks/tapbundle": "^5.0.23",
"@types/node": "^20.14.8"
},
"files": [
"ts/**/*",
@ -73,4 +81,4 @@
"browserslist": [
"last 1 chrome versions"
]
}
}

7644
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

116
readme.md
View File

@ -19,7 +19,7 @@ npm install @push.rocks/smartfile
First, ensure you're working in an environment that supports ECMAScript modules (ESM) and TypeScript. Heres how youd generally import and use `@push.rocks/smartfile`:
```typescript
import { SmartFile, StreamFile, VirtualDirectory, fs, memory } from '@push.rocks/smartfile';
import { SmartFile, StreamFile, VirtualDirectory, fs, memory, interpreter } from '@push.rocks/smartfile';
```
### Working with `SmartFile`
@ -53,14 +53,13 @@ const largeFile: StreamFile = await StreamFile.fromPath('./largeInput/largeFile.
await largeFile.writeToDisk('./largeOutput/largeFileCopy.mp4');
```
### Managing Virtual Directories
### Working with Virtual Directories
Handling multiple files as if they were part of a file system:
```typescript
const virtualDir = new VirtualDirectory();
await virtualDir.addSmartfiles([smartFile1, smartFile2]);
await virtualDir.saveToDisk('./virtualDirOutput');
const virtualDir = await VirtualDirectory.fromFsDirPath('./data/inputDir');
await virtualDir.saveToDisk('./data/outputDir');
```
### File System Operations
@ -94,7 +93,7 @@ For specialized file operations, such as editing the contents of a file or strea
```typescript
const smartFileToEdit: SmartFile = await SmartFile.fromFilePath('./editableFile.txt');
await smartFileToEdit.editContentAsString((content) => content.replace(/originalText/g, 'newText'));
await smartFileToEdit.editContentAsString(async (content) => content.replace(/originalText/g, 'newText'));
await smartFileToEdit.write();
```
@ -117,13 +116,110 @@ const bufferSmartFile: SmartFile = await SmartFile.fromBuffer('./bufferFile.txt'
await bufferSmartFile.write();
```
#### Uploading a File
### Using `VirtualDirectory` for Complex File Management
Stream files or buffers can be integrated with web frameworks to handle file uploads efficiently, utilizing streams to reduce memory footprint.
`VirtualDirectory` simplifies the management of multiple files that are otherwise scattered across different directories or created at runtime.
### Conclusion
#### Creating a `VirtualDirectory`
With `@push.rocks/smartfile`, managing files in Node.js using TypeScript becomes significantly more straightforward and efficient. Utilizing the provided classes and methods, you can handle a wide range of file operations, from basic read/write tasks to complex operations such as virtual directory management and file streaming, with minimal boilerplate code and maximum efficiency.
To create a `VirtualDirectory` from an existing file directory:
```typescript
const virtualDirectory = await VirtualDirectory.fromFsDirPath('./someDirectory');
```
#### Adding More Files
You can add more `SmartFile` instances to your `VirtualDirectory`:
```typescript
const additionalFiles = [
await SmartFile.fromFilePath('./anotherDirectory/file1.txt'),
await SmartFile.fromFilePath('./anotherDirectory/file2.txt')
];
virtualDirectory.addSmartfiles(additionalFiles);
```
#### Saving `VirtualDirectory` to Disk
Save your entire `VirtualDirectory` to disk:
```typescript
await virtualDirectory.saveToDisk('./outputDirectory');
```
### Utilizing StreamFile for Efficient File Handling
Using `StreamFile` can be especially beneficial for large files or when performing streaming operations.
#### Streaming from a URL
`StreamFile` provides capabilities to stream files directly from URLs, making it easier to work with remote content.
```typescript
const urlStreamFile: StreamFile = await StreamFile.fromUrl('https://example.com/largefile.zip');
await urlStreamFile.writeToDisk('./downloadedFiles/largefile.zip');
```
### Combining Buffer and Stream Approaches
Create `StreamFile` from a buffer for efficient, multi-use streams.
```typescript
const buffer = Buffer.from('Streaming buffer content');
const bufferStreamFile = StreamFile.fromBuffer(buffer, 'bufferBasedStream.txt');
await bufferStreamFile.writeToDisk('./streams/bufferBasedStream.txt');
```
### Read and Write Operations with StreamFile
Perform read and write operations efficiently using `StreamFile`.
```typescript
const fileStream = await StreamFile.fromPath('./inputData/largeFile.data');
await fileStream.writeToDisk('./outputData/largeFileCopy.data');
```
Check for completeness of your read and write operations, ensuring the integrity of file content.
```typescript
const readBuffer = await fileStream.getContentAsBuffer();
console.log(readBuffer.toString());
```
### Ensuring File Readiness for Streaming
Ensure a file is ready for streaming or create a custom readable stream incorporating data transformation.
```typescript
const smartReadStream = new SmartReadStream('./incomingData/sample.data');
smartReadStream.on('data', (chunk) => {
console.log('New Data Chunk:', chunk.toString());
});
```
### File Transformation with SmartReadStream
Perform transformations on the stream of data while reading:
```typescript
smartReadStream.on('data', (chunk) => {
// Perform some transformation
const transformedChunk = chunk.toString().toUpperCase();
console.log('Transformed Data Chunk:', transformedChunk);
});
```
### Streaming with SmartReadStream
Stream data from a `SmartReadStream` to a file efficiently managing large datasets.
```typescript
const transformedWriteStream = fs.createWriteStream('./processedData/transformed.data');
smartReadStream.pipe(transformedWriteStream);
```
`@push.rocks/smartfile` significantly simplifies the handling of complex file operations in Node.js projects, making these tasks straightforward while maintaining efficiency and ease of use. Explore and leverage these features to enhance your project's file management capabilities.
## License and Legal Information

View File

@ -1,8 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: '@push.rocks/smartfile',
version: '11.0.14',
description: 'provides a robust suite of tools for managing files in Node.js using TypeScript.'
version: '11.0.21',
description: 'Provides comprehensive tools for efficient file management in Node.js using TypeScript, including handling streams, virtual directories, and various file operations.'
}

View File

@ -306,4 +306,11 @@ export class SmartFile extends plugins.smartjson.Smartjson {
stream.push(null); // Push null to signify the end of the stream (EOF)
return stream;
}
/**
* Returns the size of the file in bytes
*/
public async getSize(): Promise<number> {
return this.contentBuffer.length;
}
}

View File

@ -10,26 +10,16 @@ type TStreamSource = (streamFile: StreamFile) => Promise<Readable>;
* It allows creating streams from a file path, a URL, or a buffer.
*/
export class StreamFile {
// INSTANCE
relativeFilePath?: string;
private streamSource: TStreamSource;
// enable stream based multi use
private cachedStreamBuffer?: Buffer;
public multiUse: boolean;
public used: boolean = false;
private constructor(streamSource: TStreamSource, relativeFilePath?: string) {
this.streamSource = streamSource;
this.relativeFilePath = relativeFilePath;
}
// STATIC
public static async fromPath(filePath: string): Promise<StreamFile> {
const streamSource: TStreamSource = async (stremFileArg) => smartfileFsStream.createReadStream(filePath);
const streamSource: TStreamSource = async (streamFileArg) => smartfileFsStream.createReadStream(filePath);
const streamFile = new StreamFile(streamSource, filePath);
streamFile.multiUse = true;
streamFile.byteLengthComputeFunction = async () => {
const stats = await smartfileFs.stat(filePath);
return stats.size;
}
return streamFile;
}
@ -37,6 +27,10 @@ export class StreamFile {
const streamSource: TStreamSource = async (streamFileArg) => plugins.smartrequest.getStream(url); // Replace with actual plugin method
const streamFile = new StreamFile(streamSource);
streamFile.multiUse = true;
streamFile.byteLengthComputeFunction = async () => {
const response = await plugins.smartrequest.getBinary(url); // TODO: switch to future .getBinaryByteLength()
return response.body.length;
}
return streamFile;
}
@ -49,6 +43,7 @@ export class StreamFile {
};
const streamFile = new StreamFile(streamSource, relativeFilePath);
streamFile.multiUse = true;
streamFile.byteLengthComputeFunction = async () => buffer.length;
return streamFile;
}
@ -91,6 +86,22 @@ export class StreamFile {
return streamFile;
}
// INSTANCE
relativeFilePath?: string;
private streamSource: TStreamSource;
// enable stream based multi use
private cachedStreamBuffer?: Buffer;
public multiUse: boolean;
public used: boolean = false;
public byteLengthComputeFunction: () => Promise<number>;
private constructor(streamSource: TStreamSource, relativeFilePath?: string) {
this.streamSource = streamSource;
this.relativeFilePath = relativeFilePath;
}
// METHODS
private checkMultiUse() {
@ -149,4 +160,15 @@ export class StreamFile {
const contentBuffer = await this.getContentAsBuffer();
return contentBuffer.toString(formatArg);
}
}
/**
* Returns the size of the file content in bytes.
*/
public async getSize(): Promise<number> {
if (this.byteLengthComputeFunction) {
return this.byteLengthComputeFunction();
} else {
return null;
}
}
}

View File

@ -190,7 +190,8 @@ export const removeManySync = (filePathArrayArg: string[]): void => {
===============================================================*/
/**
*
* reads a file content to an object
* good for JSON, YAML, TOML, etc.
* @param filePathArg
* @param fileTypeArg
* @returns {any}
@ -206,7 +207,7 @@ export const toObjectSync = (filePathArg, fileTypeArg?) => {
* reads a file content to a String
*/
export const toStringSync = (filePath: string): string => {
const encoding = plugins.smartmime.getEncoding(filePath);
const encoding = plugins.smartmime.getEncodingForPathSync(filePath);
let fileString: string | Buffer = plugins.fsExtra.readFileSync(filePath, encoding);
if (Buffer.isBuffer(fileString)) {
fileString = fileString.toString('binary');
@ -485,7 +486,6 @@ export let toFs = async (
return await done.promise;
};
export const stat = async (filePathArg: string) => {
return plugins.fsPromises.stat(filePathArg);
}

View File

@ -1,9 +1,10 @@
// node native scope
import * as fs from 'fs';
import * as fsPromises from 'fs/promises';
import * as path from 'path';
import * as stream from 'stream';
export { fs, path, stream };
export { fs, fsPromises, path, stream };
// @pushrocks scope
import * as lik from '@push.rocks/lik';