push.rocks/smartarchive
3
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

feat(classes.smartarchive): Support URL streams, recursive archive unpacking and filesystem export; improve ZIP/GZIP/BZIP2 robustness; CI and package metadata updates

Browse Source
This commit is contained in:
Jürgen Kunz
2025-08-18 01:29:06 +00:00
parent ed5f590b5f
commit 780db4921e
23 changed files with 812 additions and 622 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
ts/classes.smartarchive.ts
View File

@@ -6,7 +6,10 @@ import { GzipTools } from './classes.gziptools.js';
import { TarTools } from './classes.tartools.js';
import { ZipTools } from './classes.ziptools.js';
import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js';
import {
ArchiveAnalyzer,
type IAnalyzedResult,
} from './classes.archiveanalyzer.js';
import type { from } from '@push.rocks/smartrx/dist_ts/smartrx.plugins.rxjs.js';
@@ -18,14 +21,19 @@ export class SmartArchive {
return smartArchiveInstance;
}
public static async fromArchiveFile(filePathArg: string): Promise<SmartArchive> {
public static async fromArchiveFile(
filePathArg: string,
): Promise<SmartArchive> {
const smartArchiveInstance = new SmartArchive();
smartArchiveInstance.sourceFilePath = filePathArg;
return smartArchiveInstance;
}
public static async fromArchiveStream(
streamArg: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform
streamArg:
| plugins.stream.Readable
| plugins.stream.Duplex
| plugins.stream.Transform,
): Promise<SmartArchive> {
const smartArchiveInstance = new SmartArchive();
smartArchiveInstance.sourceStream = streamArg;
@@ -41,13 +49,19 @@ export class SmartArchive {
public sourceUrl: string;
public sourceFilePath: string;
public sourceStream: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform;
public sourceStream:
| plugins.stream.Readable
| plugins.stream.Duplex
| plugins.stream.Transform;
public archiveName: string;
public singleFileMode: boolean = false;
public addedDirectories: string[] = [];
public addedFiles: (plugins.smartfile.SmartFile | plugins.smartfile.StreamFile)[] = [];
public addedFiles: (
| plugins.smartfile.SmartFile
| plugins.smartfile.StreamFile
)[] = [];
public addedUrls: string[] = [];
constructor() {}
@@ -81,24 +95,35 @@ export class SmartArchive {
// return archiveStream;
}
public async exportToFs(targetDir: string, fileNameArg?: string): Promise<void> {
public async exportToFs(
targetDir: string,
fileNameArg?: string,
): Promise<void> {
const done = plugins.smartpromise.defer<void>();
const streamFileStream = await this.exportToStreamOfStreamFiles();
streamFileStream.pipe(
new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
writeFunction: async (
streamFileArg: plugins.smartfile.StreamFile,
streamtools,
) => {
const done = plugins.smartpromise.defer<void>();
console.log(streamFileArg.relativeFilePath ? streamFileArg.relativeFilePath : 'no relative path');
console.log(
streamFileArg.relativeFilePath
? streamFileArg.relativeFilePath
: 'no relative path',
);
const streamFile = streamFileArg;
const readStream = await streamFile.createReadStream();
await plugins.smartfile.fs.ensureDir(targetDir);
const writePath = plugins.path.join(
targetDir,
streamFile.relativeFilePath || fileNameArg
streamFile.relativeFilePath || fileNameArg,
);
await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
const writeStream =
plugins.smartfile.fsStream.createWriteStream(writePath);
readStream.pipe(writeStream);
writeStream.on('finish', () => {
done.resolve();
@@ -108,15 +133,16 @@ export class SmartArchive {
finalFunction: async () => {
done.resolve();
},
})
}),
);
return done.promise;
}
public async exportToStreamOfStreamFiles() {
const streamFileIntake = new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
objectMode: true,
});
const streamFileIntake =
new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
objectMode: true,
});
const archiveStream = await this.getArchiveStream();
const createAnalyzedStream = () => this.archiveAnalyzer.getAnalyzedStream();
@@ -125,15 +151,21 @@ export class SmartArchive {
plugins.smartstream.createTransformFunction<IAnalyzedResult, any>(
async (analyzedResultChunk) => {
if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
const tarStream =
analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
tarStream.on('entry', async (header, stream, next) => {
if (header.type === 'directory') {
console.log(`tar stream directory: ${header.name} ... skipping!`);
console.log(
`tar stream directory: ${header.name} ... skipping!`,
);
next();
return;
}
console.log(`tar stream file: ${header.name}`);
const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
const streamfile = plugins.smartfile.StreamFile.fromStream(
stream,
header.name,
);
streamFileIntake.push(streamfile);
stream.on('end', function () {
next(); // ready for next entry
@@ -143,20 +175,30 @@ export class SmartArchive {
console.log('finished');
streamFileIntake.signalEnd();
});
analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream);
analyzedResultChunk.resultStream.pipe(
analyzedResultChunk.decompressionStream,
);
} else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
analyzedResultChunk.resultStream
.pipe(analyzedResultChunk.decompressionStream)
.pipe(new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
streamFileIntake.push(streamFileArg);
},
finalFunction: async () => {
streamFileIntake.signalEnd();
}
}));
} else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) {
.pipe(
new plugins.smartstream.SmartDuplex({
objectMode: true,
writeFunction: async (
streamFileArg: plugins.smartfile.StreamFile,
streamtools,
) => {
streamFileIntake.push(streamFileArg);
},
finalFunction: async () => {
streamFileIntake.signalEnd();
},
}),
);
} else if (
analyzedResultChunk.isArchive &&
analyzedResultChunk.decompressionStream
) {
analyzedResultChunk.resultStream
.pipe(analyzedResultChunk.decompressionStream)
.pipe(createAnalyzedStream())
@@ -164,7 +206,7 @@ export class SmartArchive {
} else {
const streamFile = plugins.smartfile.StreamFile.fromStream(
analyzedResultChunk.resultStream,
analyzedResultChunk.fileType?.ext
analyzedResultChunk.fileType?.ext,
);
streamFileIntake.push(streamFile);
streamFileIntake.signalEnd();
@@ -172,7 +214,7 @@ export class SmartArchive {
},
{
objectMode: true,
}
},
);
archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream());