2023-11-06 17:14:21 +00:00
|
|
|
import * as plugins from './plugins.js';
|
|
|
|
import * as paths from './paths.js';
|
|
|
|
|
2024-03-16 23:29:42 +00:00
|
|
|
import { Bzip2Tools } from './classes.bzip2tools.js';
|
2023-11-06 17:14:21 +00:00
|
|
|
import { GzipTools } from './classes.gziptools.js';
|
|
|
|
import { TarTools } from './classes.tartools.js';
|
2024-03-16 23:29:42 +00:00
|
|
|
import { ZipTools } from './classes.ziptools.js';
|
2023-11-06 17:14:21 +00:00
|
|
|
|
|
|
|
import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js';
|
|
|
|
|
|
|
|
import type { from } from '@push.rocks/smartrx/dist_ts/smartrx.plugins.rxjs.js';
|
|
|
|
|
|
|
|
export class SmartArchive {
|
|
|
|
// STATIC
|
|
|
|
public static async fromArchiveUrl(urlArg: string): Promise<SmartArchive> {
|
|
|
|
const smartArchiveInstance = new SmartArchive();
|
|
|
|
smartArchiveInstance.sourceUrl = urlArg;
|
|
|
|
return smartArchiveInstance;
|
|
|
|
}
|
|
|
|
|
|
|
|
public static async fromArchiveFile(filePathArg: string): Promise<SmartArchive> {
|
|
|
|
const smartArchiveInstance = new SmartArchive();
|
|
|
|
smartArchiveInstance.sourceFilePath = filePathArg;
|
|
|
|
return smartArchiveInstance;
|
|
|
|
}
|
|
|
|
|
|
|
|
public static async fromArchiveStream(
|
|
|
|
streamArg: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform
|
|
|
|
): Promise<SmartArchive> {
|
|
|
|
const smartArchiveInstance = new SmartArchive();
|
|
|
|
smartArchiveInstance.sourceStream = streamArg;
|
|
|
|
return smartArchiveInstance;
|
|
|
|
}
|
|
|
|
|
|
|
|
// INSTANCE
|
2024-06-08 10:44:40 +00:00
|
|
|
public tarTools = new TarTools();
|
2024-06-08 10:46:43 +00:00
|
|
|
public zipTools = new ZipTools();
|
|
|
|
public gzipTools = new GzipTools();
|
|
|
|
public bzip2Tools = new Bzip2Tools(this);
|
2023-11-06 17:14:21 +00:00
|
|
|
public archiveAnalyzer = new ArchiveAnalyzer(this);
|
|
|
|
|
|
|
|
public sourceUrl: string;
|
|
|
|
public sourceFilePath: string;
|
|
|
|
public sourceStream: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform;
|
|
|
|
|
|
|
|
public archiveName: string;
|
|
|
|
public singleFileMode: boolean = false;
|
|
|
|
|
|
|
|
public addedDirectories: string[] = [];
|
|
|
|
public addedFiles: (plugins.smartfile.SmartFile | plugins.smartfile.StreamFile)[] = [];
|
|
|
|
public addedUrls: string[] = [];
|
|
|
|
|
|
|
|
constructor() {}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* gets the original archive stream
|
|
|
|
*/
|
|
|
|
public async getArchiveStream() {
|
|
|
|
if (this.sourceStream) {
|
|
|
|
return this.sourceStream;
|
|
|
|
}
|
|
|
|
if (this.sourceUrl) {
|
|
|
|
const urlStream = await plugins.smartrequest.getStream(this.sourceUrl);
|
|
|
|
return urlStream;
|
|
|
|
}
|
|
|
|
if (this.sourceFilePath) {
|
|
|
|
const fileStream = plugins.smartfile.fs.toReadStream(this.sourceFilePath);
|
|
|
|
return fileStream;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
public async exportToTarGzStream() {
|
|
|
|
const tarPackStream = await this.tarTools.getPackStream();
|
|
|
|
const gzipStream = await this.gzipTools.getCompressionStream();
|
|
|
|
// const archiveStream = tarPackStream.pipe(gzipStream);
|
|
|
|
// return archiveStream;
|
|
|
|
}
|
|
|
|
|
2023-11-07 03:19:54 +00:00
|
|
|
public async exportToFs(targetDir: string, fileNameArg?: string): Promise<void> {
|
2023-11-06 18:38:36 +00:00
|
|
|
const done = plugins.smartpromise.defer<void>();
|
|
|
|
const streamFileStream = await this.exportToStreamOfStreamFiles();
|
2024-03-16 23:29:42 +00:00
|
|
|
streamFileStream.pipe(
|
|
|
|
new plugins.smartstream.SmartDuplex({
|
|
|
|
objectMode: true,
|
2024-03-16 23:35:17 +00:00
|
|
|
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
|
2024-03-16 23:29:42 +00:00
|
|
|
const done = plugins.smartpromise.defer<void>();
|
2024-03-16 23:35:17 +00:00
|
|
|
console.log(streamFileArg.relativeFilePath ? streamFileArg.relativeFilePath : 'no relative path');
|
|
|
|
const streamFile = streamFileArg;
|
2024-03-16 23:29:42 +00:00
|
|
|
const readStream = await streamFile.createReadStream();
|
|
|
|
await plugins.smartfile.fs.ensureDir(targetDir);
|
|
|
|
const writePath = plugins.path.join(
|
|
|
|
targetDir,
|
|
|
|
streamFile.relativeFilePath || fileNameArg
|
|
|
|
);
|
|
|
|
await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
|
|
|
|
const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
|
|
|
|
readStream.pipe(writeStream);
|
|
|
|
writeStream.on('finish', () => {
|
|
|
|
done.resolve();
|
|
|
|
});
|
|
|
|
await done.promise;
|
|
|
|
},
|
|
|
|
finalFunction: async () => {
|
2023-11-11 17:28:50 +00:00
|
|
|
done.resolve();
|
2024-03-16 23:29:42 +00:00
|
|
|
},
|
|
|
|
})
|
|
|
|
);
|
2023-11-06 18:38:36 +00:00
|
|
|
return done.promise;
|
|
|
|
}
|
2023-11-06 17:14:21 +00:00
|
|
|
|
|
|
|
public async exportToStreamOfStreamFiles() {
|
|
|
|
const streamFileIntake = new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
|
|
|
|
objectMode: true,
|
|
|
|
});
|
|
|
|
const archiveStream = await this.getArchiveStream();
|
|
|
|
const createAnalyzedStream = () => this.archiveAnalyzer.getAnalyzedStream();
|
|
|
|
|
|
|
|
// lets create a function that can be called multiple times to unpack layers of archives
|
|
|
|
const createUnpackStream = () =>
|
|
|
|
plugins.smartstream.createTransformFunction<IAnalyzedResult, any>(
|
|
|
|
async (analyzedResultChunk) => {
|
2023-11-07 03:19:54 +00:00
|
|
|
if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
|
2023-11-06 18:46:02 +00:00
|
|
|
const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
|
2024-03-16 23:29:42 +00:00
|
|
|
tarStream.on('entry', async (header, stream, next) => {
|
2024-06-06 18:59:04 +00:00
|
|
|
if (header.type === 'directory') {
|
|
|
|
console.log(`tar stream directory: ${header.name} ... skipping!`);
|
|
|
|
next();
|
|
|
|
return;
|
|
|
|
}
|
2024-06-06 18:48:02 +00:00
|
|
|
console.log(`tar stream file: ${header.name}`);
|
2024-03-16 23:29:42 +00:00
|
|
|
const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
|
|
|
|
streamFileIntake.push(streamfile);
|
|
|
|
stream.on('end', function () {
|
|
|
|
next(); // ready for next entry
|
|
|
|
});
|
|
|
|
});
|
2023-11-06 18:46:02 +00:00
|
|
|
tarStream.on('finish', function () {
|
2023-11-07 03:19:54 +00:00
|
|
|
console.log('finished');
|
2023-11-06 18:46:02 +00:00
|
|
|
streamFileIntake.signalEnd();
|
2023-11-07 03:19:54 +00:00
|
|
|
});
|
2023-11-06 17:14:21 +00:00
|
|
|
analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream);
|
2024-03-16 23:29:42 +00:00
|
|
|
} else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
|
|
|
|
analyzedResultChunk.resultStream
|
|
|
|
.pipe(analyzedResultChunk.decompressionStream)
|
|
|
|
.pipe(new plugins.smartstream.SmartDuplex({
|
|
|
|
objectMode: true,
|
|
|
|
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
|
|
|
|
streamFileIntake.push(streamFileArg);
|
|
|
|
},
|
2024-03-16 23:42:19 +00:00
|
|
|
finalFunction: async () => {
|
|
|
|
streamFileIntake.signalEnd();
|
|
|
|
}
|
2024-03-16 23:29:42 +00:00
|
|
|
}));
|
2023-11-06 17:14:21 +00:00
|
|
|
} else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) {
|
|
|
|
analyzedResultChunk.resultStream
|
|
|
|
.pipe(analyzedResultChunk.decompressionStream)
|
|
|
|
.pipe(createAnalyzedStream())
|
|
|
|
.pipe(createUnpackStream());
|
|
|
|
} else {
|
|
|
|
const streamFile = plugins.smartfile.StreamFile.fromStream(
|
|
|
|
analyzedResultChunk.resultStream,
|
2023-11-07 03:19:54 +00:00
|
|
|
analyzedResultChunk.fileType?.ext
|
2023-11-06 17:14:21 +00:00
|
|
|
);
|
|
|
|
streamFileIntake.push(streamFile);
|
2023-11-07 08:56:57 +00:00
|
|
|
streamFileIntake.signalEnd();
|
2023-11-06 17:14:21 +00:00
|
|
|
}
|
2024-03-16 23:29:42 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
objectMode: true,
|
2023-11-11 18:55:19 +00:00
|
|
|
}
|
2023-11-06 17:14:21 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream());
|
|
|
|
return streamFileIntake;
|
|
|
|
}
|
|
|
|
}
|