import * as plugins from './plugins.js'; import * as paths from './paths.js'; import { Bzip2Tools } from './classes.bzip2tools.js'; import { GzipTools } from './classes.gziptools.js'; import { TarTools } from './classes.tartools.js'; import { ZipTools } from './classes.ziptools.js'; import { ArchiveAnalyzer, type IAnalyzedResult, } from './classes.archiveanalyzer.js'; import type { from } from '@push.rocks/smartrx/dist_ts/smartrx.plugins.rxjs.js'; export class SmartArchive { // STATIC public static async fromArchiveUrl(urlArg: string): Promise { const smartArchiveInstance = new SmartArchive(); smartArchiveInstance.sourceUrl = urlArg; return smartArchiveInstance; } public static async fromArchiveFile( filePathArg: string, ): Promise { const smartArchiveInstance = new SmartArchive(); smartArchiveInstance.sourceFilePath = filePathArg; return smartArchiveInstance; } public static async fromArchiveStream( streamArg: | plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform, ): Promise { const smartArchiveInstance = new SmartArchive(); smartArchiveInstance.sourceStream = streamArg; return smartArchiveInstance; } // INSTANCE public tarTools = new TarTools(); public zipTools = new ZipTools(); public gzipTools = new GzipTools(); public bzip2Tools = new Bzip2Tools(this); public archiveAnalyzer = new ArchiveAnalyzer(this); public sourceUrl: string; public sourceFilePath: string; public sourceStream: | plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform; public archiveName: string; public singleFileMode: boolean = false; public addedDirectories: string[] = []; public addedFiles: ( | plugins.smartfile.SmartFile | plugins.smartfile.StreamFile )[] = []; public addedUrls: string[] = []; constructor() {} /** * gets the original archive stream */ public async getArchiveStream() { if (this.sourceStream) { return this.sourceStream; } if (this.sourceUrl) { const response = await plugins.smartrequest.SmartRequest.create() .url(this.sourceUrl) .get(); const webStream = response.stream(); // @ts-ignore - Web stream to Node.js stream conversion const urlStream = plugins.stream.Readable.fromWeb(webStream); return urlStream; } if (this.sourceFilePath) { const fileStream = plugins.smartfile.fs.toReadStream(this.sourceFilePath); return fileStream; } } public async exportToTarGzStream() { const tarPackStream = await this.tarTools.getPackStream(); const gzipStream = await this.gzipTools.getCompressionStream(); // const archiveStream = tarPackStream.pipe(gzipStream); // return archiveStream; } public async exportToFs( targetDir: string, fileNameArg?: string, ): Promise { const done = plugins.smartpromise.defer(); const streamFileStream = await this.exportToStreamOfStreamFiles(); streamFileStream.pipe( new plugins.smartstream.SmartDuplex({ objectMode: true, writeFunction: async ( streamFileArg: plugins.smartfile.StreamFile, streamtools, ) => { const done = plugins.smartpromise.defer(); console.log( streamFileArg.relativeFilePath ? streamFileArg.relativeFilePath : 'no relative path', ); const streamFile = streamFileArg; const readStream = await streamFile.createReadStream(); await plugins.smartfile.fs.ensureDir(targetDir); const writePath = plugins.path.join( targetDir, streamFile.relativeFilePath || fileNameArg, ); await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath)); const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath); readStream.pipe(writeStream); writeStream.on('finish', () => { done.resolve(); }); await done.promise; }, finalFunction: async () => { done.resolve(); }, }), ); return done.promise; } public async exportToStreamOfStreamFiles() { const streamFileIntake = new plugins.smartstream.StreamIntake({ objectMode: true, }); const archiveStream = await this.getArchiveStream(); const createAnalyzedStream = () => this.archiveAnalyzer.getAnalyzedStream(); // lets create a function that can be called multiple times to unpack layers of archives const createUnpackStream = () => plugins.smartstream.createTransformFunction( async (analyzedResultChunk) => { if (analyzedResultChunk.fileType?.mime === 'application/x-tar') { const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract; tarStream.on('entry', async (header, stream, next) => { if (header.type === 'directory') { console.log( `tar stream directory: ${header.name} ... skipping!`, ); next(); return; } console.log(`tar stream file: ${header.name}`); const streamfile = plugins.smartfile.StreamFile.fromStream( stream, header.name, ); streamFileIntake.push(streamfile); stream.on('end', function () { next(); // ready for next entry }); }); tarStream.on('finish', function () { console.log('finished'); streamFileIntake.signalEnd(); }); analyzedResultChunk.resultStream.pipe( analyzedResultChunk.decompressionStream, ); } else if (analyzedResultChunk.fileType?.mime === 'application/zip') { analyzedResultChunk.resultStream .pipe(analyzedResultChunk.decompressionStream) .pipe( new plugins.smartstream.SmartDuplex({ objectMode: true, writeFunction: async ( streamFileArg: plugins.smartfile.StreamFile, streamtools, ) => { streamFileIntake.push(streamFileArg); }, finalFunction: async () => { streamFileIntake.signalEnd(); }, }), ); } else if ( analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream ) { analyzedResultChunk.resultStream .pipe(analyzedResultChunk.decompressionStream) .pipe(createAnalyzedStream()) .pipe(createUnpackStream()); } else { const streamFile = plugins.smartfile.StreamFile.fromStream( analyzedResultChunk.resultStream, analyzedResultChunk.fileType?.ext, ); streamFileIntake.push(streamFile); streamFileIntake.signalEnd(); } }, { objectMode: true, }, ); archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream()); return streamFileIntake; } }