smartarchive/ts/classes.smartarchive.ts

import * as plugins from './plugins.js';
import * as paths from './paths.js';

import { Bzip2Tools } from './classes.bzip2tools.js';
import { GzipTools } from './classes.gziptools.js';
import { TarTools } from './classes.tartools.js';
import { ZipTools } from './classes.ziptools.js';

import {
  ArchiveAnalyzer,
  type IAnalyzedResult,
} from './classes.archiveanalyzer.js';

import type { from } from '@push.rocks/smartrx/dist_ts/smartrx.plugins.rxjs.js';

export class SmartArchive {
  // STATIC
  public static async fromArchiveUrl(urlArg: string): Promise<SmartArchive> {
    const smartArchiveInstance = new SmartArchive();
    smartArchiveInstance.sourceUrl = urlArg;
    return smartArchiveInstance;
  }

  public static async fromArchiveFile(
    filePathArg: string,
  ): Promise<SmartArchive> {
    const smartArchiveInstance = new SmartArchive();
    smartArchiveInstance.sourceFilePath = filePathArg;
    return smartArchiveInstance;
  }

  public static async fromArchiveStream(
    streamArg:
      | plugins.stream.Readable
      | plugins.stream.Duplex
      | plugins.stream.Transform,
  ): Promise<SmartArchive> {
    const smartArchiveInstance = new SmartArchive();
    smartArchiveInstance.sourceStream = streamArg;
    return smartArchiveInstance;
  }

  // INSTANCE
  public tarTools = new TarTools();
  public zipTools = new ZipTools();
  public gzipTools = new GzipTools();
  public bzip2Tools = new Bzip2Tools(this);
  public archiveAnalyzer = new ArchiveAnalyzer(this);

  public sourceUrl: string;
  public sourceFilePath: string;
  public sourceStream:
    | plugins.stream.Readable
    | plugins.stream.Duplex
    | plugins.stream.Transform;

  public archiveName: string;
  public singleFileMode: boolean = false;

  public addedDirectories: string[] = [];
  public addedFiles: (
    | plugins.smartfile.SmartFile
    | plugins.smartfile.StreamFile
  )[] = [];
  public addedUrls: string[] = [];

  constructor() {}

  /**
   * gets the original archive stream
   */
  public async getArchiveStream() {
    if (this.sourceStream) {
      return this.sourceStream;
    }
    if (this.sourceUrl) {
      const response = await plugins.smartrequest.SmartRequest.create()
        .url(this.sourceUrl)
        .get();
      const webStream = response.stream();
      // @ts-ignore - Web stream to Node.js stream conversion
      const urlStream = plugins.stream.Readable.fromWeb(webStream);
      return urlStream;
    }
    if (this.sourceFilePath) {
      const fileStream = plugins.smartfile.fs.toReadStream(this.sourceFilePath);
      return fileStream;
    }
  }

  public async exportToTarGzStream() {
    const tarPackStream = await this.tarTools.getPackStream();
    const gzipStream = await this.gzipTools.getCompressionStream();
    // const archiveStream = tarPackStream.pipe(gzipStream);
    // return archiveStream;
  }

  public async exportToFs(
    targetDir: string,
    fileNameArg?: string,
  ): Promise<void> {
    const done = plugins.smartpromise.defer<void>();
    const streamFileStream = await this.exportToStreamOfStreamFiles();
    streamFileStream.pipe(
      new plugins.smartstream.SmartDuplex({
        objectMode: true,
        writeFunction: async (
          streamFileArg: plugins.smartfile.StreamFile,
          streamtools,
        ) => {
          const done = plugins.smartpromise.defer<void>();
          console.log(
            streamFileArg.relativeFilePath
              ? streamFileArg.relativeFilePath
              : 'no relative path',
          );
          const streamFile = streamFileArg;
          const readStream = await streamFile.createReadStream();
          await plugins.smartfile.fs.ensureDir(targetDir);
          const writePath = plugins.path.join(
            targetDir,
            streamFile.relativeFilePath || fileNameArg,
          );
          await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
          const writeStream =
            plugins.smartfile.fsStream.createWriteStream(writePath);
          readStream.pipe(writeStream);
          writeStream.on('finish', () => {
            done.resolve();
          });
          await done.promise;
        },
        finalFunction: async () => {
          done.resolve();
        },
      }),
    );
    return done.promise;
  }

  public async exportToStreamOfStreamFiles() {
    const streamFileIntake =
      new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
        objectMode: true,
      });
    const archiveStream = await this.getArchiveStream();
    const createAnalyzedStream = () => this.archiveAnalyzer.getAnalyzedStream();

    // lets create a function that can be called multiple times to unpack layers of archives
    const createUnpackStream = () =>
      plugins.smartstream.createTransformFunction<IAnalyzedResult, any>(
        async (analyzedResultChunk) => {
          if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
            const tarStream =
              analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
            tarStream.on('entry', async (header, stream, next) => {
              if (header.type === 'directory') {
                console.log(
                  `tar stream directory: ${header.name} ... skipping!`,
                );
                stream.resume(); // Consume directory stream
                stream.on('end', () => next());
                return;
              }
              console.log(`tar stream file: ${header.name}`);
              
              // Create a PassThrough stream to buffer the data
              const passThrough = new plugins.stream.PassThrough();
              const streamfile = plugins.smartfile.StreamFile.fromStream(
                passThrough,
                header.name,
              );
              
              // Push the streamfile immediately
              streamFileIntake.push(streamfile);
              
              // Pipe the tar entry stream to the passthrough
              stream.pipe(passThrough);
              
              // Move to next entry when this one ends
              stream.on('end', () => {
                passThrough.end();
                next();
              });
            });
            tarStream.on('finish', function () {
              console.log('tar extraction finished');
              // Only signal end if this is the final stream
              streamFileIntake.signalEnd();
            });
            analyzedResultChunk.resultStream.pipe(
              analyzedResultChunk.decompressionStream,
            );
          } else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
            analyzedResultChunk.resultStream
              .pipe(analyzedResultChunk.decompressionStream)
              .pipe(
                new plugins.smartstream.SmartDuplex({
                  objectMode: true,
                  writeFunction: async (
                    streamFileArg: plugins.smartfile.StreamFile,
                    streamtools,
                  ) => {
                    streamFileIntake.push(streamFileArg);
                  },
                  finalFunction: async () => {
                    streamFileIntake.signalEnd();
                  },
                }),
              );
          } else if (
            analyzedResultChunk.isArchive &&
            analyzedResultChunk.decompressionStream
          ) {
            // For nested archives (like gzip containing tar)
            const nestedStream = analyzedResultChunk.resultStream
              .pipe(analyzedResultChunk.decompressionStream)
              .pipe(createAnalyzedStream())
              .pipe(createUnpackStream());
            
            // Don't signal end here - let the nested unpacker handle it
          } else {
            const streamFile = plugins.smartfile.StreamFile.fromStream(
              analyzedResultChunk.resultStream,
              analyzedResultChunk.fileType?.ext,
            );
            streamFileIntake.push(streamFile);
            streamFileIntake.signalEnd();
          }
        },
        {
          objectMode: true,
        },
      );

    archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream());
    return streamFileIntake;
  }
}