feat(imagestore): now processing images with extraction, retagging, repackaging and long term storage

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/docker',
-  version: '1.1.4',
+  version: '1.2.0',
   description: 'Provides easy communication with Docker remote API from Node.js, with TypeScript support.'
 }

ts/classes.container.ts

@@ -2,7 +2,7 @@ import * as plugins from './plugins.js';
 import * as interfaces from './interfaces/index.js';
 
 import { DockerHost } from './classes.host.js';
-import { logger } from './logging.js';
+import { logger } from './logger.js';
 
 export class DockerContainer {
   // STATIC

ts/classes.host.ts

@@ -1,10 +1,12 @@
 import * as plugins from './plugins.js';
+import * as paths from './paths.js';
 import { DockerContainer } from './classes.container.js';
 import { DockerNetwork } from './classes.network.js';
 import { DockerService } from './classes.service.js';
-import { logger } from './logging.js';
+import { logger } from './logger.js';
 import path from 'path';
-import type { DockerImageStore } from './classes.imagestore.js';
+import { DockerImageStore } from './classes.imagestore.js';
+import { DockerImage } from './classes.image.js';
 
 export interface IAuthData {
   serveraddress: string;
@@ -18,6 +20,8 @@ export interface IDockerHostConstructorOptions {
 }
 
 export class DockerHost {
+  public options: IDockerHostConstructorOptions;
+
   /**
    * the path where the docker sock can be found
    */
@@ -30,6 +34,12 @@ export class DockerHost {
    * @param pathArg
    */
   constructor(optionsArg: IDockerHostConstructorOptions) {
+    this.options = {
+      ...{
+        imageStoreDir: plugins.path.join(paths.nogitDir, 'temp-docker-image-store'),
+      },
+      ...optionsArg,
+    }
     let pathToUse: string;
     if (optionsArg.dockerSockPath) {
       pathToUse = optionsArg.dockerSockPath;
@@ -48,6 +58,17 @@ export class DockerHost {
     }
     console.log(`using docker sock at ${pathToUse}`);
     this.socketPath = pathToUse;
+    this.imageStore = new DockerImageStore(this, {
+      bucketDir: null,
+      localDirPath: this.options.imageStoreDir,
+    })
+  }
+
+  public async start() {
+    await this.imageStore.start();
+  }
+  public async stop() {
+    await this.imageStore.stop();
   }
 
   /**
@@ -81,6 +102,9 @@ export class DockerHost {
     });
   }
 
+  // ==============
+  // NETWORKS
+  // ==============
   /**
    * gets all networks
    */
@@ -89,9 +113,23 @@ export class DockerHost {
   }
 
   /**
-   *
+   * create a network
    */
+  public async createNetwork(optionsArg: Parameters<typeof DockerNetwork.createNetwork>[1]) {
+    return await DockerNetwork.createNetwork(this, optionsArg);
+  }
 
+  /**
+   * get a network by name
+   */
+  public async getNetworkByName(networkNameArg: string) {
+    return await DockerNetwork.getNetworkByName(this, networkNameArg);
+  }
+
+
+  // ==============
+  // CONTAINERS
+  // ==============
   /**
    * gets all containers
    */
@@ -100,6 +138,10 @@ export class DockerHost {
     return containerArray;
   }
 
+  // ==============
+  // SERVICES
+  // ==============
+
   /**
    * gets all services
    */
@@ -108,6 +150,24 @@ export class DockerHost {
     return serviceArray;
   }
 
+  // ==============
+  // IMAGES
+  // ==============
+
+  /**
+   * get all images
+   */
+  public async getImages() {
+    return await DockerImage.getImages(this);
+  }
+
+  /**
+   * get an image by name
+   */
+  public async getImageByName(imageNameArg: string) {
+    return await DockerImage.getImageByName(this, imageNameArg);
+  }
+
   /**
    *
    */

ts/classes.image.ts

@@ -1,8 +1,11 @@
 import * as plugins from './plugins.js';
 import * as interfaces from './interfaces/index.js';
 import { DockerHost } from './classes.host.js';
-import { logger } from './logging.js';
+import { logger } from './logger.js';
 
+/**
+ * represents a docker image on the remote docker host
+ */
 export class DockerImage {
   // STATIC
   public static async getImages(dockerHost: DockerHost) {
@@ -14,7 +17,7 @@ export class DockerImage {
     return images;
   }
 
-  public static async findImageByName(dockerHost: DockerHost, imageNameArg: string) {
+  public static async getImageByName(dockerHost: DockerHost, imageNameArg: string) {
     const images = await this.getImages(dockerHost);
     const result = images.find((image) => {
       if (image.RepoTags) {
@@ -67,7 +70,7 @@ export class DockerImage {
     );
     if (response.statusCode < 300) {
       logger.log('info', `Successfully pulled image ${imageUrlObject.imageUrl} from the registry`);
-      const image = await DockerImage.findImageByName(dockerHostArg, imageUrlObject.imageOriginTag);
+      const image = await DockerImage.getImageByName(dockerHostArg, imageUrlObject.imageOriginTag);
       return image;
     } else {
       logger.log('error', `Failed at the attempt of creating a new image`);

ts/classes.imagestore.ts

@@ -1,5 +1,6 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
+import { logger } from './logger.js';
 import type { DockerHost } from './classes.host.js';
 
 export interface IDockerImageStoreConstructorOptions {
@@ -22,19 +23,78 @@ export class DockerImageStore {
 
   // Method to store tar stream
   public async storeImage(imageName: string, tarStream: plugins.smartstream.stream.Readable): Promise<void> {
-    const imagePath = plugins.path.join(this.options.localDirPath, `${imageName}.tar`);
+    logger.log('info', `Storing image ${imageName}...`);
+    const uniqueProcessingId = plugins.smartunique.shortId();
 
+    const initialTarDownloadPath = plugins.path.join(this.options.localDirPath, `${uniqueProcessingId}.tar`);
+    const extractionDir = plugins.path.join(this.options.localDirPath, uniqueProcessingId);
     // Create a write stream to store the tar file
-    const writeStream = plugins.smartfile.fsStream.createWriteStream(imagePath);
+    const writeStream = plugins.smartfile.fsStream.createWriteStream(initialTarDownloadPath);
 
-    return new Promise((resolve, reject) => {
+    // lets wait for the write stream to finish
+    await new Promise((resolve, reject) => {
       tarStream.pipe(writeStream);
-
       writeStream.on('finish', resolve);
       writeStream.on('error', reject);
     });
+    logger.log('info', `Image ${imageName} stored locally for processing. Extracting...`);
+
+    // lets process the image
+    const tarArchive = await plugins.smartarchive.SmartArchive.fromArchiveFile(initialTarDownloadPath);
+    await tarArchive.exportToFs(extractionDir);
+    logger.log('info', `Image ${imageName} extracted.`);
+    await plugins.smartfile.fs.remove(initialTarDownloadPath);
+    logger.log('info', `deleted original tar to save space.`);
+    logger.log('info', `now repackaging for s3...`);
+    const smartfileIndexJson = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(extractionDir, 'index.json'));
+    const smartfileManifestJson = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(extractionDir, 'manifest.json'));
+    const smartfileOciLayoutJson = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(extractionDir, 'oci-layout'));
+    const smartfileRepositoriesJson = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(extractionDir, 'repositories'));
+    const indexJson = JSON.parse(smartfileIndexJson.contents.toString());
+    const manifestJson = JSON.parse(smartfileManifestJson.contents.toString());
+    const ociLayoutJson = JSON.parse(smartfileOciLayoutJson.contents.toString());
+    const repositoriesJson = JSON.parse(smartfileRepositoriesJson.contents.toString());
+
+    indexJson.manifests[0].annotations['io.containerd.image.name'] = imageName;
+    manifestJson[0].RepoTags[0] = imageName;
+    const repoFirstKey = Object.keys(repositoriesJson)[0];
+    const repoFirstValue = repositoriesJson[repoFirstKey];
+    repositoriesJson[imageName] = repoFirstValue;
+    delete repositoriesJson[repoFirstKey];
+
+    smartfileIndexJson.contents = Buffer.from(JSON.stringify(indexJson, null, 2));
+    smartfileManifestJson.contents = Buffer.from(JSON.stringify(manifestJson, null, 2));
+    smartfileOciLayoutJson.contents = Buffer.from(JSON.stringify(ociLayoutJson, null, 2));
+    smartfileRepositoriesJson.contents = Buffer.from(JSON.stringify(repositoriesJson, null, 2));
+    await Promise.all([
+      smartfileIndexJson.write(),
+      smartfileManifestJson.write(),
+      smartfileOciLayoutJson.write(),
+      smartfileRepositoriesJson.write(),
+    ]);
+
+    logger.log('info', 'repackaging archive for s3...');
+    const tartools = new plugins.smartarchive.TarTools();
+    const newTarPack = await tartools.packDirectory(extractionDir);
+    const finalTarName = `${uniqueProcessingId}.processed.tar`;
+    const finalTarPath = plugins.path.join(this.options.localDirPath, finalTarName);
+    const finalWriteStream = plugins.smartfile.fsStream.createWriteStream(finalTarPath);
+    await new Promise((resolve, reject) => {
+      newTarPack.finalize();
+      newTarPack.pipe(finalWriteStream);
+      finalWriteStream.on('finish', resolve);
+      finalWriteStream.on('error', reject);
+    });
+    logger.log('ok', `Repackaged image ${imageName} for s3.`);
+    await plugins.smartfile.fs.remove(extractionDir);
   }
 
+  public async start() {
+    await plugins.smartfile.fs.ensureEmptyDir(this.options.localDirPath);
+  }
+
+  public async stop() {}
+
   // Method to retrieve tar stream
   public async getImage(imageName: string): Promise<plugins.smartstream.stream.Readable> {
     const imagePath = plugins.path.join(this.options.localDirPath, `${imageName}.tar`);

ts/classes.network.ts

@@ -3,7 +3,7 @@ import * as interfaces from './interfaces/index.js';
 
 import { DockerHost } from './classes.host.js';
 import { DockerService } from './classes.service.js';
-import { logger } from './logging.js';
+import { logger } from './logger.js';
 
 export class DockerNetwork {
   public static async getNetworks(dockerHost: DockerHost): Promise<DockerNetwork[]> {

ts/classes.service.ts

@@ -4,7 +4,7 @@ import * as interfaces from './interfaces/index.js';
 import { DockerHost } from './classes.host.js';
 import { DockerImage } from './classes.image.js';
 import { DockerSecret } from './classes.secret.js';
-import { logger } from './logging.js';
+import { logger } from './logger.js';
 
 export class DockerService {
   // STATIC

ts/logger.ts

@@ -0,0 +1,5 @@
+import * as plugins from './plugins.js';
+import { commitinfo } from './00_commitinfo_data.js';
+
+export const logger = plugins.smartlog.Smartlog.createForCommitinfo(commitinfo);
+logger.enableConsole();

ts/logging.ts

@@ -1,3 +0,0 @@
-import * as plugins from './plugins.js';
-
-export const logger = new plugins.smartlog.ConsoleLog();

ts/plugins.ts

@@ -5,6 +5,7 @@ export { path };
 
 // @pushrocks scope
 import * as lik from '@push.rocks/lik';
+import * as smartarchive from '@push.rocks/smartarchive';
 import * as smartbucket from '@push.rocks/smartbucket';
 import * as smartfile from '@push.rocks/smartfile';
 import * as smartjson from '@push.rocks/smartjson';
@@ -15,10 +16,12 @@ import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrequest from '@push.rocks/smartrequest';
 import * as smartstring from '@push.rocks/smartstring';
 import * as smartstream from '@push.rocks/smartstream';
+import * as smartunique from '@push.rocks/smartunique';
 import * as smartversion from '@push.rocks/smartversion';
 
 export {
   lik,
+  smartarchive,
   smartbucket,
   smartfile,
   smartjson,
@@ -29,6 +32,7 @@ export {
   smartrequest,
   smartstring,
   smartstream,
+  smartunique,
   smartversion,
 };