feat(images): can now import and export images, start work on local 100% JS oci imageregistry

package.json

@@ -34,24 +34,26 @@
   "homepage": "https://gitlab.com/mojoio/docker#readme",
   "dependencies": {
     "@push.rocks/lik": "^6.0.15",
-    "@push.rocks/smartfile": "^11.0.14",
-    "@push.rocks/smartjson": "^5.0.19",
-    "@push.rocks/smartlog": "^3.0.1",
+    "@push.rocks/smartarchive": "^4.0.22",
+    "@push.rocks/smartfile": "^11.0.16",
+    "@push.rocks/smartjson": "^5.0.20",
+    "@push.rocks/smartlog": "^3.0.6",
     "@push.rocks/smartnetwork": "^3.0.0",
     "@push.rocks/smartpath": "^5.0.18",
     "@push.rocks/smartpromise": "^4.0.3",
     "@push.rocks/smartrequest": "^2.0.22",
+    "@push.rocks/smartstream": "^3.0.44",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/smartversion": "^3.0.5",
     "@tsclass/tsclass": "^4.0.54",
     "rxjs": "^7.5.7"
   },
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.25",
+    "@git.zone/tsbuild": "^2.1.80",
     "@git.zone/tsrun": "^1.2.12",
     "@git.zone/tstest": "^1.0.90",
     "@push.rocks/tapbundle": "^5.0.23",
-    "@types/node": "20.10.0"
+    "@types/node": "20.14.1"
   },
   "files": [
     "ts/**/*",

test/test.nonci.node.ts

@@ -1,4 +1,8 @@
 import { expect, tap } from '@push.rocks/tapbundle';
+
+import * as plugins from '../ts/plugins.js';
+import * as paths from '../ts/paths.js';
+
 import * as docker from '../ts/index.js';
 
 let testDockerHost: docker.DockerHost;
@@ -40,8 +44,10 @@ tap.test('should remove a network', async () => {
 // Images
 tap.test('should pull an image from imagetag', async () => {
   const image = await docker.DockerImage.createFromRegistry(testDockerHost, {
-    imageUrl: 'hosttoday/ht-docker-node',
-    imageTag: 'alpine',
+    creationObject: {
+      imageUrl: 'hosttoday/ht-docker-node',
+      imageTag: 'alpine',
+    },
   });
   expect(image).toBeInstanceOf(docker.DockerImage);
   console.log(image);
@@ -93,7 +99,9 @@ tap.test('should create a service', async () => {
     contentArg: '{"hi": "wow"}',
   });
   const testImage = await docker.DockerImage.createFromRegistry(testDockerHost, {
-    imageUrl: 'registry.gitlab.com/hosttoday/ht-docker-static',
+    creationObject: {
+      imageUrl: 'code.foss.global/host.today/ht-docker-node:latest',
+    }
   });
   const testService = await docker.DockerService.createService(testDockerHost, {
     image: testImage,
@@ -110,4 +118,34 @@ tap.test('should create a service', async () => {
   await testSecret.remove();
 });
 
-tap.start();
+tap.skip.test('should export images', async (toolsArg) => {
+  const done = toolsArg.defer();
+  const testImage = await docker.DockerImage.createFromRegistry(testDockerHost, {
+    creationObject: {
+      imageUrl: 'code.foss.global/host.today/ht-docker-node:latest',
+    }
+  });
+  const fsWriteStream = plugins.smartfile.fsStream.createWriteStream(
+    plugins.path.join(paths.nogitDir, 'testimage.tar')
+  );
+  const exportStream = await testImage.exportToTarStream();
+  exportStream.pipe(fsWriteStream).on('finish', () => {
+    done.resolve();
+  });
+  await done.promise;
+});
+
+tap.test('should import images', async (toolsArg) => {
+  const done = toolsArg.defer();
+  const fsReadStream = plugins.smartfile.fsStream.createReadStream(
+    plugins.path.join(paths.nogitDir, 'testimage.tar')
+  );
+  await docker.DockerImage.createFromTarStream(testDockerHost, {
+    tarStream: fsReadStream,
+    creationObject: {
+      imageUrl: 'code.foss.global/host.today/ht-docker-node:latest',
+    }
+  })
+})
+
+export default tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/docker',
-  version: '1.0.112',
+  version: '1.1.0',
   description: 'Provides easy communication with Docker remote API from Node.js, with TypeScript support.'
 }

ts/classes.container.ts

@@ -1,8 +1,8 @@
-import * as plugins from './docker.plugins.js';
+import * as plugins from './plugins.js';
 import * as interfaces from './interfaces/index.js';
 
-import { DockerHost } from './docker.classes.host.js';
-import { logger } from './docker.logging.js';
+import { DockerHost } from './classes.host.js';
+import { logger } from './logging.js';
 
 export class DockerContainer {
   // STATIC

ts/classes.host.ts

@@ -1,8 +1,8 @@
-import * as plugins from './docker.plugins.js';
-import { DockerContainer } from './docker.classes.container.js';
-import { DockerNetwork } from './docker.classes.network.js';
-import { DockerService } from './docker.classes.service.js';
-import { logger } from './docker.logging.js';
+import * as plugins from './plugins.js';
+import { DockerContainer } from './classes.container.js';
+import { DockerNetwork } from './classes.network.js';
+import { DockerService } from './classes.service.js';
+import { logger } from './logging.js';
 import path from 'path';
 
 export interface IAuthData {
@@ -70,7 +70,7 @@ export class DockerHost {
     await this.auth({
       username: gitlabAuthArray[0],
       password: gitlabAuthArray[1],
-      serveraddress: 'registry.gitlab.com',
+      serveraddress: registryUrlArg,
     });
   }
 
@@ -174,7 +174,7 @@ export class DockerHost {
     return response;
   }
 
-  public async requestStreaming(methodArg: string, routeArg: string, dataArg = {}) {
+  public async requestStreaming(methodArg: string, routeArg: string, readStream?: plugins.smartstream.stream.Readable) {
     const requestUrl = `${this.socketPath}${routeArg}`;
     const response = await plugins.smartrequest.request(
       requestUrl,
@@ -188,7 +188,20 @@ export class DockerHost {
         requestBody: null,
         keepAlive: false,
       },
-      true
+      true,
+      (readStream ? reqArg => {
+        let counter = 0;
+        const smartduplex = new plugins.smartstream.SmartDuplex({
+          writeFunction: async (chunkArg) => {
+            if (counter % 1000 === 0) {
+              console.log(`posting chunk ${counter}`);
+            }
+            counter++;
+            return chunkArg;
+          }
+        });
+        readStream.pipe(smartduplex).pipe(reqArg);
+      } : null),
     );
     console.log(response.statusCode);
     console.log(response.body);

ts/classes.image.ts

@@ -1,7 +1,7 @@
-import * as plugins from './docker.plugins.js';
+import * as plugins from './plugins.js';
 import * as interfaces from './interfaces/index.js';
-import { DockerHost } from './docker.classes.host.js';
-import { logger } from './docker.logging.js';
+import { DockerHost } from './classes.host.js';
+import { logger } from './logging.js';
 
 export class DockerImage {
   // STATIC
@@ -28,7 +28,9 @@ export class DockerImage {
 
   public static async createFromRegistry(
     dockerHostArg: DockerHost,
-    creationObject: interfaces.IImageCreationDescriptor
+    optionsArg: {
+      creationObject: interfaces.IImageCreationDescriptor
+    }
   ): Promise<DockerImage> {
     // lets create a sanatized imageUrlObject
     const imageUrlObject: {
@@ -36,8 +38,8 @@ export class DockerImage {
       imageTag: string;
       imageOriginTag: string;
     } = {
-      imageUrl: creationObject.imageUrl,
-      imageTag: creationObject.imageTag,
+      imageUrl: optionsArg.creationObject.imageUrl,
+      imageTag: optionsArg.creationObject.imageTag,
       imageOriginTag: null,
     };
     if (imageUrlObject.imageUrl.includes(':')) {
@@ -72,6 +74,19 @@ export class DockerImage {
     }
   }
 
+  /**
+   * 
+   * @param dockerHostArg
+   * @param tarStreamArg
+   */
+  public static async createFromTarStream(dockerHostArg: DockerHost, optionsArg: {
+    creationObject: interfaces.IImageCreationDescriptor,
+    tarStream: plugins.smartstream.stream.Readable,
+  }) {
+    const response = await dockerHostArg.requestStreaming('POST', '/images/load', optionsArg.tarStream);
+    return response;
+  }
+
   public static async tagImageByIdOrName(
     dockerHost: DockerHost,
     idOrNameArg: string,
@@ -126,7 +141,9 @@ export class DockerImage {
    */
   public async pullLatestImageFromRegistry(): Promise<boolean> {
     const updatedImage = await DockerImage.createFromRegistry(this.dockerHost, {
-      imageUrl: this.RepoTags[0],
+      creationObject: {
+        imageUrl: this.RepoTags[0],
+      },
     });
     Object.assign(this, updatedImage);
     // TODO: Compare image digists before and after
@@ -141,4 +158,33 @@ export class DockerImage {
       return '0.0.0';
     }
   }
+
+  /**
+   * exports an image to a tar ball
+   */
+  public async exportToTarStream(): Promise<plugins.smartstream.stream.Readable> {
+    console.log(`Exporting image ${this.RepoTags[0]} to tar stream.`);
+    const response = await this.dockerHost.requestStreaming('GET', `/images/${encodeURIComponent(this.RepoTags[0])}/get`);
+    let counter = 0;
+    const webduplexStream = new plugins.smartstream.SmartDuplex({
+      writeFunction: async (chunk, tools) => {
+        if (counter % 1000 === 0)
+          console.log(`Got chunk: ${counter}`);
+        counter++;
+        return chunk;
+      }
+    });
+    response.on('data', (chunk) => {
+      if (!webduplexStream.write(chunk)) {
+        response.pause();
+        webduplexStream.once('drain', () => {
+          response.resume();
+        })
+      };
+    });
+    response.on('end', () => {
+      webduplexStream.end();
+    })
+    return webduplexStream;
+  }
 }

ts/classes.imagestore.ts

@@ -0,0 +1,40 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+
+export interface IDockerImageStoreConstructorOptions {
+  dirPath: string;
+}
+
+export class DockerImageStore {
+  public options: IDockerImageStoreConstructorOptions;
+
+  constructor(optionsArg: IDockerImageStoreConstructorOptions) {
+    this.options = optionsArg;
+  }
+
+  // Method to store tar stream
+  public async storeImage(imageName: string, tarStream: NodeJS.ReadableStream): Promise<void> {
+    const imagePath = plugins.path.join(this.options.dirPath, `${imageName}.tar`);
+
+    // Create a write stream to store the tar file
+    const writeStream = plugins.smartfile.fsStream.createWriteStream(imagePath);
+
+    return new Promise((resolve, reject) => {
+      tarStream.pipe(writeStream);
+
+      writeStream.on('finish', resolve);
+      writeStream.on('error', reject);
+    });
+  }
+
+  // Method to retrieve tar stream
+  public async getImage(imageName: string): Promise<plugins.smartstream.stream.Readable> {
+    const imagePath = plugins.path.join(this.options.dirPath, `${imageName}.tar`);
+
+    if (!(await plugins.smartfile.fs.fileExists(imagePath))) {
+      throw new Error(`Image ${imageName} does not exist.`);
+    }
+
+    return plugins.smartfile.fsStream.createReadStream(imagePath);
+  }
+}

ts/classes.network.ts

@@ -1,9 +1,9 @@
-import * as plugins from './docker.plugins.js';
+import * as plugins from './plugins.js';
 import * as interfaces from './interfaces/index.js';
 
-import { DockerHost } from './docker.classes.host.js';
-import { DockerService } from './docker.classes.service.js';
-import { logger } from './docker.logging.js';
+import { DockerHost } from './classes.host.js';
+import { DockerService } from './classes.service.js';
+import { logger } from './logging.js';
 
 export class DockerNetwork {
   public static async getNetworks(dockerHost: DockerHost): Promise<DockerNetwork[]> {

ts/classes.secret.ts

@@ -1,5 +1,5 @@
-import * as plugins from './docker.plugins.js';
-import { DockerHost } from './docker.classes.host.js';
+import * as plugins from './plugins.js';
+import { DockerHost } from './classes.host.js';
 
 // interfaces
 import * as interfaces from './interfaces/index.js';

ts/classes.service.ts

@@ -1,10 +1,10 @@
-import * as plugins from './docker.plugins.js';
+import * as plugins from './plugins.js';
 import * as interfaces from './interfaces/index.js';
 
-import { DockerHost } from './docker.classes.host.js';
-import { DockerImage } from './docker.classes.image.js';
-import { DockerSecret } from './docker.classes.secret.js';
-import { logger } from './docker.logging.js';
+import { DockerHost } from './classes.host.js';
+import { DockerImage } from './classes.image.js';
+import { DockerSecret } from './classes.secret.js';
+import { logger } from './logging.js';
 
 export class DockerService {
   // STATIC
@@ -232,7 +232,9 @@ export class DockerService {
 
     await this.reReadFromDockerEngine();
     const dockerImage = await DockerImage.createFromRegistry(this.dockerHostRef, {
-      imageUrl: this.Spec.TaskTemplate.ContainerSpec.Image,
+      creationObject: {
+        imageUrl: this.Spec.TaskTemplate.ContainerSpec.Image,
+      }
     });
 
     const imageVersion = new plugins.smartversion.SmartVersion(dockerImage.Labels.version);

ts/index.ts

@@ -1,6 +1,6 @@
-export * from './docker.classes.host.js';
-export * from './docker.classes.container.js';
-export * from './docker.classes.image.js';
-export * from './docker.classes.network.js';
-export * from './docker.classes.secret.js';
-export * from './docker.classes.service.js';
+export * from './classes.host.js';
+export * from './classes.container.js';
+export * from './classes.image.js';
+export * from './classes.network.js';
+export * from './classes.secret.js';
+export * from './classes.service.js';

ts/interfaces/container.ts

@@ -1,4 +1,4 @@
-import { DockerNetwork } from '../docker.classes.network.js';
+import { DockerNetwork } from '../classes.network.js';
 
 export interface IContainerCreationDescriptor {
   Hostname: string;

ts/interfaces/service.ts

@@ -1,9 +1,9 @@
-import * as plugins from '../docker.plugins.js';
+import * as plugins from '../plugins.js';
 
 import * as interfaces from './index.js';
-import { DockerNetwork } from '../docker.classes.network.js';
-import { DockerSecret } from '../docker.classes.secret.js';
-import { DockerImage } from '../docker.classes.image.js';
+import { DockerNetwork } from '../classes.network.js';
+import { DockerSecret } from '../classes.secret.js';
+import { DockerImage } from '../classes.image.js';
 
 export interface IServiceCreationDescriptor {
   name: string;

ts/logging.ts

@@ -1,3 +1,3 @@
-import * as plugins from './docker.plugins.js';
+import * as plugins from './plugins.js';
 
 export const logger = new plugins.smartlog.ConsoleLog();

ts/paths.ts

@@ -0,0 +1,9 @@
+import * as plugins from './plugins.js';
+
+export const packageDir = plugins.path.resolve(
+  plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
+  '../'
+);
+
+export const nogitDir = plugins.path.resolve(packageDir, '.nogit/');
+plugins.smartfile.fs.ensureDir(nogitDir);

ts/plugins.ts

@@ -13,6 +13,7 @@ import * as smartpath from '@push.rocks/smartpath';
 import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrequest from '@push.rocks/smartrequest';
 import * as smartstring from '@push.rocks/smartstring';
+import * as smartstream from '@push.rocks/smartstream';
 import * as smartversion from '@push.rocks/smartversion';
 
 export {
@@ -25,6 +26,7 @@ export {
   smartpromise,
   smartrequest,
   smartstring,
+  smartstream,
   smartversion,
 };