apiclient.xyz/docker
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(imagestore): now processing images with extraction, retagging, repackaging and long term storage

Browse Source
This commit is contained in:
Philipp Kunz
2024-06-08 15:03:19 +02:00
parent 740f83114c
commit c0cebbe614
13 changed files with 241 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
ts/classes.imagestore.ts
View File

@@ -1,5 +1,6 @@
import * as plugins from './plugins.js';
import * as paths from './paths.js';
import { logger } from './logger.js';
import type { DockerHost } from './classes.host.js';
export interface IDockerImageStoreConstructorOptions {
@@ -22,19 +23,78 @@ export class DockerImageStore {
// Method to store tar stream
public async storeImage(imageName: string, tarStream: plugins.smartstream.stream.Readable): Promise<void> {
const imagePath = plugins.path.join(this.options.localDirPath, `${imageName}.tar`);
logger.log('info', `Storing image ${imageName}...`);
const uniqueProcessingId = plugins.smartunique.shortId();
const initialTarDownloadPath = plugins.path.join(this.options.localDirPath, `${uniqueProcessingId}.tar`);
const extractionDir = plugins.path.join(this.options.localDirPath, uniqueProcessingId);
// Create a write stream to store the tar file
const writeStream = plugins.smartfile.fsStream.createWriteStream(imagePath);
const writeStream = plugins.smartfile.fsStream.createWriteStream(initialTarDownloadPath);
return new Promise((resolve, reject) => {
// lets wait for the write stream to finish
await new Promise((resolve, reject) => {
tarStream.pipe(writeStream);
writeStream.on('finish', resolve);
writeStream.on('error', reject);
});
logger.log('info', `Image ${imageName} stored locally for processing. Extracting...`);
// lets process the image
const tarArchive = await plugins.smartarchive.SmartArchive.fromArchiveFile(initialTarDownloadPath);
await tarArchive.exportToFs(extractionDir);
logger.log('info', `Image ${imageName} extracted.`);
await plugins.smartfile.fs.remove(initialTarDownloadPath);
logger.log('info', `deleted original tar to save space.`);
logger.log('info', `now repackaging for s3...`);
const smartfileIndexJson = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(extractionDir, 'index.json'));
const smartfileManifestJson = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(extractionDir, 'manifest.json'));
const smartfileOciLayoutJson = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(extractionDir, 'oci-layout'));
const smartfileRepositoriesJson = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(extractionDir, 'repositories'));
const indexJson = JSON.parse(smartfileIndexJson.contents.toString());
const manifestJson = JSON.parse(smartfileManifestJson.contents.toString());
const ociLayoutJson = JSON.parse(smartfileOciLayoutJson.contents.toString());
const repositoriesJson = JSON.parse(smartfileRepositoriesJson.contents.toString());
indexJson.manifests[0].annotations['io.containerd.image.name'] = imageName;
manifestJson[0].RepoTags[0] = imageName;
const repoFirstKey = Object.keys(repositoriesJson)[0];
const repoFirstValue = repositoriesJson[repoFirstKey];
repositoriesJson[imageName] = repoFirstValue;
delete repositoriesJson[repoFirstKey];
smartfileIndexJson.contents = Buffer.from(JSON.stringify(indexJson, null, 2));
smartfileManifestJson.contents = Buffer.from(JSON.stringify(manifestJson, null, 2));
smartfileOciLayoutJson.contents = Buffer.from(JSON.stringify(ociLayoutJson, null, 2));
smartfileRepositoriesJson.contents = Buffer.from(JSON.stringify(repositoriesJson, null, 2));
await Promise.all([
smartfileIndexJson.write(),
smartfileManifestJson.write(),
smartfileOciLayoutJson.write(),
smartfileRepositoriesJson.write(),
]);
logger.log('info', 'repackaging archive for s3...');
const tartools = new plugins.smartarchive.TarTools();
const newTarPack = await tartools.packDirectory(extractionDir);
const finalTarName = `${uniqueProcessingId}.processed.tar`;
const finalTarPath = plugins.path.join(this.options.localDirPath, finalTarName);
const finalWriteStream = plugins.smartfile.fsStream.createWriteStream(finalTarPath);
await new Promise((resolve, reject) => {
newTarPack.finalize();
newTarPack.pipe(finalWriteStream);
finalWriteStream.on('finish', resolve);
finalWriteStream.on('error', reject);
});
logger.log('ok', `Repackaged image ${imageName} for s3.`);
await plugins.smartfile.fs.remove(extractionDir);
}
public async start() {
await plugins.smartfile.fs.ensureEmptyDir(this.options.localDirPath);
}
public async stop() {}
// Method to retrieve tar stream
public async getImage(imageName: string): Promise<plugins.smartstream.stream.Readable> {
const imagePath = plugins.path.join(this.options.localDirPath, `${imageName}.tar`);