From 2e7e8ae5cfd2256c97f022a32d46a86641dc1170 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Mon, 10 Jun 2024 00:15:01 +0200 Subject: [PATCH] fix(core): update --- package.json | 7 ++-- pnpm-lock.yaml | 79 +++++++++++++++++++++++++++------------- qenv.yml | 6 +++ test/test.nonci.node.ts | 15 +++++++- ts/00_commitinfo_data.ts | 2 +- ts/classes.host.ts | 18 +++++++++ ts/classes.image.ts | 2 +- ts/classes.imagestore.ts | 6 +++ 8 files changed, 104 insertions(+), 31 deletions(-) create mode 100644 qenv.yml diff --git a/package.json b/package.json index 1de30a5..5281c3a 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,8 @@ "homepage": "https://gitlab.com/mojoio/docker#readme", "dependencies": { "@push.rocks/lik": "^6.0.15", - "@push.rocks/smartarchive": "^4.0.36", - "@push.rocks/smartbucket": "^3.0.10", + "@push.rocks/smartarchive": "^4.0.37", + "@push.rocks/smartbucket": "^3.0.13", "@push.rocks/smartfile": "^11.0.20", "@push.rocks/smartjson": "^5.0.20", "@push.rocks/smartlog": "^3.0.7", @@ -47,13 +47,14 @@ "@push.rocks/smartstring": "^4.0.15", "@push.rocks/smartunique": "^3.0.9", "@push.rocks/smartversion": "^3.0.5", - "@tsclass/tsclass": "^4.0.54", + "@tsclass/tsclass": "^4.0.55", "rxjs": "^7.5.7" }, "devDependencies": { "@git.zone/tsbuild": "^2.1.80", "@git.zone/tsrun": "^1.2.12", "@git.zone/tstest": "^1.0.90", + "@push.rocks/qenv": "^6.0.5", "@push.rocks/tapbundle": "^5.0.23", "@types/node": "20.14.2" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2143fc5..2674508 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,11 +12,11 @@ importers: specifier: ^6.0.15 version: 6.0.15 '@push.rocks/smartarchive': - specifier: ^4.0.36 - version: 4.0.36 + specifier: ^4.0.37 + version: 4.0.37 '@push.rocks/smartbucket': - specifier: ^3.0.10 - version: 3.0.10 + specifier: ^3.0.13 + version: 3.0.13 '@push.rocks/smartfile': specifier: ^11.0.20 version: 11.0.20 @@ -51,8 +51,8 @@ importers: specifier: ^3.0.5 version: 3.0.5 '@tsclass/tsclass': - specifier: ^4.0.54 - version: 4.0.54 + specifier: ^4.0.55 + version: 4.0.55 rxjs: specifier: ^7.5.7 version: 7.8.1 @@ -66,6 +66,9 @@ importers: '@git.zone/tstest': specifier: ^1.0.90 version: 1.0.90(@types/node@20.14.2) + '@push.rocks/qenv': + specifier: ^6.0.5 + version: 6.0.5 '@push.rocks/tapbundle': specifier: ^5.0.23 version: 5.0.23 @@ -108,6 +111,9 @@ packages: resolution: {integrity: sha512-8lLmua6AVh/8SLJRRVD6V8p73Hir9w5mJrhE+IPpILG31KKlI9iz5zmBYKcWPS59qSfgP9RaSBQSHHE81WKuEw==} engines: {node: '>=6.9.0'} + '@configvault.io/interfaces@1.0.17': + resolution: {integrity: sha512-bEcCUR2VBDJsTin8HQh8Uw/mlYl2v8A3jMIaQ+MTB9Hrqd6CZL2dL7iJdWyFl/3EIX+LDxWFR+Oq7liIq7w+1Q==} + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -344,14 +350,17 @@ packages: '@push.rocks/lik@6.0.15': resolution: {integrity: sha512-rZxln6l4NAU931MTxnsjy1pue+S3AXtDCidHH/tbkqBtrWIzWuXduo6Nz3zYkndbD64Knyta7F60JRvcOe4XqA==} - '@push.rocks/smartarchive@4.0.36': - resolution: {integrity: sha512-zFd0xKK2TsE/mOxsxp4MrAmH9skFOJijMG0VuHjkYB5WRspvoaOssMypiILCXy/m7vEhqAVGmgrk8Ed0p4gDMQ==} + '@push.rocks/qenv@6.0.5': + resolution: {integrity: sha512-Id/eSKKqSDUGe+0Cp5HEJ58J1iVv1jQseLUMs9kFTPYwG+NJSETUCRsJV50w5cPv8bRFcSkSU+xVbUbOc1p29A==} + + '@push.rocks/smartarchive@4.0.37': + resolution: {integrity: sha512-pqAEZZY5uoZV9g1/8dPys4vQTCtSpOBf46+NcR1F+/RJCtqhg+emLeFXJDO+mBU/u8+upEfbQDACp5p/GvW1PA==} '@push.rocks/smartbrowser@2.0.6': resolution: {integrity: sha512-Ne+KCVhV/DROc1rHRRw59K6h0+LpQAK9fdOUtgDZ7laLPmB/tmnbUh3IuRDNcIY1iVA9pydoobwjnTjVgio9eQ==} - '@push.rocks/smartbucket@3.0.10': - resolution: {integrity: sha512-KzEDWtrZbs5gD042LGa61jtIgOz6+MmQIIccPO0d2YQRNpgrO+CAAKyhOHgvWaPPTlaj72XGM8pBhn/2hCdSUA==} + '@push.rocks/smartbucket@3.0.13': + resolution: {integrity: sha512-pw0l6ymgXMx/xxRxSd2Ha3M+vtgU/vh0BiD+blGBgiNLzysqKItuecES/oCdElfdpVkeTN++XfGJ1FunyLAq9A==} '@push.rocks/smartbuffer@3.0.4': resolution: {integrity: sha512-TLfhx/JD61YC8XGO9TI6Ux6US38R14HaIM84QT8hZZod8axfXrg+h8xA8tMUBpSV8PXsQy9LzxmOq0Il1fmDXw==} @@ -615,8 +624,8 @@ packages: '@tsclass/tsclass@3.0.48': resolution: {integrity: sha512-hC65UvDlp9qvsl6OcIZXz0JNiWZ0gyzsTzbXpg215sGxopgbkOLCr6E0s4qCTnweYm95gt2AdY95uP7M7kExaQ==} - '@tsclass/tsclass@4.0.54': - resolution: {integrity: sha512-v+Xc7M0BKNT79/kx7S5Jsc17zj+acUuMDxJtKbQgdU4H8ke3aHAHJr2KicXJeXDTcn41ZEbwJPQ1cc+bjy8bZw==} + '@tsclass/tsclass@4.0.55': + resolution: {integrity: sha512-zg774JF90/3/rJ7xk4LyGgxcUzxdKIQcwtBVxez4LhvegESxvHiFmX42WL105iBpE53ISJ8sctLWlwG1JQZdlA==} '@tsconfig/node10@1.0.11': resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==} @@ -1364,6 +1373,10 @@ packages: resolution: {integrity: sha512-M2SovcRxD4+vC493Uc2GZVcZaj66CCJhWurC4viynVSTvrpErCShNcDz1lAho6n9REQKvL/ll4A4/fw6Y9z8nw==} hasBin: true + fast-xml-parser@4.4.0: + resolution: {integrity: sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==} + hasBin: true + fastq@1.17.1: resolution: {integrity: sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==} @@ -2517,8 +2530,8 @@ packages: resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} engines: {node: '>=12.20'} - type-fest@4.18.2: - resolution: {integrity: sha512-+suCYpfJLAe4OXS6+PPXjW3urOS4IoP9waSiLuXfLgqZODKw/aWwASvzqE886wA0kQgGy0mIWyhd87VpqIy6Xg==} + type-fest@4.20.0: + resolution: {integrity: sha512-MBh+PHUHHisjXf4tlx0CFWoMdjx8zCMLJHOjnV1prABYZFHqtFOyauCIK2/7w4oIfwkF8iNhLtnJEfVY2vn3iw==} engines: {node: '>=16'} type-is@1.6.18: @@ -2756,7 +2769,7 @@ snapshots: '@push.rocks/smartstream': 3.0.44 '@push.rocks/smarttime': 4.0.6 '@push.rocks/webstore': 2.0.14 - '@tsclass/tsclass': 4.0.54 + '@tsclass/tsclass': 4.0.55 '@types/express': 4.17.21 body-parser: 1.20.2 cors: 2.8.5 @@ -2801,6 +2814,10 @@ snapshots: js-tokens: 4.0.0 picocolors: 1.0.0 + '@configvault.io/interfaces@1.0.17': + dependencies: + '@api.global/typedrequest-interfaces': 3.0.19 + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -3054,7 +3071,15 @@ snapshots: '@types/symbol-tree': 3.2.5 symbol-tree: 3.2.4 - '@push.rocks/smartarchive@4.0.36': + '@push.rocks/qenv@6.0.5': + dependencies: + '@api.global/typedrequest': 3.0.23 + '@configvault.io/interfaces': 1.0.17 + '@push.rocks/smartfile': 11.0.20 + '@push.rocks/smartlog': 3.0.7 + '@push.rocks/smartpath': 5.0.18 + + '@push.rocks/smartarchive@4.0.37': dependencies: '@push.rocks/smartdelay': 3.0.5 '@push.rocks/smartfile': 11.0.20 @@ -3083,7 +3108,7 @@ snapshots: - supports-color - utf-8-validate - '@push.rocks/smartbucket@3.0.10': + '@push.rocks/smartbucket@3.0.13': dependencies: '@push.rocks/smartmime': 2.0.2 '@push.rocks/smartpath': 5.0.18 @@ -3091,7 +3116,7 @@ snapshots: '@push.rocks/smartrx': 3.0.7 '@push.rocks/smartstream': 3.0.44 '@push.rocks/smartunique': 3.0.9 - '@tsclass/tsclass': 4.0.54 + '@tsclass/tsclass': 4.0.55 minio: 8.0.0 '@push.rocks/smartbuffer@3.0.4': @@ -3229,7 +3254,7 @@ snapshots: '@push.rocks/smartlog-interfaces@3.0.2': dependencies: '@api.global/typedrequest-interfaces': 2.0.2 - '@tsclass/tsclass': 4.0.54 + '@tsclass/tsclass': 4.0.55 '@push.rocks/smartlog@3.0.7': dependencies: @@ -3289,7 +3314,7 @@ snapshots: '@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartpuppeteer': 2.0.2 '@push.rocks/smartunique': 3.0.9 - '@tsclass/tsclass': 4.0.54 + '@tsclass/tsclass': 4.0.55 '@types/express': 4.17.21 express: 4.19.2 pdf-lib: 1.17.1 @@ -3343,7 +3368,7 @@ snapshots: '@push.rocks/smartxml': 1.0.8 '@push.rocks/smartyaml': 2.0.5 '@push.rocks/webrequest': 3.0.37 - '@tsclass/tsclass': 4.0.54 + '@tsclass/tsclass': 4.0.55 '@push.rocks/smartsocket@2.0.27': dependencies: @@ -3630,9 +3655,9 @@ snapshots: dependencies: type-fest: 2.19.0 - '@tsclass/tsclass@4.0.54': + '@tsclass/tsclass@4.0.55': dependencies: - type-fest: 4.18.2 + type-fest: 4.20.0 '@tsconfig/node10@1.0.11': {} @@ -4476,6 +4501,10 @@ snapshots: dependencies: strnum: 1.0.5 + fast-xml-parser@4.4.0: + dependencies: + strnum: 1.0.5 + fastq@1.17.1: dependencies: reusify: 1.0.4 @@ -5086,7 +5115,7 @@ snapshots: browser-or-node: 2.1.1 buffer-crc32: 1.0.0 eventemitter3: 5.0.1 - fast-xml-parser: 4.3.6 + fast-xml-parser: 4.4.0 ipaddr.js: 2.2.0 lodash: 4.17.21 mime-types: 2.1.35 @@ -5690,7 +5719,7 @@ snapshots: type-fest@2.19.0: {} - type-fest@4.18.2: {} + type-fest@4.20.0: {} type-is@1.6.18: dependencies: diff --git a/qenv.yml b/qenv.yml new file mode 100644 index 0000000..bc362e5 --- /dev/null +++ b/qenv.yml @@ -0,0 +1,6 @@ +required: + - S3_ENDPOINT + - S3_ACCESSKEY + - S3_ACCESSSECRET + - S3_BUCKET + \ No newline at end of file diff --git a/test/test.nonci.node.ts b/test/test.nonci.node.ts index 0c1d7fc..92984a5 100644 --- a/test/test.nonci.node.ts +++ b/test/test.nonci.node.ts @@ -1,4 +1,7 @@ import { expect, tap } from '@push.rocks/tapbundle'; +import { Qenv } from '@push.rocks/qenv'; + +const testQenv = new Qenv('./', './.nogit/'); import * as plugins from '../ts/plugins.js'; import * as paths from '../ts/paths.js'; @@ -136,7 +139,7 @@ tap.skip.test('should export images', async (toolsArg) => { await done.promise; }); -tap.skip.test('should import images', async (toolsArg) => { +tap.test('should import images', async (toolsArg) => { const done = toolsArg.defer(); const fsReadStream = plugins.smartfile.fsStream.createReadStream( plugins.path.join(paths.nogitDir, 'testimage.tar') @@ -150,6 +153,16 @@ tap.skip.test('should import images', async (toolsArg) => { }); tap.test('should expose a working DockerImageStore', async () => { + // lets first add am s3 target + const s3Descriptor = { + endpoint: await testQenv.getEnvVarOnDemand('S3_ENDPOINT'), + accessKey: await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'), + accessSecret: await testQenv.getEnvVarOnDemand('S3_ACCESSSECRET'), + bucketName: await testQenv.getEnvVarOnDemand('S3_BUCKET'), + }; + await testDockerHost.addS3Storage(s3Descriptor); + + // await testDockerHost.imageStore.storeImage('hello', plugins.smartfile.fsStream.createReadStream(plugins.path.join(paths.nogitDir, 'testimage.tar'))); }) diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 27b9e79..1a323c9 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@apiclient.xyz/docker', - version: '1.2.0', + version: '1.2.1', description: 'Provides easy communication with Docker remote API from Node.js, with TypeScript support.' } diff --git a/ts/classes.host.ts b/ts/classes.host.ts index 29cdeae..4a73db7 100644 --- a/ts/classes.host.ts +++ b/ts/classes.host.ts @@ -28,6 +28,7 @@ export class DockerHost { public socketPath: string; private registryToken: string = ''; public imageStore: DockerImageStore; + public smartBucket: plugins.smartbucket.SmartBucket; /** * the constructor to instantiate a new docker sock instance @@ -274,4 +275,21 @@ export class DockerHost { console.log(response.body); return response; } + + /** + * add s3 storage + * @param optionsArg + */ + public async addS3Storage(optionsArg: plugins.tsclass.storage.IS3Descriptor) { + this.smartBucket = new plugins.smartbucket.SmartBucket(optionsArg); + if (!optionsArg.bucketName) { + throw new Error('bucketName is required'); + } + const bucket = await this.smartBucket.getBucketByName(optionsArg.bucketName); + let wantedDirectory = await bucket.getBaseDirectory(); + if (optionsArg.directoryPath) { + wantedDirectory = await wantedDirectory.getSubDirectoryByName(optionsArg.directoryPath); + } + this.imageStore.options.bucketDir = wantedDirectory; + } } diff --git a/ts/classes.image.ts b/ts/classes.image.ts index c6007f0..2298e56 100644 --- a/ts/classes.image.ts +++ b/ts/classes.image.ts @@ -166,7 +166,7 @@ export class DockerImage { * exports an image to a tar ball */ public async exportToTarStream(): Promise { - console.log(`Exporting image ${this.RepoTags[0]} to tar stream.`); + logger.log('info', `Exporting image ${this.RepoTags[0]} to tar stream.`); const response = await this.dockerHost.requestStreaming('GET', `/images/${encodeURIComponent(this.RepoTags[0])}/get`); let counter = 0; const webduplexStream = new plugins.smartstream.SmartDuplex({ diff --git a/ts/classes.imagestore.ts b/ts/classes.imagestore.ts index 72f57cd..b2bf3d9 100644 --- a/ts/classes.imagestore.ts +++ b/ts/classes.imagestore.ts @@ -87,6 +87,12 @@ export class DockerImageStore { }); logger.log('ok', `Repackaged image ${imageName} for s3.`); await plugins.smartfile.fs.remove(extractionDir); + const finalTarReadStream = plugins.smartfile.fsStream.createReadStream(finalTarPath); + await this.options.bucketDir.fastPutStream({ + stream: finalTarReadStream, + path: `${imageName}.tar`, + }); + await plugins.smartfile.fs.remove(finalTarPath); } public async start() {