1.0.23

.gitlab-ci.yml

@@ -38,22 +38,34 @@ snyk:
 # test stage
 # ====================
 
-testLTS:
+testStable:
   stage: test
   script:
   - npmci npm prepare
-  - npmci node install lts
+  - npmci node install stable
   - npmci npm install
   - npmci npm test
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
   - docker
+  - priv
+
+testBuild:
+  stage: test
+  script:
+  - npmci npm prepare
+  - npmci node install stable
+  - npmci npm install
+  - npmci command npm run build
+  coverage: /\d+.?\d+?\%\s*coverage/
+  tags:
+  - docker
   - notpriv
 
 release:
   stage: release
   script:
-  - npmci node install lts
+  - npmci node install stable
   - npmci npm publish
   only:
   - tags
@@ -69,6 +81,7 @@ codequality:
   allow_failure: true
   script:
     - npmci command npm install -g tslint typescript
+    - npmci npm prepare
     - npmci npm install
     - npmci command "tslint -c tslint.json ./ts/**/*.ts"
   tags:
@@ -86,9 +99,9 @@ trigger:
   - notpriv
 
 pages:
-  image: hosttoday/ht-docker-node:npmci
   stage: metadata
   script:
+    - npmci node install lts
     - npmci command npm install -g @gitzone/tsdoc
     - npmci npm prepare
     - npmci npm install

.vscode/launch.json

@@ -0,0 +1,29 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "current file",
+      "type": "node",
+      "request": "launch",
+      "args": [
+        "${relativeFile}"
+      ],
+      "runtimeArgs": ["-r", "@gitzone/tsrun"],
+      "cwd": "${workspaceRoot}",
+      "protocol": "inspector",
+      "internalConsoleOptions": "openOnSessionStart"
+    },
+    {
+      "name": "test.ts",
+      "type": "node",
+      "request": "launch",
+      "args": [
+        "test/test.ts"
+      ],
+      "runtimeArgs": ["-r", "@gitzone/tsrun"],
+      "cwd": "${workspaceRoot}",
+      "protocol": "inspector",
+      "internalConsoleOptions": "openOnSessionStart"
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,20 @@
+{
+  "json.schemas": [
+    {
+      "fileMatch": ["/npmextra.json"],
+      "schema": {
+        "type": "object",
+        "properties": {
+          "npmci": {
+            "type": "object",
+            "description": "settings for npmci"
+          },
+          "gitzone": {
+            "type": "object",
+            "description": "settings for gitzone"
+          }
+        }
+      }
+    }
+  ]
+}

npmextra.json

@@ -1,8 +1,6 @@
 {
   "npmci": {
-    "npmGlobalTools": [
-      "npmts"
-    ]
+    "npmGlobalTools": []
   },
   "gitzone": {
     "module": {

package.json

@@ -1,34 +1,39 @@
 {
   "name": "@pushrocks/smartbucket",
-  "version": "1.0.4",
+  "version": "1.0.23",
   "description": "simple cloud independent object storage",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
   "author": "Lossless GmbH",
   "license": "UNLICENSED",
   "scripts": {
-    "test": "(npmts)",
+    "test": "(tstest test/)",
     "format": "(gitzone format)",
-    "build": "echo \"Not needed for now\""
+    "build": "(tsbuild)"
   },
   "devDependencies": {
-    "@pushrocks/tapbundle": "^3.0.11",
-    "tslint": "^5.18.0",
+    "@gitzone/tsbuild": "^2.1.17",
+    "@gitzone/tstest": "^1.0.28",
+    "@pushrocks/tapbundle": "^3.0.13",
+    "tslint": "^5.20.0",
     "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
-    "@google-cloud/storage": "^3.0.2",
-    "@pushrocks/smartpromise": "^3.0.2",
-    "@types/minio": "^7.0.2",
-    "minio": "^7.0.10"
+    "@pushrocks/qenv": "^4.0.6",
+    "@pushrocks/smartpath": "^4.0.1",
+    "@pushrocks/smartpromise": "^3.0.6",
+    "@pushrocks/streamfunction": "^1.0.24",
+    "@types/minio": "^7.0.3",
+    "minio": "^7.0.12"
   },
-  "private": true,
+  "private": false,
   "files": [
-    "ts/*",
-    "ts_web/*",
-    "dist/*",
-    "dist_web/*",
-    "assets/*",
+    "ts/**/*",
+    "ts_web/**/*",
+    "dist/**/*",
+    "dist_web/**/*",
+    "dist_ts_web/**/*",
+    "assets/**/*",
     "cli.js",
     "npmextra.json",
     "readme.md"

qenv.yml

@@ -0,0 +1,3 @@
+required:
+  - S3_KEY
+  - S3_SECRET

readme.md

@@ -21,6 +21,6 @@ simple cloud independent object storage
 For further information read the linked docs at the top of this readme.
 
 > MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
-| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy.html)
+| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
 
-[![repo-footer](https://pushrocks.gitlab.io/assets/repo-footer.svg)](https://maintainedby.lossless.com)
+[![repo-footer](https://lossless.gitlab.io/publicrelations/repofooter.svg)](https://maintainedby.lossless.com)

test/test.ts

@@ -1,8 +1,87 @@
 import { expect, tap } from '@pushrocks/tapbundle';
+import { Qenv } from '@pushrocks/qenv';
+
 import * as smartbucket from '../ts/index';
 
-tap.test('first test', async () => {
-  console.log('hi');
+const testQenv = new Qenv('./', './.nogit/');
+
+let testSmartbucket: smartbucket.SmartBucket;
+let myBucket: smartbucket.Bucket;
+let baseDirectory: smartbucket.Directory;
+
+tap.test('should create a valid smartbucket', async () => {
+  testSmartbucket = new smartbucket.SmartBucket({
+    accessKey: testQenv.getEnvVarOnDemand('S3_KEY'),
+    accessSecret: testQenv.getEnvVarOnDemand('S3_SECRET'),
+    endpoint: 'fra1.digitaloceanspaces.com'
+  });
+});
+
+tap.skip.test('should create testbucket', async () => {
+  await testSmartbucket.createBucket('testzone');
+});
+
+tap.skip.test('should remove testbucket', async () => {
+  await testSmartbucket.removeBucket('testzone');
+});
+
+tap.test('should get a bucket', async () => {
+  myBucket = await testSmartbucket.getBucketByName('testzone');
+  expect(myBucket).to.be.instanceOf(smartbucket.Bucket);
+  expect(myBucket.name).to.equal('testzone');
+});
+
+// Fast operations
+tap.test('should store data in bucket fast', async () => {
+  await myBucket.fastStore('hithere/socool.txt', 'hi there!');
+});
+
+tap.test('should get data in bucket', async () => {
+  const fileString = await myBucket.fastGet('hithere/socool.txt');
+  console.log(fileString);
+});
+
+tap.test('should delete data in bucket', async () => {
+  await myBucket.fastRemove('hithere/socool.txt');
+});
+
+// fs operations
+
+tap.test('prepare for directory style tests', async () => {
+  await myBucket.fastStore('dir1/file1.txt', 'dir1/file1.txt content');
+  await myBucket.fastStore('dir1/file2.txt', 'dir1/file2.txt content');
+  await myBucket.fastStore('dir2/file1.txt', 'dir2/file1.txt content');
+  await myBucket.fastStore('dir3/file1.txt', 'dir3/file1.txt content');
+  await myBucket.fastStore('dir3/dir4/file1.txt', 'dir3/dir4/file1.txt content');
+  await myBucket.fastStore('file1.txt', 'file1 content');
+});
+
+tap.test('should get base directory', async () => {
+  baseDirectory = await myBucket.getBaseDirectory();
+  const directories = await baseDirectory.listDirectories();
+  console.log('Found the following directories:');
+  console.log(directories);
+  expect(directories.length).to.equal(3);
+  const files = await baseDirectory.listFiles();
+  console.log('Found the following files:');
+  console.log(files);
+  expect(files.length).to.equal(1);
+});
+
+tap.test('should correctly build paths for sub directories', async () => {
+  const dir4 = await baseDirectory.getSubDirectoryByName('dir3/dir4');
+  expect(dir4).to.be.instanceOf(smartbucket.Directory);
+  const dir4BasePath = dir4.getBasePath();
+  console.log(dir4BasePath);
+});
+
+tap.test('clean up directory style tests', async () => {
+  await myBucket.fastRemove('dir1/file1.txt');
+  await myBucket.fastRemove('dir1/file2.txt');
+  await myBucket.fastRemove('dir2/file1.txt');
+  await myBucket.fastRemove('dir3/file1.txt');
+  await myBucket.fastRemove('dir3/dir4/file1.txt');
+  await myBucket.fastRemove('file1.txt');
 });
 
 tap.start();

ts/index.ts

@@ -1 +1,4 @@
-import * as plugins from './smartbucket.plugins';
+export * from './smartbucket.classes.smartbucket';
+export * from './smartbucket.classes.bucket';
+export * from './smartbucket.classes.directory';
+export * from './smartbucket.classes.file';

ts/smartbucket.classes.bucket.ts

@@ -0,0 +1,98 @@
+import * as plugins from './smartbucket.plugins';
+import { SmartBucket } from './smartbucket.classes.smartbucket';
+import { Directory } from './smartbucket.classes.directory';
+
+export class Bucket {
+  public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string) {
+    const buckets = await smartbucketRef.minioClient.listBuckets();
+    const foundBucket = buckets.find(bucket => {
+      return bucket.name === bucketNameArg;
+    });
+
+    if (foundBucket) {
+      console.log(`bucket with name ${bucketNameArg} exists.`);
+      console.log(`Taking this as base for new Bucket instance`);
+      return new this(smartbucketRef, bucketNameArg);
+    } else {
+      return null;
+    }
+  }
+
+  public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
+    await smartbucketRef.minioClient.makeBucket(bucketName, 'ams3').catch(e => console.log(e));
+    return new Bucket(smartbucketRef, bucketName);
+  }
+
+  public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
+    await smartbucketRef.minioClient.removeBucket(bucketName).catch(e => console.log(e));
+  }
+
+  public smartbucketRef: SmartBucket;
+  public name: string;
+
+  constructor(smartbucketRef: SmartBucket, bucketName: string) {
+    this.smartbucketRef = smartbucketRef;
+    this.name = bucketName;
+  }
+
+  /**
+   * gets the base directory of the bucket
+   */
+  public async getBaseDirectory() {
+    return new Directory(this, null, '');
+  }
+
+  // ===============
+  // Fast Operations
+  // ===============
+
+  /**
+   * store file
+   */
+  public async fastStore(pathArg: string, fileContent: string) {
+    const streamIntake = new plugins.streamfunction.Intake();
+    const putPromise = this.smartbucketRef.minioClient
+      .putObject(this.name, pathArg, streamIntake.getReadable())
+      .catch(e => console.log(e));
+    streamIntake.pushData(fileContent);
+    streamIntake.signalEnd();
+    await putPromise;
+  }
+
+  /**
+   * get file
+   */
+  public async fastGet(pathArg: string) {
+    const done = plugins.smartpromise.defer();
+    const fileStream = await this.smartbucketRef.minioClient
+      .getObject(this.name, pathArg)
+      .catch(e => console.log(e));
+    let completeFile: string = '';
+    const duplexStream = plugins.streamfunction.createDuplexStream<Buffer, Buffer>(
+      async chunk => {
+        const chunkString = chunk.toString();
+        completeFile += chunkString;
+        return chunk;
+      },
+      async cb => {
+        done.resolve();
+        return Buffer.from('');
+      }
+    );
+
+    if (!fileStream) {
+      return null;
+    }
+
+    fileStream.pipe(duplexStream);
+    await done.promise;
+    return completeFile;
+  }
+
+  /**
+   * removeObject
+   */
+  public async fastRemove(pathArg: string) {
+    await this.smartbucketRef.minioClient.removeObject(this.name, pathArg);
+  }
+}

ts/smartbucket.classes.directory.ts

@@ -0,0 +1,184 @@
+import * as plugins from './smartbucket.plugins';
+import { Bucket } from './smartbucket.classes.bucket';
+import { File } from './smartbucket.classes.file';
+
+export class Directory {
+  public bucketRef: Bucket;
+  public parentDirectoryRef: Directory;
+  public name: string;
+
+  public tree: string[];
+  public files: string[];
+  public folders: string[];
+
+  constructor(bucketRefArg: Bucket, parentDiretory: Directory, name: string) {
+    this.bucketRef = bucketRefArg;
+    this.parentDirectoryRef = parentDiretory;
+    this.name = name;
+  }
+
+  /**
+   * returns an array of parent directories
+   */
+  public getParentDirectories(): Directory[] {
+    let parentDirectories: Directory[] = [];
+    if (this.parentDirectoryRef) {
+      parentDirectories.push(this.parentDirectoryRef);
+      parentDirectories = parentDirectories.concat(this.parentDirectoryRef.getParentDirectories());
+    }
+    return parentDirectories;
+  }
+
+  /**
+   * returns the directory level
+   */
+  public getDirectoryLevel(): number {
+    return this.getParentDirectories().length;
+  }
+
+  /**
+   * updates the base path
+   */
+  public getBasePath(): string {
+    const parentDirectories = this.getParentDirectories();
+    let basePath = '';
+    for (const parentDir of parentDirectories) {
+      if (parentDir.name === '') {
+        basePath = this.name;
+        continue;
+      }
+      basePath = parentDir.name + '/' + this.name;
+    }
+    return basePath;
+  }
+
+  /**
+   * lists all files
+   */
+  public async listFiles(): Promise<File[]> {
+    const done = plugins.smartpromise.defer();
+    const fileNameStream = await this.bucketRef.smartbucketRef.minioClient.listObjectsV2(
+      this.bucketRef.name,
+      this.getBasePath()
+    );
+    const fileArray: File[] = [];
+    const duplexStream = plugins.streamfunction.createDuplexStream<plugins.minio.BucketItem, void>(
+      async bucketItem => {
+        if(!bucketItem.name) {
+          return;
+        }
+        let subtractedPath = bucketItem.name.replace(this.getBasePath(), '');
+        if (subtractedPath.startsWith('/')) {
+          subtractedPath = subtractedPath.substr(1);
+        }
+        if (!subtractedPath.includes('/')) {
+          fileArray.push(new File(this, subtractedPath));
+        }
+        
+      },
+      async tools => {
+        done.resolve();
+      }
+    );
+    fileNameStream.pipe(duplexStream);
+    await done.promise;
+    return fileArray;
+  }
+
+  /**
+   * lists all folders
+   */
+  public async listDirectories(): Promise<Directory[]> {
+    const done = plugins.smartpromise.defer();
+    const completeDirStream = await this.bucketRef.smartbucketRef.minioClient.listObjectsV2(
+      this.bucketRef.name,
+      this.getBasePath(),
+      true
+    );
+    const directoryArray: Directory[] = [];
+    const duplexStream = plugins.streamfunction.createDuplexStream<plugins.minio.BucketItem, void>(
+      async bucketItem => {
+        let subtractedPath = bucketItem.name.replace(this.getBasePath(), '');
+        if (subtractedPath.startsWith('/')) {
+          subtractedPath = subtractedPath.substr(1);
+        }
+        if (subtractedPath.includes('/')) {
+          const dirName = subtractedPath.split('/')[0];
+          if (directoryArray.find(directory => directory.name === dirName)) {
+            return;
+          }
+          directoryArray.push(new Directory(this.bucketRef, this, dirName));
+        }
+      },
+      async tools => {
+        done.resolve();
+      }
+    );
+    completeDirStream.pipe(duplexStream);
+    await done.promise;
+    return directoryArray;
+  }
+
+  /**
+   * gets an array that has all objects with a certain prefix;
+   */
+  public async getTreeArray() {
+    const treeArray = await this.bucketRef.smartbucketRef.minioClient.listObjectsV2(
+      this.bucketRef.name,
+      this.getBasePath(),
+      true
+    );
+  }
+
+  /**
+   * gets a sub directory
+   */
+  public async getSubDirectoryByName(dirNameArg: string): Promise<Directory> {
+    const dirNameArray = dirNameArg.split('/');
+
+    const getDirectory = async (directoryArg: Directory, dirNameToSearch: string) => {
+      const directories = await directoryArg.listDirectories();
+      return directories.find(directory => {
+        return directory.name === dirNameToSearch;
+      });
+    };
+    let wantedDirectory: Directory;
+    for (const dirNameToSearch of dirNameArray) {
+      const directoryToSearchIn = wantedDirectory ? wantedDirectory : this;
+      wantedDirectory = await getDirectory(directoryToSearchIn, dirNameToSearch);
+    }
+    return wantedDirectory;
+  }
+
+  /**
+   * moves the directory
+   */
+  public async move() {
+    // TODO
+  }
+
+  /**
+   * creates a file within this directory
+   * @param relativePathArg
+   */
+  public async createFile(relativePathArg) {
+    let completeFilePath: string = '';
+  }
+
+  // file operations
+  public async fastStore(pathArg: string, contentArg: string) {
+    const path = plugins.path.join(this.getBasePath(), pathArg);
+    await this.bucketRef.fastStore(path, contentArg);
+  }
+
+  public async fastGet(pathArg: string) {
+    const path = plugins.path.join(this.getBasePath(), pathArg);
+    const result = await this.bucketRef.fastGet(path);
+    return result;
+  }
+
+  public async fastRemove(pathArg: string) {
+    const path = plugins.path.join(this.getBasePath(), pathArg);
+    await this.bucketRef.fastRemove(path);
+  }
+}

ts/smartbucket.classes.file.ts

@@ -0,0 +1,93 @@
+import * as plugins from './smartbucket.plugins';
+import { Directory } from './smartbucket.classes.directory';
+
+export interface IFileMetaData {
+  name: string;
+  fileType: string;
+  size: string;
+}
+
+export class File {
+  // STATIC
+  public static async createFileFromString(
+    dirArg: Directory,
+    fileName: string,
+    fileContent: string
+  ) {
+    await this.createFileFromBuffer(dirArg, fileName, Buffer.from(fileContent));
+  }
+
+  public static async createFileFromBuffer(
+    directoryRef: Directory,
+    fileName: string,
+    fileContent: Buffer
+  ) {
+    const filePath = plugins.path.join(directoryRef.getBasePath(), fileName);
+    const streamIntake = new plugins.streamfunction.Intake();
+    const putPromise = directoryRef.bucketRef.smartbucketRef.minioClient
+      .putObject(this.name, filePath, streamIntake.getReadable())
+      .catch(e => console.log(e));
+    streamIntake.pushData(fileContent);
+    streamIntake.signalEnd();
+    await putPromise;
+  }
+
+  // INSTANCE
+  public parentDirectoryRef: Directory;
+  public name: string;
+
+  public path: string;
+  public metaData: IFileMetaData;
+
+  constructor(directoryRefArg: Directory, fileName: string) {
+    this.parentDirectoryRef = directoryRefArg;
+    this.name = fileName;
+  }
+
+  public async getContentAsString() {
+    const fileBuffer = await this.getContentAsBuffer();
+    return fileBuffer.toString();
+  }
+
+  public async getContentAsBuffer() {
+    const done = plugins.smartpromise.defer();
+    const fileStream = await this.parentDirectoryRef.bucketRef.smartbucketRef.minioClient
+      .getObject(this.parentDirectoryRef.bucketRef.name, this.path)
+      .catch(e => console.log(e));
+    let completeFile = new Buffer('');
+    const duplexStream = plugins.streamfunction.createDuplexStream<Buffer, Buffer>(
+      async chunk => {
+        completeFile = Buffer.concat([chunk]);
+        return chunk;
+      },
+      async cb => {
+        done.resolve();
+        return Buffer.from('');
+      }
+    );
+
+    if (!fileStream) {
+      return null;
+    }
+
+    fileStream.pipe(duplexStream);
+    await done.promise;
+    return completeFile;
+  }
+
+  public async streamContent() {
+    throw new Error('not yet implemented');
+    // TODO
+  }
+
+  /**
+   * removes this file
+   */
+  public async remove() {
+    await this.parentDirectoryRef.bucketRef.smartbucketRef.minioClient.removeObject(
+      this.parentDirectoryRef.bucketRef.name,
+      this.path
+    );
+    await this.parentDirectoryRef.listFiles();
+  }
+}

ts/smartbucket.classes.smartbucket.ts

@@ -1,30 +1,41 @@
 import * as plugins from './smartbucket.plugins';
+import { Bucket } from './smartbucket.classes.bucket';
 
 export interface ISmartBucketConfig {
-  provider: 'google';
-  projectId: string;
-  bucketName: string;
+  endpoint: string;
+  accessKey: string;
+  accessSecret: string;
 }
 
 export class SmartBucket {
-  config: ISmartBucketConfig;
-  private _googleBucket;
+  public config: ISmartBucketConfig;
+
+  public minioClient: plugins.minio.Client;
+
   /**
    * the constructor of SmartBucket
    */
   constructor(configArg: ISmartBucketConfig) {
     this.config = configArg;
+    this.minioClient = new plugins.minio.Client({
+      endPoint: this.config.endpoint,
+      port: 443,
+      useSSL: true,
+      accessKey: this.config.accessKey,
+      secretKey: this.config.accessSecret
+    });
   }
 
-  /**
-   * initializes the Smartbucket
-   */
-  async init() {
-    if (this.config.provider === 'google') {
-      const storage = new plugins.googleCloudStorage.Storage({
-        projectId: this.config.projectId
-      });
-      storage.createBucket(this.config.bucketName, () => {});
-    }
+  public async createBucket(bucketNameArg: string) {
+    const bucket = await Bucket.createBucketByName(this, bucketNameArg);
+    return bucket;
+  }
+
+  public async removeBucket(bucketName: string) {
+    await Bucket.removeBucketByName(this, bucketName);
+  }
+
+  public async getBucketByName(bucketName: string) {
+    return Bucket.getBucketByName(this, bucketName);
   }
 }

ts/smartbucket.plugins.ts

@@ -1,9 +1,15 @@
-import * as smartpromise from '@pushrocks/smartpromise';
+// node native
+import * as path from 'path';
 
-export { smartpromise };
+export { path };
+
+import * as smartpath from '@pushrocks/smartpath';
+import * as smartpromise from '@pushrocks/smartpromise';
+import * as streamfunction from '@pushrocks/streamfunction';
+
+export { smartpath, smartpromise, streamfunction };
 
 // third party scope
-import * as googleCloudStorage from '@google-cloud/storage';
 import * as minio from 'minio';
 
-export { googleCloudStorage };
+export { minio };