1.0.39

.gitlab-ci.yml

@@ -19,23 +19,36 @@ mirror:
   stage: security
   script:
     - npmci git mirror
+  only:
+    - tags
   tags:
     - lossless
     - docker
     - notpriv
 
-audit:
+auditProductionDependencies:
+  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  stage: security
+  script:
+    - npmci npm prepare
+    - npmci command npm install --production --ignore-scripts
+    - npmci command npm config set registry https://registry.npmjs.org
+    - npmci command npm audit --audit-level=high --only=prod --production
+  tags:
+    - docker
+  allow_failure: true
+
+auditDevDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
     - npmci npm prepare
     - npmci command npm install --ignore-scripts
     - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high
+    - npmci command npm audit --audit-level=high --only=dev
   tags:
-    - lossless
     - docker
-    - notpriv
+  allow_failure: true
 
 # ====================
 # test stage
@@ -50,9 +63,7 @@ testStable:
     - npmci npm test
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
-    - lossless
     - docker
-    - priv
 
 testBuild:
   stage: test
@@ -63,9 +74,7 @@ testBuild:
     - npmci command npm run build
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
-    - lossless
     - docker
-    - notpriv
 
 release:
   stage: release
@@ -85,6 +94,8 @@ release:
 codequality:
   stage: metadata
   allow_failure: true
+  only:
+    - tags
   script:
     - npmci command npm install -g tslint typescript
     - npmci npm prepare

.vscode/settings.json

@@ -15,7 +15,7 @@
             "properties": {
               "projectType": {
                 "type": "string",
-                "enum": ["website", "element", "service", "npm"]
+                "enum": ["website", "element", "service", "npm", "wcc"]
               }
             }
           }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushrocks/smartnpm",
-  "version": "1.0.16",
+  "version": "1.0.39",
   "private": false,
   "description": "interface with npm to retrieve package information",
   "main": "dist_ts/index.js",
@@ -12,19 +12,23 @@
     "build": "(tsbuild --web)"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.24",
+    "@gitzone/tsbuild": "^2.1.25",
     "@gitzone/tsrun": "^1.2.12",
-    "@gitzone/tstest": "^1.0.33",
+    "@gitzone/tstest": "^1.0.54",
-    "@pushrocks/tapbundle": "^3.2.1",
+    "@pushrocks/tapbundle": "^3.2.14",
-    "@types/node": "^14.0.14",
+    "@types/node": "^15.0.2",
-    "tslint": "^6.1.2",
+    "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
     "@pushrocks/consolecolor": "^2.0.1",
-    "@pushrocks/smartarchive": "^1.0.12",
+    "@pushrocks/levelcache": "^1.0.10",
-    "@pushrocks/smartfile": "^7.0.12",
+    "@pushrocks/smartarchive": "^2.0.4",
-    "@pushrocks/smartrequest": "^1.1.47",
+    "@pushrocks/smartfile": "^8.0.10",
+    "@pushrocks/smartpromise": "^3.1.5",
+    "@pushrocks/smartrequest": "^1.1.51",
+    "@pushrocks/smarttime": "^3.0.38",
+    "@pushrocks/smartversion": "^2.0.7",
     "package-json": "^6.5.0"
   },
   "files": [
@@ -38,5 +42,8 @@
     "cli.js",
     "npmextra.json",
     "readme.md"
+  ],
+  "browserslist": [
+    "last 1 chrome versions"
   ]
 }

readme.md

@@ -27,13 +27,6 @@ Platform support | [![Supports Windows 10](https://badgen.net/badge/supports%20W
 
 Use TypeScript for best in class instellisense.
 
-For further information read the linked docs at the top of this README.
-
-> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
-> | By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy.html)
-
-[![repo-footer](https://pushrocks.gitlab.io/assets/repo-footer.svg)](https://push.rocks)
-
 ## Contribution
 
 We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)

test/test.ts

@@ -17,7 +17,7 @@ tap.test('should create valid instances', async () => {
 
 tap.test('should produce a valid search string and this return npmts', async () => {
   const packages = await npmRegistry.searchOnNpm({
-    name: '@pushrocks/smartupdate'
+    name: '@pushrocks/smartupdate',
   });
   expect(packages[0].name).to.equal('@pushrocks/smartupdate');
 });
@@ -25,22 +25,40 @@ tap.test('should produce a valid search string and this return npmts', async ()
 // lets test things with the verdaccio registry
 tap.test('should create a verdaccio registry', async () => {
   verdaccioRegistry = new NpmRegistry({
-    npmRegistryUrl: 'https://verdaccio.lossless.one'
+    npmRegistryUrl: 'https://verdaccio.lossless.one',
   });
   expect(verdaccioRegistry).to.be.instanceOf(smartnpm.NpmRegistry);
 });
 
 tap.test('should get package from verdaccio', async () => {
   const npmPackage = await verdaccioRegistry.getPackageInfo('@pushrocks/smartupdate');
-  console.log(npmPackage);
   expect(npmPackage.license).to.equal('MIT');
 });
 
 tap.test('should get a specific file from a package', async () => {
-  const bundleFile = await verdaccioRegistry.getFileFromPackage(
+  const wantedFile = await verdaccioRegistry.getFileFromPackage(
     '@pushrocks/websetup',
-    'dist_bundle/bundle.js'
+    'ts/index.ts'
   );
+  console.log(wantedFile.contentBuffer.toString());
+});
+
+tap.test('should get a specific file from a package', async () => {
+  const wantedFiles = await verdaccioRegistry.getFilesFromPackage(
+    '@pushrocks/websetup',
+    'ts/'
+  );
+  for(const file of wantedFiles) {
+    console.log(file.path);
+  }
+});
+
+tap.test('should not get a nonexisting file from a package', async () => {
+  const wantedFileNotThere = await verdaccioRegistry.getFileFromPackage(
+    '@pushrocks/websetup',
+    'ts/notthere'
+  );
+  expect(wantedFileNotThere).to.be.null
 });
 
 tap.start();

ts/smartnpm.classes.npmpackage.ts

@@ -1,13 +1,35 @@
 import * as plugins from './smartnpm.plugins';
 import { NpmRegistry } from './smartnpm.classes.npmregistry';
+import { PackageDisttag } from './smartnpm.classes.packagedisttag';
+import { PackageVersion, IVersionData } from './smartnpm.classes.packageversion';
 
 export class NpmPackage {
-  public static async createFromFullMetadata(
+  public static async createFromFullMetadataAndVersionData(
     npmRegistryArg: NpmRegistry,
-    fullMetadata: plugins.packageJson.FullMetadata
+    fullMetadataArg: plugins.packageJson.FullMetadata,
+    versionsDataArg: {
+      name: string;
+      'dist-tags': { [key: string]: string };
+      versions: { [key: string]: IVersionData };
+    }
   ) {
     const npmPackage = new NpmPackage(npmRegistryArg);
-    Object.assign(npmPackage, fullMetadata);
+    Object.assign(npmPackage, fullMetadataArg);
+    npmPackage.allVersions = [];
+    npmPackage.allDistTags = [];
+    for (const versionArg of Object.keys(versionsDataArg.versions)) {
+      const packageVersion = PackageVersion.createFromVersionData(
+        versionsDataArg.versions[versionArg]
+      );
+      npmPackage.allVersions.push(packageVersion);
+    }
+    for (const distTagArg of Object.keys(versionsDataArg['dist-tags'])) {
+      const packageDistTag = new PackageDisttag(
+        distTagArg,
+        versionsDataArg['dist-tags'][distTagArg]
+      );
+      npmPackage.allDistTags.push(packageDistTag);
+    }
     return npmPackage;
   }
 
@@ -15,6 +37,8 @@ export class NpmPackage {
   public name: string = null;
   public scope: string = null;
   public version: string = null;
+  public allVersions: PackageVersion[];
+  public allDistTags: PackageDisttag[];
   public description: string = null;
   public keywords: string[] = null;
   public date: string;
@@ -48,9 +72,9 @@ export class NpmPackage {
   } = null;
   public searchScore: number = null;
 
-  public npmRegistry: NpmRegistry;
+  public npmRegistryRef: NpmRegistry;
   constructor(npmRegistryArg: NpmRegistry) {
-    this.npmRegistry = npmRegistryArg;
+    this.npmRegistryRef = npmRegistryArg;
   }
 
   /**
@@ -58,6 +82,106 @@ export class NpmPackage {
    */
   public async saveToDisk(targetDir: string) {
     const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
-    await smartarchiveInstance.extractArchiveFromUrl(this.dist.tarball, targetDir);
+    await smartarchiveInstance.extractArchiveFromUrlToFs(this.dist.tarball, targetDir);
+  }
+
+  /**
+   * saves the complete package to cache
+   */
+  public async saveToCache() {}
+
+  /**
+   * get files from package
+   */
+  public async getFilesFromPackage(
+    filePath: string,
+    optionsArg: {
+      distTag?: string;
+      version?: string;
+    },
+    returnOnFirstArg = false
+  ): Promise<plugins.smartfile.Smartfile[]> {
+    const done = plugins.smartpromise.defer<plugins.smartfile.Smartfile[]>();
+    const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
+    let tarballUrl = this.dist?.tarball;
+    if (optionsArg?.version || optionsArg?.distTag) {
+      if (optionsArg.distTag && optionsArg.version) {
+        throw new Error('Please either specify version OR disttag, not both.');
+      }
+      let targetVersionString: string;
+      if (optionsArg.distTag) {
+        const targetDistTag = this.allDistTags.find((distTag) => {
+          return distTag.name === optionsArg.distTag;
+        });
+        if (targetDistTag) {
+          targetVersionString = targetDistTag.targetVersion;
+        }
+      } else {
+        targetVersionString = optionsArg.version;
+      }
+
+      // lets find the best matching release
+      const bestMatchingVersion = this.getBestMatchingVersion(targetVersionString);
+      if (!bestMatchingVersion) {
+        return null;
+      }
+      tarballUrl = this.allVersions.find(
+        (packageVersion) => packageVersion.version === bestMatchingVersion
+      ).dist.tarball;
+    }
+    const fileObservable = await smartarchiveInstance.extractArchiveFromUrlToObservable(tarballUrl);
+    const wantedFilePath = plugins.path.join('package', filePath);
+    const allMatchingFiles: plugins.smartfile.Smartfile[] = [];
+    const subscription = fileObservable.subscribe(
+      (fileArg) => {
+        // returnOnFirstArg requires exact match
+        if (returnOnFirstArg && fileArg.path === wantedFilePath) {
+          // lets resolve with the wanted file
+          done.resolve([fileArg]);
+          subscription.unsubscribe();
+        } else if(!returnOnFirstArg && fileArg.path.startsWith(wantedFilePath)) {
+          allMatchingFiles.push(fileArg);
+        }
+      },
+      (err) => {
+        console.log(err);
+      },
+      () => {
+        done.resolve(allMatchingFiles);
+        subscription.unsubscribe();
+      }
+    );
+    return done.promise;
+  }
+
+  /**
+   * get files from package
+   */
+   public async getFileFromPackage(
+    filePath: string,
+    optionsArg?: {
+      distTag?: string;
+      version?: string;
+    }
+  ): Promise<plugins.smartfile.Smartfile> {
+    const result = await this.getFilesFromPackage(filePath, optionsArg, true);
+    return result[0] || null;
+  }
+
+  /**
+   * updates the package with information from the registry
+   */
+  update() {}
+
+  /** */
+  public getBestMatchingVersion(versionArg: string): string {
+    // lets find the best matching release
+    const targetVersion = plugins.smartversion.SmartVersion.fromFuzzyString(versionArg);
+    const versionStrings = this.allVersions.map((packageVersion) => packageVersion.version);
+    const bestMatchingVersion = targetVersion.getBestMatch(versionStrings);
+    if (!bestMatchingVersion) {
+      return null;
+    }
+    return bestMatchingVersion;
   }
 }

ts/smartnpm.classes.npmregistry.ts

@@ -6,6 +6,7 @@ import { ISearchObject } from './smartnpm.interfaces';
 
 // classes
 import { NpmPackage } from './smartnpm.classes.npmpackage';
+import { ICacheDescriptor, RegistryCache } from './smartnpm.classes.registrycache';
 
 export interface INpmRegistryConstructorOptions {
   npmRegistryUrl?: string;
@@ -13,7 +14,7 @@ export interface INpmRegistryConstructorOptions {
 
 export class NpmRegistry {
   public options: INpmRegistryConstructorOptions;
-  public registry: string;
+  public registryCache: RegistryCache;
   private searchDomain = 'https://api.npms.io/v2/search?q=';
 
   constructor(optionsArg: INpmRegistryConstructorOptions = {}) {
@@ -24,6 +25,7 @@ export class NpmRegistry {
       ...defaultOptions,
       ...optionsArg,
     };
+    this.registryCache = new RegistryCache(this);
   }
 
   /**
@@ -34,8 +36,15 @@ export class NpmRegistry {
     const fullMetadata = await plugins.packageJson(packageName, {
       registryUrl: this.options.npmRegistryUrl,
       fullMetadata: true,
+    }).catch(err => {
+      console.log(err);
+      return null;
     });
-    const npmPackage = await NpmPackage.createFromFullMetadata(this, fullMetadata);
+    const versionData = await plugins.packageJson(packageName, {
+      registryUrl: this.options.npmRegistryUrl,
+      allVersions: true
+    });
+    const npmPackage = await NpmPackage.createFromFullMetadataAndVersionData(this, fullMetadata, versionData as any);
     return npmPackage;
   }
 
@@ -45,6 +54,7 @@ export class NpmRegistry {
    * @param targetDir
    */
   public async savePackageToDisk(packageName: string, targetDir: string): Promise<void> {
+    plugins.smartfile.fs.ensureDirSync(paths.nogitDir);
     const npmPackage = await this.getPackageInfo(packageName);
     await npmPackage.saveToDisk(targetDir);
   }
@@ -52,15 +62,69 @@ export class NpmRegistry {
   /**
    * gets a file from a package as Smartfile
    */
-  public async getFileFromPackage(packageName: string, filePath: string) {
+  public async getFileFromPackage(packageNameArg: string, filePathArg: string, optionsArg?: {
-    const baseDir = plugins.path.join(paths.nogitDir, packageName.replace('/', '__'));
+    distTag?: string;
+    version?: string;
+  }): Promise<plugins.smartfile.Smartfile> {
+    // lets create a cache descriptor
+    const cacheDescriptor: ICacheDescriptor = {
+      registryUrl: this.options.npmRegistryUrl,
+      packageName: packageNameArg,
+      filePath: filePathArg,
+      distTag: optionsArg?.distTag,
+      version: optionsArg?.version
+    };
+
+    // lets see if we have something cached
+    const cachedFile: plugins.smartfile.Smartfile = await this.registryCache.getCachedFile(cacheDescriptor);
+    
+    // lets handle both occasions
+    if (!cachedFile) {
+      const npmPackage = await this.getPackageInfo(packageNameArg);
+      if (!optionsArg?.version && !optionsArg?.distTag) {
+        const latestAvailable = npmPackage.allDistTags.find(packageArg => packageArg.name === 'latest');
+        if (!latestAvailable) {
+          optionsArg = {
+            version: npmPackage.getBestMatchingVersion('*')
+          };
+        }
+      }
+      const fileResult = await npmPackage.getFileFromPackage(filePathArg, optionsArg);
+      if (fileResult) {
+        this.registryCache.cacheSmartFile(cacheDescriptor, fileResult);
+      }
+      return fileResult;
+    } else {
+      return cachedFile;
+    }
+  }
+
+  public async getFilesFromPackage(packageNameArg: string, filePath: string, optionsArg?: {
+    distTag?: string;
+    version?: string;
+  }): Promise<plugins.smartfile.Smartfile[]> {
+    const npmPackage = await this.getPackageInfo(packageNameArg);
+    if (!optionsArg?.version && !optionsArg?.distTag) {
+      const latestAvailable = npmPackage.allDistTags.find(packageDistTagArg => packageDistTagArg.name === 'latest');
+      if (!latestAvailable) {
+        optionsArg = {
+          version: npmPackage.getBestMatchingVersion('*')
+        };
+      }
+    }
+    return npmPackage.getFilesFromPackage(filePath, optionsArg);
+  }
+
+  public async getPackageAsSmartfileVirtualDir(packageNameArg: string): Promise<plugins.smartfile.VirtualDirectory> {
+    /**
+     * TODO: rewrite as memory only
+     */
+    const baseDir = plugins.path.join(paths.nogitDir, packageNameArg.replace('/', '__'));
     await plugins.smartfile.fs.ensureDir(baseDir);
-    await this.savePackageToDisk(packageName, baseDir);
+    await this.savePackageToDisk(packageNameArg, baseDir);
-    const smartfile = await plugins.smartfile.Smartfile.fromFilePath(
+    const virtualDir = await plugins.smartfile.VirtualDirectory.fromFsDirPath(baseDir);
-      plugins.path.join(baseDir, 'package', filePath)
-    );
     await plugins.smartfile.fs.remove(baseDir);
-    return smartfile;
+    return virtualDir;
   }
 
   /**

ts/smartnpm.classes.packagedisttag.ts

@@ -0,0 +1,11 @@
+import * as plugins from './smartnpm.plugins';
+
+export class PackageDisttag {
+  name: string;
+  targetVersion: string;
+
+  constructor(nameArg: string, targetVersionArg: string) {
+    this.name = nameArg;
+    this.targetVersion = targetVersionArg;
+  }
+}

ts/smartnpm.classes.packageversion.ts

@@ -0,0 +1,31 @@
+import * as plugins from './smartnpm.plugins';
+
+export interface IVersionData {
+  name: string;
+  version: string;
+  dependencies: { [key: string]: string };
+  devDependencies: { [key: string]: string };
+  dist: {
+    integrity: string;
+    shasum: string;
+    tarball: string;
+  };
+}
+
+export class PackageVersion implements IVersionData {
+  public static createFromVersionData(versionDataArg: IVersionData) {
+    const packageVersion = new PackageVersion();
+    Object.assign(packageVersion, versionDataArg);
+    return packageVersion;
+  }
+
+  name: string;
+  version: string;
+  dependencies: { [key: string]: string };
+  devDependencies: { [key: string]: string };
+  dist: {
+    integrity: string;
+    shasum: string;
+    tarball: string;
+  };
+}

ts/smartnpm.classes.registrycache.ts

@@ -0,0 +1,50 @@
+import { NpmRegistry } from './smartnpm.classes.npmregistry';
+import * as plugins from './smartnpm.plugins';
+
+
+export interface ICacheDescriptor {
+  registryUrl: string;
+  packageName: string;
+  filePath: string;
+  distTag?: string;
+  version?: string;
+}
+
+export class RegistryCache {
+  npmregistryRef: NpmRegistry;
+  public levelCache: plugins.levelcache.LevelCache;
+
+  constructor(npmRegistryRefArg: NpmRegistry) {
+    this.npmregistryRef = npmRegistryRefArg;
+    this.levelCache = new plugins.levelcache.LevelCache({
+      cacheId: encodeURIComponent(this.npmregistryRef.options.npmRegistryUrl),
+      
+    });
+  }
+
+  public async getCachedFile (cacheDescriptorArg: ICacheDescriptor): Promise<plugins.smartfile.Smartfile> {
+    const cacheEntry = await this.levelCache.retrieveCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg));
+    if (cacheEntry) {
+      return plugins.smartfile.Smartfile.fromFoldedJson(cacheEntry.contents.toString());
+    }
+    return null;
+  }
+
+  public async cacheSmartFile (cacheDescriptorArg: ICacheDescriptor, smartfileArg: plugins.smartfile.Smartfile) {
+    if (smartfileArg && cacheDescriptorArg.version) {
+      await this.levelCache.storeCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg), new plugins.levelcache.CacheEntry({
+        contents: Buffer.from(smartfileArg.foldToJson()),
+        ttl: plugins.smarttime.getMilliSecondsFromUnits({hours: 1})
+      }));
+    } else {
+      await this.levelCache.storeCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg), new plugins.levelcache.CacheEntry({
+        contents: Buffer.from(smartfileArg.foldToJson()),
+        ttl: plugins.smarttime.getMilliSecondsFromUnits({minutes: 1})
+      }));
+    }
+  }
+
+  public getCacheDescriptorAsString(cacheDescriptorArg?: ICacheDescriptor) {
+    return `${cacheDescriptorArg.registryUrl}//+//${cacheDescriptorArg.packageName}//+//${cacheDescriptorArg.filePath}//+//${cacheDescriptorArg.distTag || cacheDescriptorArg.version}`;
+  }
+}

ts/smartnpm.paths.ts

@@ -2,4 +2,3 @@ import * as plugins from './smartnpm.plugins';
 
 export const packageDir = plugins.path.join(__dirname, '../');
 export const nogitDir = plugins.path.join(packageDir, '.nogit/');
-plugins.smartfile.fs.ensureDirSync(nogitDir);

ts/smartnpm.plugins.ts

@@ -5,11 +5,15 @@ export { path };
 
 // @pushrocks scope
 import * as consolecolor from '@pushrocks/consolecolor';
+import * as levelcache from '@pushrocks/levelcache';
 import * as smartarchive from '@pushrocks/smartarchive';
 import * as smartfile from '@pushrocks/smartfile';
+import * as smartpromise from '@pushrocks/smartpromise';
 import * as smartrequest from '@pushrocks/smartrequest';
+import * as smartversion from '@pushrocks/smartversion';
+import * as smarttime from '@pushrocks/smarttime';
 
-export { consolecolor, smartarchive, smartfile, smartrequest };
+export { consolecolor, levelcache, smartarchive, smartfile, smartpromise, smartrequest, smartversion, smarttime };
 
 // third party scope
 import packageJson from 'package-json';