1.0.38

.gitlab-ci.yml

@@ -36,6 +36,7 @@ auditProductionDependencies:
     - npmci command npm audit --audit-level=high --only=prod --production
   tags:
     - docker
+  allow_failure: true
 
 auditDevDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushrocks/smartnpm",
-  "version": "1.0.21",
+  "version": "1.0.38",
   "private": false,
   "description": "interface with npm to retrieve package information",
   "main": "dist_ts/index.js",
@@ -14,18 +14,21 @@
   "devDependencies": {
     "@gitzone/tsbuild": "^2.1.25",
     "@gitzone/tsrun": "^1.2.12",
-    "@gitzone/tstest": "^1.0.52",
+    "@gitzone/tstest": "^1.0.54",
     "@pushrocks/tapbundle": "^3.2.14",
-    "@types/node": "^14.14.41",
+    "@types/node": "^15.0.2",
     "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
     "@pushrocks/consolecolor": "^2.0.1",
+    "@pushrocks/levelcache": "^1.0.10",
     "@pushrocks/smartarchive": "^2.0.4",
-    "@pushrocks/smartfile": "^8.0.9",
-    "@pushrocks/smartpromise": "^3.1.3",
+    "@pushrocks/smartfile": "^8.0.10",
+    "@pushrocks/smartpromise": "^3.1.5",
     "@pushrocks/smartrequest": "^1.1.51",
+    "@pushrocks/smarttime": "^3.0.38",
+    "@pushrocks/smartversion": "^2.0.7",
     "package-json": "^6.5.0"
   },
   "files": [

test/test.ts

@@ -32,16 +32,25 @@ tap.test('should create a verdaccio registry', async () => {
 
 tap.test('should get package from verdaccio', async () => {
   const npmPackage = await verdaccioRegistry.getPackageInfo('@pushrocks/smartupdate');
-  console.log(npmPackage);
   expect(npmPackage.license).to.equal('MIT');
 });
 
 tap.test('should get a specific file from a package', async () => {
-  const bundleFile = await verdaccioRegistry.getFileFromPackage(
+  const wantedFile = await verdaccioRegistry.getFileFromPackage(
     '@pushrocks/websetup',
     'ts/index.ts'
   );
-  console.log(bundleFile.contentBuffer.toString());
+  console.log(wantedFile.contentBuffer.toString());
+});
+
+tap.test('should get a specific file from a package', async () => {
+  const wantedFiles = await verdaccioRegistry.getFilesFromPackage(
+    '@pushrocks/websetup',
+    'ts/'
+  );
+  for(const file of wantedFiles) {
+    console.log(file.path);
+  }
 });
 
 tap.start();

ts/smartnpm.classes.npmpackage.ts

@@ -1,13 +1,35 @@
 import * as plugins from './smartnpm.plugins';
 import { NpmRegistry } from './smartnpm.classes.npmregistry';
+import { PackageDisttag } from './smartnpm.classes.packagedisttag';
+import { PackageVersion, IVersionData } from './smartnpm.classes.packageversion';
 
 export class NpmPackage {
-  public static async createFromFullMetadata(
+  public static async createFromFullMetadataAndVersionData(
     npmRegistryArg: NpmRegistry,
-    fullMetadata: plugins.packageJson.FullMetadata
+    fullMetadataArg: plugins.packageJson.FullMetadata,
+    versionsDataArg: {
+      name: string;
+      'dist-tags': { [key: string]: string };
+      versions: { [key: string]: IVersionData };
+    }
   ) {
     const npmPackage = new NpmPackage(npmRegistryArg);
-    Object.assign(npmPackage, fullMetadata);
+    Object.assign(npmPackage, fullMetadataArg);
+    npmPackage.allVersions = [];
+    npmPackage.allDistTags = [];
+    for (const versionArg of Object.keys(versionsDataArg.versions)) {
+      const packageVersion = PackageVersion.createFromVersionData(
+        versionsDataArg.versions[versionArg]
+      );
+      npmPackage.allVersions.push(packageVersion);
+    }
+    for (const distTagArg of Object.keys(versionsDataArg['dist-tags'])) {
+      const packageDistTag = new PackageDisttag(
+        distTagArg,
+        versionsDataArg['dist-tags'][distTagArg]
+      );
+      npmPackage.allDistTags.push(packageDistTag);
+    }
     return npmPackage;
   }
 
@@ -15,6 +37,8 @@ export class NpmPackage {
   public name: string = null;
   public scope: string = null;
   public version: string = null;
+  public allVersions: PackageVersion[];
+  public allDistTags: PackageDisttag[];
   public description: string = null;
   public keywords: string[] = null;
   public date: string;
@@ -62,34 +86,102 @@ export class NpmPackage {
   }
 
   /**
-   * saves the package to memory
+   * saves the complete package to cache
    */
-  public async saveToMemory() {}
+  public async saveToCache() {}
 
   /**
-   * get file from package
+   * get files from package
    */
-  public async getFileFromPackage(filePath: string): Promise<plugins.smartfile.Smartfile> {
-    const done = plugins.smartpromise.defer<plugins.smartfile.Smartfile>();
+  public async getFilesFromPackage(
+    filePath: string,
+    optionsArg: {
+      distTag?: string;
+      version?: string;
+    },
+    returnOnFirstArg = false
+  ): Promise<plugins.smartfile.Smartfile[]> {
+    const done = plugins.smartpromise.defer<plugins.smartfile.Smartfile[]>();
     const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
-    const fileObservable = await smartarchiveInstance.extractArchiveFromUrlToObservable(
-      this.dist.tarball
-    );
+    let tarballUrl = this.dist?.tarball;
+    if (optionsArg?.version || optionsArg?.distTag) {
+      if (optionsArg.distTag && optionsArg.version) {
+        throw new Error('Please either specify version OR disttag, not both.');
+      }
+      let targetVersionString: string;
+      if (optionsArg.distTag) {
+        const targetDistTag = this.allDistTags.find((distTag) => {
+          return distTag.name === optionsArg.distTag;
+        });
+        if (targetDistTag) {
+          targetVersionString = targetDistTag.targetVersion;
+        }
+      } else {
+        targetVersionString = optionsArg.version;
+      }
+
+      // lets find the best matching release
+      const bestMatchingVersion = this.getBestMatchingVersion(targetVersionString);
+      if (!bestMatchingVersion) {
+        return null;
+      }
+      tarballUrl = this.allVersions.find(
+        (packageVersion) => packageVersion.version === bestMatchingVersion
+      ).dist.tarball;
+    }
+    const fileObservable = await smartarchiveInstance.extractArchiveFromUrlToObservable(tarballUrl);
     const wantedFilePath = plugins.path.join('package', filePath);
+    const allMatchingFiles: plugins.smartfile.Smartfile[] = [];
     const subscription = fileObservable.subscribe(
       (fileArg) => {
-        if (fileArg.path === wantedFilePath) {
-          done.resolve(fileArg);
+        // returnOnFirstArg requires exact match
+        if (returnOnFirstArg && fileArg.path === wantedFilePath) {
+          // lets resolve with the wanted file
+          done.resolve([fileArg]);
           subscription.unsubscribe();
+        } else if(!returnOnFirstArg && fileArg.path.startsWith(wantedFilePath)) {
+          allMatchingFiles.push(fileArg);
         }
       },
       (err) => {
         console.log(err);
       },
       () => {
+        done.resolve(allMatchingFiles);
         subscription.unsubscribe();
       }
     );
     return done.promise;
   }
+
+  /**
+   * get files from package
+   */
+   public async getFileFromPackage(
+    filePath: string,
+    optionsArg?: {
+      distTag?: string;
+      version?: string;
+    }
+  ): Promise<plugins.smartfile.Smartfile> {
+    const result = await this.getFilesFromPackage(filePath, optionsArg, true);
+    return result[0];
+  }
+
+  /**
+   * updates the package with information from the registry
+   */
+  update() {}
+
+  /** */
+  public getBestMatchingVersion(versionArg: string): string {
+    // lets find the best matching release
+    const targetVersion = plugins.smartversion.SmartVersion.fromFuzzyString(versionArg);
+    const versionStrings = this.allVersions.map((packageVersion) => packageVersion.version);
+    const bestMatchingVersion = targetVersion.getBestMatch(versionStrings);
+    if (!bestMatchingVersion) {
+      return null;
+    }
+    return bestMatchingVersion;
+  }
 }

ts/smartnpm.classes.npmregistry.ts

@@ -6,6 +6,7 @@ import { ISearchObject } from './smartnpm.interfaces';
 
 // classes
 import { NpmPackage } from './smartnpm.classes.npmpackage';
+import { ICacheDescriptor, RegistryCache } from './smartnpm.classes.registrycache';
 
 export interface INpmRegistryConstructorOptions {
   npmRegistryUrl?: string;
@@ -13,7 +14,7 @@ export interface INpmRegistryConstructorOptions {
 
 export class NpmRegistry {
   public options: INpmRegistryConstructorOptions;
-  public registry: string;
+  public registryCache: RegistryCache;
   private searchDomain = 'https://api.npms.io/v2/search?q=';
 
   constructor(optionsArg: INpmRegistryConstructorOptions = {}) {
@@ -24,6 +25,7 @@ export class NpmRegistry {
       ...defaultOptions,
       ...optionsArg,
     };
+    this.registryCache = new RegistryCache(this);
   }
 
   /**
@@ -34,8 +36,15 @@ export class NpmRegistry {
     const fullMetadata = await plugins.packageJson(packageName, {
       registryUrl: this.options.npmRegistryUrl,
       fullMetadata: true,
+    }).catch(err => {
+      console.log(err);
+      return null;
     });
-    const npmPackage = await NpmPackage.createFromFullMetadata(this, fullMetadata);
+    const versionData = await plugins.packageJson(packageName, {
+      registryUrl: this.options.npmRegistryUrl,
+      allVersions: true
+    });
+    const npmPackage = await NpmPackage.createFromFullMetadataAndVersionData(this, fullMetadata, versionData as any);
     return npmPackage;
   }
 
@@ -53,12 +62,61 @@ export class NpmRegistry {
   /**
    * gets a file from a package as Smartfile
    */
-  public async getFileFromPackage(packageNameArg: string, filePath: string): Promise<plugins.smartfile.Smartfile> {
+  public async getFileFromPackage(packageNameArg: string, filePathArg: string, optionsArg?: {
+    distTag?: string;
+    version?: string;
+  }): Promise<plugins.smartfile.Smartfile> {
+    // lets create a cache descriptor
+    const cacheDescriptor: ICacheDescriptor = {
+      registryUrl: this.options.npmRegistryUrl,
+      packageName: packageNameArg,
+      filePath: filePathArg,
+      distTag: optionsArg?.distTag,
+      version: optionsArg?.version
+    };
+
+    // lets see if we have something cached
+    const cachedFile: plugins.smartfile.Smartfile = await this.registryCache.getCachedFile(cacheDescriptor);
+    
+    // lets handle both occasions
+    if (!cachedFile) {
+      const npmPackage = await this.getPackageInfo(packageNameArg);
+      if (!optionsArg?.version && !optionsArg?.distTag) {
+        const latestAvailable = npmPackage.allDistTags.find(packageArg => packageArg.name === 'latest');
+        if (!latestAvailable) {
+          optionsArg = {
+            version: npmPackage.getBestMatchingVersion('*')
+          };
+        }
+      }
+      const fileResult = await npmPackage.getFileFromPackage(filePathArg, optionsArg);
+      this.registryCache.cacheSmartFile(cacheDescriptor, fileResult);
+      return fileResult;
+    } else {
+      return cachedFile;
+    }
+  }
+
+  public async getFilesFromPackage(packageNameArg: string, filePath: string, optionsArg?: {
+    distTag?: string;
+    version?: string;
+  }): Promise<plugins.smartfile.Smartfile[]> {
     const npmPackage = await this.getPackageInfo(packageNameArg);
-    return npmPackage.getFileFromPackage(filePath);
+    if (!optionsArg?.version && !optionsArg?.distTag) {
+      const latestAvailable = npmPackage.allDistTags.find(packageDistTagArg => packageDistTagArg.name === 'latest');
+      if (!latestAvailable) {
+        optionsArg = {
+          version: npmPackage.getBestMatchingVersion('*')
+        };
+      }
+    }
+    return npmPackage.getFilesFromPackage(filePath, optionsArg);
   }
 
   public async getPackageAsSmartfileVirtualDir(packageNameArg: string): Promise<plugins.smartfile.VirtualDirectory> {
+    /**
+     * TODO: rewrite as memory only
+     */
     const baseDir = plugins.path.join(paths.nogitDir, packageNameArg.replace('/', '__'));
     await plugins.smartfile.fs.ensureDir(baseDir);
     await this.savePackageToDisk(packageNameArg, baseDir);

ts/smartnpm.classes.packagedisttag.ts

@@ -0,0 +1,11 @@
+import * as plugins from './smartnpm.plugins';
+
+export class PackageDisttag {
+  name: string;
+  targetVersion: string;
+
+  constructor(nameArg: string, targetVersionArg: string) {
+    this.name = nameArg;
+    this.targetVersion = targetVersionArg;
+  }
+}

ts/smartnpm.classes.packageversion.ts

@@ -0,0 +1,31 @@
+import * as plugins from './smartnpm.plugins';
+
+export interface IVersionData {
+  name: string;
+  version: string;
+  dependencies: { [key: string]: string };
+  devDependencies: { [key: string]: string };
+  dist: {
+    integrity: string;
+    shasum: string;
+    tarball: string;
+  };
+}
+
+export class PackageVersion implements IVersionData {
+  public static createFromVersionData(versionDataArg: IVersionData) {
+    const packageVersion = new PackageVersion();
+    Object.assign(packageVersion, versionDataArg);
+    return packageVersion;
+  }
+
+  name: string;
+  version: string;
+  dependencies: { [key: string]: string };
+  devDependencies: { [key: string]: string };
+  dist: {
+    integrity: string;
+    shasum: string;
+    tarball: string;
+  };
+}

ts/smartnpm.classes.registrycache.ts

@@ -0,0 +1,50 @@
+import { NpmRegistry } from './smartnpm.classes.npmregistry';
+import * as plugins from './smartnpm.plugins';
+
+
+export interface ICacheDescriptor {
+  registryUrl: string;
+  packageName: string;
+  filePath: string;
+  distTag?: string;
+  version?: string;
+}
+
+export class RegistryCache {
+  npmregistryRef: NpmRegistry;
+  public levelCache: plugins.levelcache.LevelCache;
+
+  constructor(npmRegistryRefArg: NpmRegistry) {
+    this.npmregistryRef = npmRegistryRefArg;
+    this.levelCache = new plugins.levelcache.LevelCache({
+      cacheId: encodeURIComponent(this.npmregistryRef.options.npmRegistryUrl),
+      
+    });
+  }
+
+  public async getCachedFile (cacheDescriptorArg: ICacheDescriptor): Promise<plugins.smartfile.Smartfile> {
+    const cacheEntry = await this.levelCache.retrieveCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg));
+    if (cacheEntry) {
+      return plugins.smartfile.Smartfile.fromFoldedJson(cacheEntry.contents.toString());
+    }
+    return null;
+  }
+
+  public async cacheSmartFile (cacheDescriptorArg: ICacheDescriptor, smartfileArg: plugins.smartfile.Smartfile) {
+    if (smartfileArg && cacheDescriptorArg.version) {
+      await this.levelCache.storeCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg), new plugins.levelcache.CacheEntry({
+        contents: Buffer.from(smartfileArg.foldToJson()),
+        ttl: plugins.smarttime.getMilliSecondsFromUnits({hours: 1})
+      }));
+    } else {
+      await this.levelCache.storeCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg), new plugins.levelcache.CacheEntry({
+        contents: Buffer.from(smartfileArg.foldToJson()),
+        ttl: plugins.smarttime.getMilliSecondsFromUnits({minutes: 1})
+      }));
+    }
+  }
+
+  public getCacheDescriptorAsString(cacheDescriptorArg?: ICacheDescriptor) {
+    return `${cacheDescriptorArg.registryUrl}//+//${cacheDescriptorArg.packageName}//+//${cacheDescriptorArg.filePath}//+//${cacheDescriptorArg.distTag || cacheDescriptorArg.version}`;
+  }
+}

ts/smartnpm.plugins.ts

@@ -5,12 +5,15 @@ export { path };
 
 // @pushrocks scope
 import * as consolecolor from '@pushrocks/consolecolor';
+import * as levelcache from '@pushrocks/levelcache';
 import * as smartarchive from '@pushrocks/smartarchive';
 import * as smartfile from '@pushrocks/smartfile';
 import * as smartpromise from '@pushrocks/smartpromise';
 import * as smartrequest from '@pushrocks/smartrequest';
+import * as smartversion from '@pushrocks/smartversion';
+import * as smarttime from '@pushrocks/smarttime';
 
-export { consolecolor, smartarchive, smartfile, smartpromise, smartrequest };
+export { consolecolor, levelcache, smartarchive, smartfile, smartpromise, smartrequest, smartversion, smarttime };
 
 // third party scope
 import packageJson from 'package-json';