1.0.37

.gitlab-ci.yml

@@ -19,23 +19,35 @@ mirror:
   stage: security
   script:
     - npmci git mirror
+  only:
+    - tags
   tags:
     - lossless
     - docker
     - notpriv
 
-audit:
+auditProductionDependencies:
+  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  stage: security
+  script:
+    - npmci npm prepare
+    - npmci command npm install --production --ignore-scripts
+    - npmci command npm config set registry https://registry.npmjs.org
+    - npmci command npm audit --audit-level=high --only=prod --production
+  tags:
+    - docker
+
+auditDevDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
     - npmci npm prepare
     - npmci command npm install --ignore-scripts
     - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high
+    - npmci command npm audit --audit-level=high --only=dev
   tags:
-    - lossless
     - docker
-    - notpriv
+  allow_failure: true
 
 # ====================
 # test stage
@@ -50,9 +62,7 @@ testStable:
     - npmci npm test
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
-    - lossless
     - docker
-    - priv
 
 testBuild:
   stage: test
@@ -63,9 +73,7 @@ testBuild:
     - npmci command npm run build
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
-    - lossless
     - docker
-    - notpriv
 
 release:
   stage: release
@@ -85,6 +93,8 @@ release:
 codequality:
   stage: metadata
   allow_failure: true
+  only:
+    - tags
   script:
     - npmci command npm install -g tslint typescript
     - npmci npm prepare

.vscode/settings.json

@@ -15,7 +15,7 @@
             "properties": {
               "projectType": {
                 "type": "string",
-                "enum": ["website", "element", "service", "npm"]
+                "enum": ["website", "element", "service", "npm", "wcc"]
               }
             }
           }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushrocks/smartnpm",
-  "version": "1.0.17",
+  "version": "1.0.37",
   "private": false,
   "description": "interface with npm to retrieve package information",
   "main": "dist_ts/index.js",
@@ -12,19 +12,23 @@
     "build": "(tsbuild --web)"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.24",
+    "@gitzone/tsbuild": "^2.1.25",
     "@gitzone/tsrun": "^1.2.12",
-    "@gitzone/tstest": "^1.0.33",
+    "@gitzone/tstest": "^1.0.54",
-    "@pushrocks/tapbundle": "^3.2.1",
+    "@pushrocks/tapbundle": "^3.2.14",
-    "@types/node": "^14.0.14",
+    "@types/node": "^15.0.2",
-    "tslint": "^6.1.2",
+    "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
     "@pushrocks/consolecolor": "^2.0.1",
-    "@pushrocks/smartarchive": "^1.0.12",
+    "@pushrocks/levelcache": "^1.0.10",
-    "@pushrocks/smartfile": "^7.0.12",
+    "@pushrocks/smartarchive": "^2.0.4",
-    "@pushrocks/smartrequest": "^1.1.47",
+    "@pushrocks/smartfile": "^8.0.10",
+    "@pushrocks/smartpromise": "^3.1.5",
+    "@pushrocks/smartrequest": "^1.1.51",
+    "@pushrocks/smarttime": "^3.0.38",
+    "@pushrocks/smartversion": "^2.0.7",
     "package-json": "^6.5.0"
   },
   "files": [
@@ -38,5 +42,8 @@
     "cli.js",
     "npmextra.json",
     "readme.md"
+  ],
+  "browserslist": [
+    "last 1 chrome versions"
   ]
 }

test/test.ts

@@ -17,7 +17,7 @@ tap.test('should create valid instances', async () => {
 
 tap.test('should produce a valid search string and this return npmts', async () => {
   const packages = await npmRegistry.searchOnNpm({
-    name: '@pushrocks/smartupdate'
+    name: '@pushrocks/smartupdate',
   });
   expect(packages[0].name).to.equal('@pushrocks/smartupdate');
 });
@@ -25,22 +25,32 @@ tap.test('should produce a valid search string and this return npmts', async ()
 // lets test things with the verdaccio registry
 tap.test('should create a verdaccio registry', async () => {
   verdaccioRegistry = new NpmRegistry({
-    npmRegistryUrl: 'https://verdaccio.lossless.one'
+    npmRegistryUrl: 'https://verdaccio.lossless.one',
   });
   expect(verdaccioRegistry).to.be.instanceOf(smartnpm.NpmRegistry);
 });
 
 tap.test('should get package from verdaccio', async () => {
   const npmPackage = await verdaccioRegistry.getPackageInfo('@pushrocks/smartupdate');
-  console.log(npmPackage);
   expect(npmPackage.license).to.equal('MIT');
 });
 
 tap.test('should get a specific file from a package', async () => {
-  const bundleFile = await verdaccioRegistry.getFileFromPackage(
+  const wantedFile = await verdaccioRegistry.getFileFromPackage(
     '@pushrocks/websetup',
-    'dist_bundle/bundle.js'
+    'ts/index.ts'
   );
+  console.log(wantedFile.contentBuffer.toString());
+});
+
+tap.test('should get a specific file from a package', async () => {
+  const wantedFiles = await verdaccioRegistry.getFilesFromPackage(
+    '@pushrocks/websetup',
+    'ts/'
+  );
+  for(const file of wantedFiles) {
+    console.log(file.path);
+  }
 });
 
 tap.start();

ts/smartnpm.classes.npmpackage.ts

@@ -1,13 +1,35 @@
 import * as plugins from './smartnpm.plugins';
 import { NpmRegistry } from './smartnpm.classes.npmregistry';
+import { PackageDisttag } from './smartnpm.classes.packagedisttag';
+import { PackageVersion, IVersionData } from './smartnpm.classes.packageversion';
 
 export class NpmPackage {
-  public static async createFromFullMetadata(
+  public static async createFromFullMetadataAndVersionData(
     npmRegistryArg: NpmRegistry,
-    fullMetadata: plugins.packageJson.FullMetadata
+    fullMetadataArg: plugins.packageJson.FullMetadata,
+    versionsDataArg: {
+      name: string;
+      'dist-tags': { [key: string]: string };
+      versions: { [key: string]: IVersionData };
+    }
   ) {
     const npmPackage = new NpmPackage(npmRegistryArg);
-    Object.assign(npmPackage, fullMetadata);
+    Object.assign(npmPackage, fullMetadataArg);
+    npmPackage.allVersions = [];
+    npmPackage.allDistTags = [];
+    for (const versionArg of Object.keys(versionsDataArg.versions)) {
+      const packageVersion = PackageVersion.createFromVersionData(
+        versionsDataArg.versions[versionArg]
+      );
+      npmPackage.allVersions.push(packageVersion);
+    }
+    for (const distTagArg of Object.keys(versionsDataArg['dist-tags'])) {
+      const packageDistTag = new PackageDisttag(
+        distTagArg,
+        versionsDataArg['dist-tags'][distTagArg]
+      );
+      npmPackage.allDistTags.push(packageDistTag);
+    }
     return npmPackage;
   }
 
@@ -15,6 +37,8 @@ export class NpmPackage {
   public name: string = null;
   public scope: string = null;
   public version: string = null;
+  public allVersions: PackageVersion[];
+  public allDistTags: PackageDisttag[];
   public description: string = null;
   public keywords: string[] = null;
   public date: string;
@@ -48,9 +72,9 @@ export class NpmPackage {
   } = null;
   public searchScore: number = null;
 
-  public npmRegistry: NpmRegistry;
+  public npmRegistryRef: NpmRegistry;
   constructor(npmRegistryArg: NpmRegistry) {
-    this.npmRegistry = npmRegistryArg;
+    this.npmRegistryRef = npmRegistryArg;
   }
 
   /**
@@ -58,6 +82,106 @@ export class NpmPackage {
    */
   public async saveToDisk(targetDir: string) {
     const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
-    await smartarchiveInstance.extractArchiveFromUrl(this.dist.tarball, targetDir);
+    await smartarchiveInstance.extractArchiveFromUrlToFs(this.dist.tarball, targetDir);
+  }
+
+  /**
+   * saves the complete package to cache
+   */
+  public async saveToCache() {}
+
+  /**
+   * get files from package
+   */
+  public async getFilesFromPackage(
+    filePath: string,
+    optionsArg: {
+      distTag?: string;
+      version?: string;
+    },
+    returnOnFirstArg = false
+  ): Promise<plugins.smartfile.Smartfile[]> {
+    const done = plugins.smartpromise.defer<plugins.smartfile.Smartfile[]>();
+    const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
+    let tarballUrl = this.dist?.tarball;
+    if (optionsArg?.version || optionsArg?.distTag) {
+      if (optionsArg.distTag && optionsArg.version) {
+        throw new Error('Please either specify version OR disttag, not both.');
+      }
+      let targetVersionString: string;
+      if (optionsArg.distTag) {
+        const targetDistTag = this.allDistTags.find((distTag) => {
+          return distTag.name === optionsArg.distTag;
+        });
+        if (targetDistTag) {
+          targetVersionString = targetDistTag.targetVersion;
+        }
+      } else {
+        targetVersionString = optionsArg.version;
+      }
+
+      // lets find the best matching release
+      const bestMatchingVersion = this.getBestMatchingVersion(targetVersionString);
+      if (!bestMatchingVersion) {
+        return null;
+      }
+      tarballUrl = this.allVersions.find(
+        (packageVersion) => packageVersion.version === bestMatchingVersion
+      ).dist.tarball;
+    }
+    const fileObservable = await smartarchiveInstance.extractArchiveFromUrlToObservable(tarballUrl);
+    const wantedFilePath = plugins.path.join('package', filePath);
+    const allMatchingFiles: plugins.smartfile.Smartfile[] = [];
+    const subscription = fileObservable.subscribe(
+      (fileArg) => {
+        // returnOnFirstArg requires exact match
+        if (returnOnFirstArg && fileArg.path === wantedFilePath) {
+          // lets resolve with the wanted file
+          done.resolve([fileArg]);
+          subscription.unsubscribe();
+        } else if(!returnOnFirstArg && fileArg.path.startsWith(wantedFilePath)) {
+          allMatchingFiles.push(fileArg);
+        }
+      },
+      (err) => {
+        console.log(err);
+      },
+      () => {
+        done.resolve(allMatchingFiles);
+        subscription.unsubscribe();
+      }
+    );
+    return done.promise;
+  }
+
+  /**
+   * get files from package
+   */
+   public async getFileFromPackage(
+    filePath: string,
+    optionsArg?: {
+      distTag?: string;
+      version?: string;
+    }
+  ): Promise<plugins.smartfile.Smartfile> {
+    const result = await this.getFilesFromPackage(filePath, optionsArg, true);
+    return result[0];
+  }
+
+  /**
+   * updates the package with information from the registry
+   */
+  update() {}
+
+  /** */
+  public getBestMatchingVersion(versionArg: string): string {
+    // lets find the best matching release
+    const targetVersion = plugins.smartversion.SmartVersion.fromFuzzyString(versionArg);
+    const versionStrings = this.allVersions.map((packageVersion) => packageVersion.version);
+    const bestMatchingVersion = targetVersion.getBestMatch(versionStrings);
+    if (!bestMatchingVersion) {
+      return null;
+    }
+    return bestMatchingVersion;
   }
 }

ts/smartnpm.classes.npmregistry.ts

@@ -6,6 +6,7 @@ import { ISearchObject } from './smartnpm.interfaces';
 
 // classes
 import { NpmPackage } from './smartnpm.classes.npmpackage';
+import { ICacheDescriptor, RegistryCache } from './smartnpm.classes.registrycache';
 
 export interface INpmRegistryConstructorOptions {
   npmRegistryUrl?: string;
@@ -13,7 +14,7 @@ export interface INpmRegistryConstructorOptions {
 
 export class NpmRegistry {
   public options: INpmRegistryConstructorOptions;
-  public registry: string;
+  public registryCache: RegistryCache;
   private searchDomain = 'https://api.npms.io/v2/search?q=';
 
   constructor(optionsArg: INpmRegistryConstructorOptions = {}) {
@@ -24,6 +25,7 @@ export class NpmRegistry {
       ...defaultOptions,
       ...optionsArg,
     };
+    this.registryCache = new RegistryCache(this);
   }
 
   /**
@@ -34,8 +36,15 @@ export class NpmRegistry {
     const fullMetadata = await plugins.packageJson(packageName, {
       registryUrl: this.options.npmRegistryUrl,
       fullMetadata: true,
+    }).catch(err => {
+      console.log(err);
+      return null;
     });
-    const npmPackage = await NpmPackage.createFromFullMetadata(this, fullMetadata);
+    const versionData = await plugins.packageJson(packageName, {
+      registryUrl: this.options.npmRegistryUrl,
+      allVersions: true
+    });
+    const npmPackage = await NpmPackage.createFromFullMetadataAndVersionData(this, fullMetadata, versionData as any);
     return npmPackage;
   }
 
@@ -45,6 +54,7 @@ export class NpmRegistry {
    * @param targetDir
    */
   public async savePackageToDisk(packageName: string, targetDir: string): Promise<void> {
+    plugins.smartfile.fs.ensureDirSync(paths.nogitDir);
     const npmPackage = await this.getPackageInfo(packageName);
     await npmPackage.saveToDisk(targetDir);
   }
@@ -52,15 +62,67 @@ export class NpmRegistry {
   /**
    * gets a file from a package as Smartfile
    */
-  public async getFileFromPackage(packageName: string, filePath: string) {
+  public async getFileFromPackage(packageNameArg: string, filePathArg: string, optionsArg?: {
-    const baseDir = plugins.path.join(paths.nogitDir, packageName.replace('/', '__'));
+    distTag?: string;
+    version?: string;
+  }): Promise<plugins.smartfile.Smartfile> {
+    // lets create a cache descriptor
+    const cacheDescriptor: ICacheDescriptor = {
+      registryUrl: this.options.npmRegistryUrl,
+      packageName: packageNameArg,
+      filePath: filePathArg,
+      distTag: optionsArg?.distTag,
+      version: optionsArg?.version
+    };
+
+    // lets see if we have something cached
+    const cachedFile: plugins.smartfile.Smartfile = await this.registryCache.getCachedFile(cacheDescriptor);
+    
+    // lets handle both occasions
+    if (!cachedFile) {
+      const npmPackage = await this.getPackageInfo(packageNameArg);
+      if (!optionsArg?.version && !optionsArg?.distTag) {
+        const latestAvailable = npmPackage.allDistTags.find(packageArg => packageArg.name === 'latest');
+        if (!latestAvailable) {
+          optionsArg = {
+            version: npmPackage.getBestMatchingVersion('*')
+          };
+        }
+      }
+      const fileResult = await npmPackage.getFileFromPackage(filePathArg, optionsArg);
+      this.registryCache.cacheSmartFile(cacheDescriptor, fileResult);
+      return fileResult;
+    } else {
+      return cachedFile;
+    }
+  }
+
+  public async getFilesFromPackage(packageNameArg: string, filePath: string, optionsArg?: {
+    distTag?: string;
+    version?: string;
+  }): Promise<plugins.smartfile.Smartfile[]> {
+    const npmPackage = await this.getPackageInfo(packageNameArg);
+    if (!optionsArg?.version && !optionsArg?.distTag) {
+      const latestAvailable = npmPackage.allDistTags.find(packageDistTagArg => packageDistTagArg.name === 'latest');
+      if (!latestAvailable) {
+        optionsArg = {
+          version: npmPackage.getBestMatchingVersion('*')
+        };
+      }
+    }
+    return npmPackage.getFilesFromPackage(filePath, optionsArg);
+  }
+
+  public async getPackageAsSmartfileVirtualDir(packageNameArg: string): Promise<plugins.smartfile.VirtualDirectory> {
+    /**
+     * TODO: rewrite as memory only
+     */
+    const baseDir = plugins.path.join(paths.nogitDir, packageNameArg.replace('/', '__'));
     await plugins.smartfile.fs.ensureDir(baseDir);
-    await this.savePackageToDisk(packageName, baseDir);
+    await this.savePackageToDisk(packageNameArg, baseDir);
-    const smartfile = await plugins.smartfile.Smartfile.fromFilePath(
+    const virtualDir = await plugins.smartfile.VirtualDirectory.fromFsDirPath(baseDir);
-      plugins.path.join(baseDir, 'package', filePath)
-    );
     await plugins.smartfile.fs.remove(baseDir);
-    return smartfile;
+    return virtualDir;
   }
 
   /**

ts/smartnpm.classes.packagedisttag.ts

@@ -0,0 +1,11 @@
+import * as plugins from './smartnpm.plugins';
+
+export class PackageDisttag {
+  name: string;
+  targetVersion: string;
+
+  constructor(nameArg: string, targetVersionArg: string) {
+    this.name = nameArg;
+    this.targetVersion = targetVersionArg;
+  }
+}

ts/smartnpm.classes.packageversion.ts

@@ -0,0 +1,31 @@
+import * as plugins from './smartnpm.plugins';
+
+export interface IVersionData {
+  name: string;
+  version: string;
+  dependencies: { [key: string]: string };
+  devDependencies: { [key: string]: string };
+  dist: {
+    integrity: string;
+    shasum: string;
+    tarball: string;
+  };
+}
+
+export class PackageVersion implements IVersionData {
+  public static createFromVersionData(versionDataArg: IVersionData) {
+    const packageVersion = new PackageVersion();
+    Object.assign(packageVersion, versionDataArg);
+    return packageVersion;
+  }
+
+  name: string;
+  version: string;
+  dependencies: { [key: string]: string };
+  devDependencies: { [key: string]: string };
+  dist: {
+    integrity: string;
+    shasum: string;
+    tarball: string;
+  };
+}

ts/smartnpm.classes.registrycache.ts

@@ -0,0 +1,50 @@
+import { NpmRegistry } from './smartnpm.classes.npmregistry';
+import * as plugins from './smartnpm.plugins';
+
+
+export interface ICacheDescriptor {
+  registryUrl: string;
+  packageName: string;
+  filePath: string;
+  distTag?: string;
+  version?: string;
+}
+
+export class RegistryCache {
+  npmregistryRef: NpmRegistry;
+  public levelCache: plugins.levelcache.LevelCache;
+
+  constructor(npmRegistryRefArg: NpmRegistry) {
+    this.npmregistryRef = npmRegistryRefArg;
+    this.levelCache = new plugins.levelcache.LevelCache({
+      cacheId: encodeURIComponent(this.npmregistryRef.options.npmRegistryUrl),
+      
+    });
+  }
+
+  public async getCachedFile (cacheDescriptorArg: ICacheDescriptor): Promise<plugins.smartfile.Smartfile> {
+    const cacheEntry = await this.levelCache.retrieveCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg));
+    if (cacheEntry) {
+      return plugins.smartfile.Smartfile.fromFoldedJson(cacheEntry.contents.toString());
+    }
+    return null;
+  }
+
+  public async cacheSmartFile (cacheDescriptorArg: ICacheDescriptor, smartfileArg: plugins.smartfile.Smartfile) {
+    if (smartfileArg && cacheDescriptorArg.version) {
+      await this.levelCache.storeCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg), new plugins.levelcache.CacheEntry({
+        contents: Buffer.from(smartfileArg.foldToJson()),
+        ttl: plugins.smarttime.getMilliSecondsFromUnits({hours: 1})
+      }));
+    } else {
+      await this.levelCache.storeCacheEntryByKey(this.getCacheDescriptorAsString(cacheDescriptorArg), new plugins.levelcache.CacheEntry({
+        contents: Buffer.from(smartfileArg.foldToJson()),
+        ttl: plugins.smarttime.getMilliSecondsFromUnits({minutes: 1})
+      }));
+    }
+  }
+
+  public getCacheDescriptorAsString(cacheDescriptorArg?: ICacheDescriptor) {
+    return `${cacheDescriptorArg.registryUrl}//+//${cacheDescriptorArg.packageName}//+//${cacheDescriptorArg.filePath}//+//${cacheDescriptorArg.distTag || cacheDescriptorArg.version}`;
+  }
+}

ts/smartnpm.paths.ts

@@ -2,4 +2,3 @@ import * as plugins from './smartnpm.plugins';
 
 export const packageDir = plugins.path.join(__dirname, '../');
 export const nogitDir = plugins.path.join(packageDir, '.nogit/');
-plugins.smartfile.fs.ensureDirSync(nogitDir);

ts/smartnpm.plugins.ts

@@ -5,11 +5,15 @@ export { path };
 
 // @pushrocks scope
 import * as consolecolor from '@pushrocks/consolecolor';
+import * as levelcache from '@pushrocks/levelcache';
 import * as smartarchive from '@pushrocks/smartarchive';
 import * as smartfile from '@pushrocks/smartfile';
+import * as smartpromise from '@pushrocks/smartpromise';
 import * as smartrequest from '@pushrocks/smartrequest';
+import * as smartversion from '@pushrocks/smartversion';
+import * as smarttime from '@pushrocks/smarttime';
 
-export { consolecolor, smartarchive, smartfile, smartrequest };
+export { consolecolor, levelcache, smartarchive, smartfile, smartpromise, smartrequest, smartversion, smarttime };
 
 // third party scope
 import packageJson from 'package-json';