feat(registry): modernize npm registry file handling and package extraction APIs

This commit is contained in:
2026-05-01 15:39:02 +00:00
parent 3bb68776fb
commit 0dedf79fa7
14 changed files with 3712 additions and 4592 deletions
+37
View File
@@ -0,0 +1,37 @@
{
"@git.zone/cli": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartnpm",
"description": "A library to interface with npm for retrieving package information and manipulation.",
"npmPackagename": "@push.rocks/smartnpm",
"license": "MIT",
"keywords": [
"npm",
"package",
"information",
"registry",
"search",
"metadata",
"version",
"dependencies"
]
},
"release": {
"registries": [
"https://verdaccio.lossless.digital",
"https://registry.npmjs.org"
],
"accessLevel": "public"
}
},
"@git.zone/tsdoc": {
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
},
"@ship.zone/szci": {
"npmGlobalTools": [],
"npmRegistryUrl": "registry.npmjs.org"
}
}
+7
View File
@@ -1,5 +1,12 @@
# Changelog
## 2026-05-01 - 2.1.0 - feat(registry)
modernize npm registry file handling and package extraction APIs
- replace legacy smartarchive and smartfile filesystem usage with SmartArchive create() chaining, SmartFs, and SmartFileFactory node adapters
- improve null safety and type annotations across registry, package, and cache classes to handle missing metadata, versions, tarballs, and search results more robustly
- update package metadata, tooling configuration, test imports, and dependency versions to align with the newer build and CI setup
## 2025-08-18 - 2.0.6 - fix(readme)
Expand README with detailed usage examples, API reference and features; add local assistant settings
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2026 Task Venture Capital GmbH
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+13 -6
View File
@@ -1,9 +1,5 @@
{
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "public"
},
"gitzone": {
"@git.zone/cli": {
"projectType": "npm",
"module": {
"githost": "code.foss.global",
@@ -22,9 +18,20 @@
"version",
"dependencies"
]
},
"release": {
"registries": [
"https://verdaccio.lossless.digital",
"https://registry.npmjs.org"
],
"accessLevel": "public"
}
},
"tsdoc": {
"@git.zone/tsdoc": {
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
},
"@ship.zone/szci": {
"npmGlobalTools": [],
"npmRegistryUrl": "registry.npmjs.org"
}
}
+48 -41
View File
@@ -6,47 +6,18 @@
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
"author": "Lossless GmbH",
"author": "Task Venture Capital GmbH <hello@task.vc>",
"license": "MIT",
"scripts": {
"test": "(tstest test/ --verbose --logfile --timeout 120)",
"build": "(tsbuild --web --allowimplicitany)",
"test": "tstest test/ --verbose --timeout 120",
"build": "tsbuild --web",
"format": "gitzone format",
"buildDocs": "tsdoc"
},
"devDependencies": {
"@git.zone/tsbuild": "^2.6.6",
"@git.zone/tsrun": "^1.3.3",
"@git.zone/tstest": "^2.3.4",
"@push.rocks/tapbundle": "^6.0.3",
"@types/node": "^20.4.4"
"repository": {
"type": "git",
"url": "https://code.foss.global/push.rocks/smartnpm.git"
},
"dependencies": {
"@push.rocks/consolecolor": "^2.0.1",
"@push.rocks/levelcache": "^3.0.6",
"@push.rocks/smartarchive": "^4.2.1",
"@push.rocks/smartfile": "^11.2.7",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartpromise": "^4.0.3",
"@push.rocks/smartrequest": "^4.2.2",
"@push.rocks/smarttime": "^4.0.4",
"@push.rocks/smartversion": "^3.0.2",
"package-json": "^8.1.1"
},
"files": [
"ts/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_*/**/*",
"dist_ts/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
"readme.md"
],
"browserslist": [
"last 1 chrome versions"
],
"keywords": [
"npm",
"package",
@@ -57,10 +28,46 @@
"version",
"dependencies"
],
"homepage": "https://code.foss.global/push.rocks/smartnpm",
"repository": {
"type": "git",
"url": "https://code.foss.global/push.rocks/smartnpm.git"
"bugs": {
"url": "https://gitlab.com/push.rocks/smartnpm/issues"
},
"packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748"
"homepage": "https://code.foss.global/push.rocks/smartnpm",
"devDependencies": {
"@git.zone/tsbuild": "^4.4.0",
"@git.zone/tsrun": "^2.0.3",
"@git.zone/tstest": "^3.6.3",
"@types/lodash.clonedeep": "^4.5.9",
"@types/node": "^25.6.0"
},
"dependencies": {
"@push.rocks/consolecolor": "^2.0.3",
"@push.rocks/levelcache": "^3.2.2",
"@push.rocks/smartarchive": "^5.2.2",
"@push.rocks/smartfile": "^13.1.3",
"@push.rocks/smartfs": "^1.5.1",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartpromise": "^4.2.3",
"@push.rocks/smartrequest": "^5.0.1",
"@push.rocks/smarttime": "^4.2.3",
"@push.rocks/smartversion": "^3.1.0",
"package-json": "^10.0.1"
},
"files": [
"ts/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_*/**/*",
"dist_ts/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
".smartconfig.json",
"license",
"npmextra.json",
"readme.md"
],
"browserslist": [
"last 1 chrome versions"
],
"packageManager": "pnpm@10.28.2"
}
+3498 -4433
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -1,4 +1,4 @@
import { expect, tap } from '@push.rocks/tapbundle';
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as smartnpm from '../ts/index.js';
import { NpmRegistry } from '../ts/index.js';
+1 -1
View File
@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartnpm',
version: '2.0.6',
version: '2.1.0',
description: 'A library to interface with npm for retrieving package information and manipulation.'
}
+39 -79
View File
@@ -6,7 +6,7 @@ import { PackageVersion, type IVersionData } from './smartnpm.classes.packagever
export class NpmPackage {
public static async createFromFullMetadataAndVersionData(
npmRegistryArg: NpmRegistry,
fullMetadataArg: plugins.packageJson.FullMetadata,
fullMetadataArg: Record<string, unknown>,
versionsDataArg: {
name: string;
'dist-tags': { [key: string]: string };
@@ -34,30 +34,30 @@ export class NpmPackage {
}
// INSTANCE
public name: string = null;
public scope: string = null;
public version: string = null;
public allVersions: PackageVersion[];
public allDistTags: PackageDisttag[];
public description: string = null;
public keywords: string[] = null;
public date: string;
public license: string;
public links: {
public name: string | null = null;
public scope: string | null = null;
public version: string | null = null;
public allVersions: PackageVersion[] = [];
public allDistTags: PackageDisttag[] = [];
public description: string | null = null;
public keywords: string[] | null = null;
public date!: string;
public license!: string;
public links!: {
npm: string;
homepage: string;
repository: string;
bugs: string;
};
public author: {
public author!: {
name: 'Lossless GmbH';
};
public publisher: {
public publisher!: {
username: 'gitzone';
email: 'npm@git.zone';
};
public maintainers: any = null;
public dist: {
public maintainers: unknown = null;
public dist!: {
integrity: string;
shasum: string;
tarball: string;
@@ -69,8 +69,8 @@ export class NpmPackage {
popularity: number;
maintenance: number;
};
} = null;
public searchScore: number = null;
} | null = null;
public searchScore: number | null = null;
public npmRegistryRef: NpmRegistry;
constructor(npmRegistryArg: NpmRegistry) {
@@ -81,9 +81,7 @@ export class NpmPackage {
* saves the package to disk
*/
public async saveToDisk(targetDir: string) {
const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
const archive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(this.dist.tarball);
await archive.exportToFs(targetDir);
await plugins.smartarchive.SmartArchive.create().url(this.dist.tarball).extract(targetDir);
}
/**
@@ -99,17 +97,15 @@ export class NpmPackage {
optionsArg: {
distTag?: string;
version?: string;
},
} = {},
returnOnFirstArg = false
): Promise<plugins.smartfile.SmartFile[]> {
const done = plugins.smartpromise.defer<plugins.smartfile.SmartFile[]>();
const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
let tarballUrl = this.dist?.tarball;
): Promise<plugins.smartfile.SmartFile[] | null> {
let tarballUrl: string | undefined = this.dist?.tarball;
if (optionsArg?.version || optionsArg?.distTag) {
if (optionsArg.distTag && optionsArg.version) {
throw new Error('Please either specify version OR disttag, not both.');
}
let targetVersionString: string;
let targetVersionString: string | undefined;
if (optionsArg.distTag) {
const targetDistTag = this.allDistTags.find((distTag) => {
return distTag.name === optionsArg.distTag;
@@ -120,6 +116,9 @@ export class NpmPackage {
} else {
targetVersionString = optionsArg.version;
}
if (!targetVersionString) {
return null;
}
// lets find the best matching release
const bestMatchingVersion = this.getBestMatchingVersion(targetVersionString);
@@ -128,59 +127,20 @@ export class NpmPackage {
}
tarballUrl = this.allVersions.find(
(packageVersion) => packageVersion.version === bestMatchingVersion
).dist.tarball;
)?.dist.tarball;
}
if (!tarballUrl) {
return null;
}
const archive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(tarballUrl);
const streamOfFiles = await archive.exportToStreamOfStreamFiles();
const wantedFilePath = plugins.path.join('package', filePath);
// Collect all stream files first
const streamFileList: any[] = [];
await new Promise<void>((resolve, reject) => {
streamOfFiles.on('data', (streamFile) => {
streamFileList.push(streamFile);
const allFiles = await plugins.smartarchive.SmartArchive.create().url(tarballUrl).toSmartFiles();
const allMatchingFiles = allFiles.filter((fileArg) => {
if (returnOnFirstArg) {
return fileArg.path === wantedFilePath;
}
return fileArg.path.startsWith(wantedFilePath);
});
streamOfFiles.on('end', resolve);
streamOfFiles.on('error', reject);
});
// Now process the collected files
const allMatchingFiles: plugins.smartfile.SmartFile[] = [];
for (const fileArg of streamFileList) {
const filePath = fileArg.relativeFilePath || fileArg.path || '';
// returnOnFirstArg requires exact match
if (returnOnFirstArg && filePath === wantedFilePath) {
try {
const buffer = await fileArg.getContentAsBuffer();
const smartFile = await plugins.smartfile.SmartFile.fromBuffer(
filePath,
buffer
);
done.resolve([smartFile]);
return done.promise;
} catch (error) {
console.error('Error processing file:', error);
}
} else if (!returnOnFirstArg && filePath.startsWith(wantedFilePath)) {
try {
const buffer = await fileArg.getContentAsBuffer();
const smartFile = await plugins.smartfile.SmartFile.fromBuffer(
filePath,
buffer
);
allMatchingFiles.push(smartFile);
} catch (error) {
console.error('Error processing file:', error);
}
}
}
done.resolve(allMatchingFiles);
return done.promise;
return returnOnFirstArg ? allMatchingFiles.slice(0, 1) : allMatchingFiles;
}
/**
@@ -192,9 +152,9 @@ export class NpmPackage {
distTag?: string;
version?: string;
}
): Promise<plugins.smartfile.SmartFile> {
): Promise<plugins.smartfile.SmartFile | null> {
const result = await this.getFilesFromPackage(filePath, optionsArg, true);
return result[0] || null;
return result?.[0] || null;
}
/**
@@ -203,7 +163,7 @@ export class NpmPackage {
update() {}
/** */
public getBestMatchingVersion(versionArg: string): string {
public getBestMatchingVersion(versionArg: string): string | null {
// lets find the best matching release
const targetVersion = plugins.smartversion.SmartVersion.fromFuzzyString(versionArg);
const versionStrings = this.allVersions.map((packageVersion) => packageVersion.version);
+31 -18
View File
@@ -13,12 +13,14 @@ export interface INpmRegistryConstructorOptions {
}
export class NpmRegistry {
public options: INpmRegistryConstructorOptions;
public options: Required<INpmRegistryConstructorOptions>;
public registryCache: RegistryCache;
public smartFs = new plugins.smartfs.SmartFs(new plugins.smartfs.SmartFsProviderNode());
public smartFileFactory = plugins.smartfile.SmartFileFactory.nodeFs();
private searchDomain = 'https://api.npms.io/v2/search?q=';
constructor(optionsArg: INpmRegistryConstructorOptions = {}) {
const defaultOptions: INpmRegistryConstructorOptions = {
const defaultOptions: Required<INpmRegistryConstructorOptions> = {
npmRegistryUrl: 'https://registry.npmjs.org',
};
this.options = {
@@ -46,10 +48,17 @@ export class NpmRegistry {
registryUrl: this.options.npmRegistryUrl,
allVersions: true,
});
if (!fullMetadata) {
throw new Error(`Could not retrieve metadata for package ${packageName}.`);
}
const npmPackage = await NpmPackage.createFromFullMetadataAndVersionData(
this,
fullMetadata,
versionData as any
versionData as {
name: string;
'dist-tags': { [key: string]: string };
versions: { [key: string]: import('./smartnpm.classes.packageversion.js').IVersionData };
}
);
return npmPackage;
}
@@ -60,7 +69,7 @@ export class NpmRegistry {
* @param targetDir
*/
public async savePackageToDisk(packageName: string, targetDir: string): Promise<void> {
plugins.smartfile.fs.ensureDirSync(paths.nogitDir);
await this.smartFs.directory(paths.nogitDir).create();
const npmPackage = await this.getPackageInfo(packageName);
await npmPackage.saveToDisk(targetDir);
}
@@ -75,7 +84,7 @@ export class NpmRegistry {
distTag?: string;
version?: string;
}
): Promise<plugins.smartfile.SmartFile> {
): Promise<plugins.smartfile.SmartFile | null> {
// lets create a cache descriptor
const cacheDescriptor: ICacheDescriptor = {
registryUrl: this.options.npmRegistryUrl,
@@ -86,7 +95,7 @@ export class NpmRegistry {
};
// lets see if we have something cached
const cachedFile: plugins.smartfile.SmartFile = await this.registryCache.getCachedFile(
const cachedFile = await this.registryCache.getCachedFile(
cacheDescriptor
);
@@ -98,9 +107,8 @@ export class NpmRegistry {
(packageArg) => packageArg.name === 'latest'
);
if (!latestAvailable) {
optionsArg = {
version: npmPackage.getBestMatchingVersion('*'),
};
const version = npmPackage.getBestMatchingVersion('*');
optionsArg = version ? { version } : undefined;
}
}
const fileResult = await npmPackage.getFileFromPackage(filePathArg, optionsArg);
@@ -120,16 +128,15 @@ export class NpmRegistry {
distTag?: string;
version?: string;
}
): Promise<plugins.smartfile.SmartFile[]> {
): Promise<plugins.smartfile.SmartFile[] | null> {
const npmPackage = await this.getPackageInfo(packageNameArg);
if (!optionsArg?.version && !optionsArg?.distTag) {
const latestAvailable = npmPackage.allDistTags.find(
(packageDistTagArg) => packageDistTagArg.name === 'latest'
);
if (!latestAvailable) {
optionsArg = {
version: npmPackage.getBestMatchingVersion('*'),
};
const version = npmPackage.getBestMatchingVersion('*');
optionsArg = version ? { version } : undefined;
}
}
return npmPackage.getFilesFromPackage(filePath, optionsArg);
@@ -142,10 +149,10 @@ export class NpmRegistry {
* TODO: rewrite as memory only
*/
const baseDir = plugins.path.join(paths.nogitDir, packageNameArg.replace('/', '__'));
await plugins.smartfile.fs.ensureDir(baseDir);
await this.smartFs.directory(baseDir).create();
await this.savePackageToDisk(packageNameArg, baseDir);
const virtualDir = await plugins.smartfile.VirtualDirectory.fromFsDirPath(baseDir);
await plugins.smartfile.fs.remove(baseDir);
const virtualDir = await this.smartFileFactory.virtualDirectoryFromPath(baseDir);
await this.smartFs.directory(baseDir).recursive().delete();
return virtualDir;
}
@@ -224,7 +231,13 @@ export class NpmRegistry {
`info: Search on npm for ${plugins.consolecolor.coloredString(searchString, 'pink')}`
);
let body: any;
let body: {
results?: Array<{
package: {
name: string;
};
}>;
} | string | undefined;
try {
const response = await plugins.smartrequest.SmartRequest.create()
.url(this.searchDomain + searchString)
@@ -238,7 +251,7 @@ export class NpmRegistry {
const packageArray: NpmPackage[] = [];
// if request failed just return it empty
if (!body || typeof body === 'string') {
if (!body || typeof body === 'string' || !Array.isArray(body.results)) {
return packageArray;
}
+5 -5
View File
@@ -19,11 +19,11 @@ export class PackageVersion implements IVersionData {
return packageVersion;
}
name: string;
version: string;
dependencies: { [key: string]: string };
devDependencies: { [key: string]: string };
dist: {
name!: string;
version!: string;
dependencies!: { [key: string]: string };
devDependencies!: { [key: string]: string };
dist!: {
integrity: string;
shasum: string;
tarball: string;
+4 -3
View File
@@ -12,6 +12,7 @@ export interface ICacheDescriptor {
export class RegistryCache {
npmregistryRef: NpmRegistry;
public levelCache: plugins.levelcache.LevelCache;
public smartFileFactory = plugins.smartfile.SmartFileFactory.nodeFs();
constructor(npmRegistryRefArg: NpmRegistry) {
this.npmregistryRef = npmRegistryRefArg;
@@ -22,12 +23,12 @@ export class RegistryCache {
public async getCachedFile(
cacheDescriptorArg: ICacheDescriptor
): Promise<plugins.smartfile.SmartFile> {
): Promise<plugins.smartfile.SmartFile | null> {
const cacheEntry = await this.levelCache.retrieveCacheEntryByKey(
this.getCacheDescriptorAsString(cacheDescriptorArg)
);
if (cacheEntry) {
return plugins.smartfile.SmartFile.fromFoldedJson(cacheEntry.contents.toString());
return this.smartFileFactory.fromFoldedJson(cacheEntry.contents.toString());
}
return null;
}
@@ -55,7 +56,7 @@ export class RegistryCache {
}
}
public getCacheDescriptorAsString(cacheDescriptorArg?: ICacheDescriptor) {
public getCacheDescriptorAsString(cacheDescriptorArg: ICacheDescriptor) {
return `${cacheDescriptorArg.registryUrl}//+//${cacheDescriptorArg.packageName}//+//${
cacheDescriptorArg.filePath
}//+//${cacheDescriptorArg.distTag || cacheDescriptorArg.version}`;
+2
View File
@@ -8,6 +8,7 @@ import * as consolecolor from '@push.rocks/consolecolor';
import * as levelcache from '@push.rocks/levelcache';
import * as smartarchive from '@push.rocks/smartarchive';
import * as smartfile from '@push.rocks/smartfile';
import * as smartfs from '@push.rocks/smartfs';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
@@ -19,6 +20,7 @@ export {
levelcache,
smartarchive,
smartfile,
smartfs,
smartpath,
smartpromise,
smartrequest,
+4 -4
View File
@@ -5,10 +5,10 @@
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"noImplicitAny": true,
"esModuleInterop": true,
"verbatimModuleSyntax": true
"verbatimModuleSyntax": true,
"types": ["node"]
},
"exclude": [
"dist_*/**/*.d.ts"
]
"exclude": ["dist_*/**/*.d.ts"]
}