push.rocks/smartnpm
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(registry): modernize npm registry file handling and package extraction APIs

Browse Source
This commit is contained in:
Jürgen Kunz
2026-05-01 15:39:02 +00:00
parent 3bb68776fb
commit 0dedf79fa7
14 changed files with 3712 additions and 4592 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ts/00_commitinfo_data.ts
+1 -1
View File
@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartnpm',
version: '2.0.6',
version: '2.1.0',
description: 'A library to interface with npm for retrieving package information and manipulation.'
}
ts/smartnpm.classes.npmpackage.ts
+39 -79
View File
@@ -6,7 +6,7 @@ import { PackageVersion, type IVersionData } from './smartnpm.classes.packagever
export class NpmPackage {
public static async createFromFullMetadataAndVersionData(
npmRegistryArg: NpmRegistry,
fullMetadataArg: plugins.packageJson.FullMetadata,
fullMetadataArg: Record<string, unknown>,
versionsDataArg: {
name: string;
'dist-tags': { [key: string]: string };
@@ -34,30 +34,30 @@ export class NpmPackage {
}
// INSTANCE
public name: string = null;
public scope: string = null;
public version: string = null;
public allVersions: PackageVersion[];
public allDistTags: PackageDisttag[];
public description: string = null;
public keywords: string[] = null;
public date: string;
public license: string;
public links: {
public name: string | null = null;
public scope: string | null = null;
public version: string | null = null;
public allVersions: PackageVersion[] = [];
public allDistTags: PackageDisttag[] = [];
public description: string | null = null;
public keywords: string[] | null = null;
public date!: string;
public license!: string;
public links!: {
npm: string;
homepage: string;
repository: string;
bugs: string;
};
public author: {
public author!: {
name: 'Lossless GmbH';
};
public publisher: {
public publisher!: {
username: 'gitzone';
email: 'npm@git.zone';
};
public maintainers: any = null;
public dist: {
public maintainers: unknown = null;
public dist!: {
integrity: string;
shasum: string;
tarball: string;
@@ -69,8 +69,8 @@ export class NpmPackage {
popularity: number;
maintenance: number;
};
} = null;
public searchScore: number = null;
} | null = null;
public searchScore: number | null = null;
public npmRegistryRef: NpmRegistry;
constructor(npmRegistryArg: NpmRegistry) {
@@ -81,9 +81,7 @@ export class NpmPackage {
* saves the package to disk
*/
public async saveToDisk(targetDir: string) {
const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
const archive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(this.dist.tarball);
await archive.exportToFs(targetDir);
await plugins.smartarchive.SmartArchive.create().url(this.dist.tarball).extract(targetDir);
}
/**
@@ -99,17 +97,15 @@ export class NpmPackage {
optionsArg: {
distTag?: string;
version?: string;
},
} = {},
returnOnFirstArg = false
): Promise<plugins.smartfile.SmartFile[]> {
const done = plugins.smartpromise.defer<plugins.smartfile.SmartFile[]>();
const smartarchiveInstance = new plugins.smartarchive.SmartArchive();
let tarballUrl = this.dist?.tarball;
): Promise<plugins.smartfile.SmartFile[] | null> {
let tarballUrl: string | undefined = this.dist?.tarball;
if (optionsArg?.version || optionsArg?.distTag) {
if (optionsArg.distTag && optionsArg.version) {
throw new Error('Please either specify version OR disttag, not both.');
}
let targetVersionString: string;
let targetVersionString: string | undefined;
if (optionsArg.distTag) {
const targetDistTag = this.allDistTags.find((distTag) => {
return distTag.name === optionsArg.distTag;
@@ -120,6 +116,9 @@ export class NpmPackage {
} else {
targetVersionString = optionsArg.version;
}
if (!targetVersionString) {
return null;
}
// lets find the best matching release
const bestMatchingVersion = this.getBestMatchingVersion(targetVersionString);
@@ -128,59 +127,20 @@ export class NpmPackage {
}
tarballUrl = this.allVersions.find(
(packageVersion) => packageVersion.version === bestMatchingVersion
).dist.tarball;
)?.dist.tarball;
}
if (!tarballUrl) {
return null;
}
const archive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(tarballUrl);
const streamOfFiles = await archive.exportToStreamOfStreamFiles();
const wantedFilePath = plugins.path.join('package', filePath);
// Collect all stream files first
const streamFileList: any[] = [];
await new Promise<void>((resolve, reject) => {
streamOfFiles.on('data', (streamFile) => {
streamFileList.push(streamFile);
});
streamOfFiles.on('end', resolve);
streamOfFiles.on('error', reject);
});
// Now process the collected files
const allMatchingFiles: plugins.smartfile.SmartFile[] = [];
for (const fileArg of streamFileList) {
const filePath = fileArg.relativeFilePath || fileArg.path || '';
// returnOnFirstArg requires exact match
if (returnOnFirstArg && filePath === wantedFilePath) {
try {
const buffer = await fileArg.getContentAsBuffer();
const smartFile = await plugins.smartfile.SmartFile.fromBuffer(
filePath,
buffer
);
done.resolve([smartFile]);
return done.promise;
} catch (error) {
console.error('Error processing file:', error);
}
} else if (!returnOnFirstArg && filePath.startsWith(wantedFilePath)) {
try {
const buffer = await fileArg.getContentAsBuffer();
const smartFile = await plugins.smartfile.SmartFile.fromBuffer(
filePath,
buffer
);
allMatchingFiles.push(smartFile);
} catch (error) {
console.error('Error processing file:', error);
}
const allFiles = await plugins.smartarchive.SmartArchive.create().url(tarballUrl).toSmartFiles();
const allMatchingFiles = allFiles.filter((fileArg) => {
if (returnOnFirstArg) {
return fileArg.path === wantedFilePath;
}
}
done.resolve(allMatchingFiles);
return done.promise;
return fileArg.path.startsWith(wantedFilePath);
});
return returnOnFirstArg ? allMatchingFiles.slice(0, 1) : allMatchingFiles;
}
/**
@@ -192,9 +152,9 @@ export class NpmPackage {
distTag?: string;
version?: string;
}
): Promise<plugins.smartfile.SmartFile> {
): Promise<plugins.smartfile.SmartFile | null> {
const result = await this.getFilesFromPackage(filePath, optionsArg, true);
return result[0] || null;
return result?.[0] || null;
}
/**
@@ -203,7 +163,7 @@ export class NpmPackage {
update() {}
/** */
public getBestMatchingVersion(versionArg: string): string {
public getBestMatchingVersion(versionArg: string): string | null {
// lets find the best matching release
const targetVersion = plugins.smartversion.SmartVersion.fromFuzzyString(versionArg);
const versionStrings = this.allVersions.map((packageVersion) => packageVersion.version);
ts/smartnpm.classes.npmregistry.ts
+31 -18
View File
@@ -13,12 +13,14 @@ export interface INpmRegistryConstructorOptions {
}
export class NpmRegistry {
public options: INpmRegistryConstructorOptions;
public options: Required<INpmRegistryConstructorOptions>;
public registryCache: RegistryCache;
public smartFs = new plugins.smartfs.SmartFs(new plugins.smartfs.SmartFsProviderNode());
public smartFileFactory = plugins.smartfile.SmartFileFactory.nodeFs();
private searchDomain = 'https://api.npms.io/v2/search?q=';
constructor(optionsArg: INpmRegistryConstructorOptions = {}) {
const defaultOptions: INpmRegistryConstructorOptions = {
const defaultOptions: Required<INpmRegistryConstructorOptions> = {
npmRegistryUrl: 'https://registry.npmjs.org',
};
this.options = {
@@ -46,10 +48,17 @@ export class NpmRegistry {
registryUrl: this.options.npmRegistryUrl,
allVersions: true,
});
if (!fullMetadata) {
throw new Error(`Could not retrieve metadata for package ${packageName}.`);
}
const npmPackage = await NpmPackage.createFromFullMetadataAndVersionData(
this,
fullMetadata,
versionData as any
versionData as {
name: string;
'dist-tags': { [key: string]: string };
versions: { [key: string]: import('./smartnpm.classes.packageversion.js').IVersionData };
}
);
return npmPackage;
}
@@ -60,7 +69,7 @@ export class NpmRegistry {
* @param targetDir
*/
public async savePackageToDisk(packageName: string, targetDir: string): Promise<void> {
plugins.smartfile.fs.ensureDirSync(paths.nogitDir);
await this.smartFs.directory(paths.nogitDir).create();
const npmPackage = await this.getPackageInfo(packageName);
await npmPackage.saveToDisk(targetDir);
}
@@ -75,7 +84,7 @@ export class NpmRegistry {
distTag?: string;
version?: string;
}
): Promise<plugins.smartfile.SmartFile> {
): Promise<plugins.smartfile.SmartFile | null> {
// lets create a cache descriptor
const cacheDescriptor: ICacheDescriptor = {
registryUrl: this.options.npmRegistryUrl,
@@ -86,7 +95,7 @@ export class NpmRegistry {
};
// lets see if we have something cached
const cachedFile: plugins.smartfile.SmartFile = await this.registryCache.getCachedFile(
const cachedFile = await this.registryCache.getCachedFile(
cacheDescriptor
);
@@ -98,9 +107,8 @@ export class NpmRegistry {
(packageArg) => packageArg.name === 'latest'
);
if (!latestAvailable) {
optionsArg = {
version: npmPackage.getBestMatchingVersion('*'),
};
const version = npmPackage.getBestMatchingVersion('*');
optionsArg = version ? { version } : undefined;
}
}
const fileResult = await npmPackage.getFileFromPackage(filePathArg, optionsArg);
@@ -120,16 +128,15 @@ export class NpmRegistry {
distTag?: string;
version?: string;
}
): Promise<plugins.smartfile.SmartFile[]> {
): Promise<plugins.smartfile.SmartFile[] | null> {
const npmPackage = await this.getPackageInfo(packageNameArg);
if (!optionsArg?.version && !optionsArg?.distTag) {
const latestAvailable = npmPackage.allDistTags.find(
(packageDistTagArg) => packageDistTagArg.name === 'latest'
);
if (!latestAvailable) {
optionsArg = {
version: npmPackage.getBestMatchingVersion('*'),
};
const version = npmPackage.getBestMatchingVersion('*');
optionsArg = version ? { version } : undefined;
}
}
return npmPackage.getFilesFromPackage(filePath, optionsArg);
@@ -142,10 +149,10 @@ export class NpmRegistry {
* TODO: rewrite as memory only
*/
const baseDir = plugins.path.join(paths.nogitDir, packageNameArg.replace('/', '__'));
await plugins.smartfile.fs.ensureDir(baseDir);
await this.smartFs.directory(baseDir).create();
await this.savePackageToDisk(packageNameArg, baseDir);
const virtualDir = await plugins.smartfile.VirtualDirectory.fromFsDirPath(baseDir);
await plugins.smartfile.fs.remove(baseDir);
const virtualDir = await this.smartFileFactory.virtualDirectoryFromPath(baseDir);
await this.smartFs.directory(baseDir).recursive().delete();
return virtualDir;
}
@@ -224,7 +231,13 @@ export class NpmRegistry {
`info: Search on npm for ${plugins.consolecolor.coloredString(searchString, 'pink')}`
);
let body: any;
let body: {
results?: Array<{
package: {
name: string;
};
}>;
} | string | undefined;
try {
const response = await plugins.smartrequest.SmartRequest.create()
.url(this.searchDomain + searchString)
@@ -238,7 +251,7 @@ export class NpmRegistry {
const packageArray: NpmPackage[] = [];
// if request failed just return it empty
if (!body || typeof body === 'string') {
if (!body || typeof body === 'string' || !Array.isArray(body.results)) {
return packageArray;
}
ts/smartnpm.classes.packageversion.ts
+5 -5
View File
@@ -19,11 +19,11 @@ export class PackageVersion implements IVersionData {
return packageVersion;
}
name: string;
version: string;
dependencies: { [key: string]: string };
devDependencies: { [key: string]: string };
dist: {
name!: string;
version!: string;
dependencies!: { [key: string]: string };
devDependencies!: { [key: string]: string };
dist!: {
integrity: string;
shasum: string;
tarball: string;
ts/smartnpm.classes.registrycache.ts
+4 -3
View File
@@ -12,6 +12,7 @@ export interface ICacheDescriptor {
export class RegistryCache {
npmregistryRef: NpmRegistry;
public levelCache: plugins.levelcache.LevelCache;
public smartFileFactory = plugins.smartfile.SmartFileFactory.nodeFs();
constructor(npmRegistryRefArg: NpmRegistry) {
this.npmregistryRef = npmRegistryRefArg;
@@ -22,12 +23,12 @@ export class RegistryCache {
public async getCachedFile(
cacheDescriptorArg: ICacheDescriptor
): Promise<plugins.smartfile.SmartFile> {
): Promise<plugins.smartfile.SmartFile | null> {
const cacheEntry = await this.levelCache.retrieveCacheEntryByKey(
this.getCacheDescriptorAsString(cacheDescriptorArg)
);
if (cacheEntry) {
return plugins.smartfile.SmartFile.fromFoldedJson(cacheEntry.contents.toString());
return this.smartFileFactory.fromFoldedJson(cacheEntry.contents.toString());
}
return null;
}
@@ -55,7 +56,7 @@ export class RegistryCache {
}
}
public getCacheDescriptorAsString(cacheDescriptorArg?: ICacheDescriptor) {
public getCacheDescriptorAsString(cacheDescriptorArg: ICacheDescriptor) {
return `${cacheDescriptorArg.registryUrl}//+//${cacheDescriptorArg.packageName}//+//${
cacheDescriptorArg.filePath
}//+//${cacheDescriptorArg.distTag || cacheDescriptorArg.version}`;
ts/smartnpm.plugins.ts
+2
View File
@@ -8,6 +8,7 @@ import * as consolecolor from '@push.rocks/consolecolor';
import * as levelcache from '@push.rocks/levelcache';
import * as smartarchive from '@push.rocks/smartarchive';
import * as smartfile from '@push.rocks/smartfile';
import * as smartfs from '@push.rocks/smartfs';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
@@ -19,6 +20,7 @@ export {
levelcache,
smartarchive,
smartfile,
smartfs,
smartpath,
smartpromise,
smartrequest,