fix(core): Normalize binary data handling across registries and add buffer helpers

This commit is contained in:
2025-11-25 22:35:31 +00:00
parent e81fa41b18
commit a78934836e
10 changed files with 1496 additions and 9 deletions

View File

@@ -1,5 +1,14 @@
# Changelog # Changelog
## 2025-11-25 - 2.2.1 - fix(core)
Normalize binary data handling across registries and add buffer helpers
- Add core/helpers.buffer.ts with isBinaryData and toBuffer utilities to consistently handle Buffer, Uint8Array, string and object inputs.
- Composer: accept Uint8Array uploads, convert to Buffer before ZIP extraction, SHA-1 calculation and storage.
- PyPI: accept multipart file content as Buffer or Uint8Array and normalize to Buffer before processing and storage.
- Maven: normalize artifact body input with toBuffer before validation and storage.
- OCI: improve upload id generation by using substring for correct random length.
## 2025-11-25 - 2.2.0 - feat(core/registrystorage) ## 2025-11-25 - 2.2.0 - feat(core/registrystorage)
Persist OCI manifest content-type in sidecar and normalize manifest body handling Persist OCI manifest content-type in sidecar and normalize manifest body handling

View File

@@ -0,0 +1,425 @@
/**
* Native Composer CLI Testing
* Tests the Composer registry implementation using the actual composer CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let composerToken: string;
let testDir: string;
let composerHome: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup Composer auth.json for authentication
*/
function setupComposerAuth(
token: string,
composerHomeArg: string,
serverUrl: string,
port: number
): string {
fs.mkdirSync(composerHomeArg, { recursive: true });
const authJson = {
'http-basic': {
[`localhost:${port}`]: {
username: 'testuser',
password: token,
},
},
};
const authPath = path.join(composerHomeArg, 'auth.json');
fs.writeFileSync(authPath, JSON.stringify(authJson, null, 2), 'utf-8');
return authPath;
}
/**
* Create a Composer project that uses our registry
*/
function createComposerProject(
projectDir: string,
serverUrl: string
): void {
fs.mkdirSync(projectDir, { recursive: true });
const composerJson = {
name: 'test/consumer-project',
description: 'Test consumer project for Composer CLI tests',
type: 'project',
require: {},
repositories: [
{
type: 'composer',
url: `${serverUrl}/composer`,
},
],
config: {
'secure-http': false,
},
};
fs.writeFileSync(
path.join(projectDir, 'composer.json'),
JSON.stringify(composerJson, null, 2),
'utf-8'
);
}
/**
* Run Composer command with custom home directory
*/
async function runComposerCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const fullCommand = `cd "${cwd}" && COMPOSER_HOME="${composerHome}" composer ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Upload a Composer package via HTTP API
*/
async function uploadComposerPackage(
vendorPackage: string,
version: string,
token: string,
serverUrl: string
): Promise<void> {
const zipData = await createComposerZip(vendorPackage, version);
const response = await fetch(`${serverUrl}/composer/packages/${vendorPackage}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/zip',
Authorization: `Bearer ${token}`,
},
body: zipData,
});
if (!response.ok) {
const body = await response.text();
throw new Error(`Failed to upload package: ${response.status} ${body}`);
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Composer CLI: should verify composer is installed', async () => {
try {
const result = await tapNodeTools.runCommand('composer --version');
console.log('Composer version output:', result.stdout.substring(0, 200));
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('Composer CLI not available, skipping native CLI tests');
// Skip remaining tests if Composer is not installed
tap.skip.test('Composer CLI: remaining tests skipped - composer not available');
return;
}
});
tap.test('Composer CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
composerToken = tokens.composerToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(composerToken).toBeTypeOf('string');
// Use port 38000 (avoids conflicts with other tests)
registryPort = 38000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-composer-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup COMPOSER_HOME directory
composerHome = path.join(testDir, '.composer');
fs.mkdirSync(composerHome, { recursive: true });
// Setup Composer auth
const authPath = setupComposerAuth(composerToken, composerHome, registryUrl, registryPort);
expect(fs.existsSync(authPath)).toEqual(true);
});
tap.test('Composer CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request
const response = await fetch(`${registryUrl}/composer/packages.json`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('Composer CLI: should upload a package via API', async () => {
const vendorPackage = 'testvendor/test-package';
const version = '1.0.0';
await uploadComposerPackage(vendorPackage, version, composerToken, registryUrl);
// Verify package exists via packages.json
const response = await fetch(`${registryUrl}/composer/packages.json`);
expect(response.status).toEqual(200);
const packagesJson = await response.json();
expect(packagesJson.packages).toBeDefined();
expect(packagesJson.packages[vendorPackage]).toBeDefined();
});
tap.test('Composer CLI: should require package from registry', async () => {
const projectDir = path.join(testDir, 'consumer-project');
createComposerProject(projectDir, registryUrl);
// Try to require the package we uploaded
const result = await runComposerCommand(
'require testvendor/test-package:1.0.0 --no-interaction',
projectDir
);
console.log('composer require output:', result.stdout);
console.log('composer require stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('Composer CLI: should verify package in vendor directory', async () => {
const projectDir = path.join(testDir, 'consumer-project');
const packageDir = path.join(projectDir, 'vendor', 'testvendor', 'test-package');
expect(fs.existsSync(packageDir)).toEqual(true);
// Check composer.json exists in package
const packageComposerPath = path.join(packageDir, 'composer.json');
expect(fs.existsSync(packageComposerPath)).toEqual(true);
});
tap.test('Composer CLI: should upload second version', async () => {
const vendorPackage = 'testvendor/test-package';
const version = '2.0.0';
await uploadComposerPackage(vendorPackage, version, composerToken, registryUrl);
// Verify both versions exist
const response = await fetch(`${registryUrl}/composer/packages.json`);
const packagesJson = await response.json();
expect(packagesJson.packages[vendorPackage]['1.0.0']).toBeDefined();
expect(packagesJson.packages[vendorPackage]['2.0.0']).toBeDefined();
});
tap.test('Composer CLI: should update to new version', async () => {
const projectDir = path.join(testDir, 'consumer-project');
// Update to version 2.0.0
const result = await runComposerCommand(
'require testvendor/test-package:2.0.0 --no-interaction',
projectDir
);
console.log('composer update output:', result.stdout);
expect(result.exitCode).toEqual(0);
// Verify composer.lock has the new version
const lockPath = path.join(projectDir, 'composer.lock');
expect(fs.existsSync(lockPath)).toEqual(true);
const lockContent = JSON.parse(fs.readFileSync(lockPath, 'utf-8'));
const pkg = lockContent.packages.find((p: any) => p.name === 'testvendor/test-package');
expect(pkg?.version).toEqual('2.0.0');
});
tap.test('Composer CLI: should search for packages', async () => {
const projectDir = path.join(testDir, 'consumer-project');
// Search for packages (may not work on all Composer versions)
const result = await runComposerCommand(
'search testvendor --no-interaction 2>&1 || true',
projectDir
);
console.log('composer search output:', result.stdout);
// Search may or may not work depending on registry implementation
// Just verify it doesn't crash
expect(result.exitCode).toBeLessThanOrEqual(1);
});
tap.test('Composer CLI: should show package info', async () => {
const projectDir = path.join(testDir, 'consumer-project');
const result = await runComposerCommand(
'show testvendor/test-package --no-interaction',
projectDir
);
console.log('composer show output:', result.stdout);
expect(result.exitCode).toEqual(0);
expect(result.stdout).toContain('testvendor/test-package');
});
tap.test('Composer CLI: should remove package', async () => {
const projectDir = path.join(testDir, 'consumer-project');
const result = await runComposerCommand(
'remove testvendor/test-package --no-interaction',
projectDir
);
console.log('composer remove output:', result.stdout);
expect(result.exitCode).toEqual(0);
// Verify package is removed from vendor
const packageDir = path.join(projectDir, 'vendor', 'testvendor', 'test-package');
expect(fs.existsSync(packageDir)).toEqual(false);
});
tap.postTask('cleanup composer cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,490 @@
/**
* Native Maven CLI Testing
* Tests the Maven registry implementation using the actual mvn CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createTestPom, createTestJar } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let mavenToken: string;
let testDir: string;
let m2Dir: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup Maven settings.xml for authentication
*/
function setupMavenSettings(
token: string,
m2DirArg: string,
serverUrl: string
): string {
fs.mkdirSync(m2DirArg, { recursive: true });
const settingsXml = `<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>test-registry</id>
<username>testuser</username>
<password>${token}</password>
</server>
</servers>
<profiles>
<profile>
<id>test-registry</id>
<repositories>
<repository>
<id>test-registry</id>
<url>${serverUrl}/maven</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
</profile>
</profiles>
<activeProfiles>
<activeProfile>test-registry</activeProfile>
</activeProfiles>
</settings>
`;
const settingsPath = path.join(m2DirArg, 'settings.xml');
fs.writeFileSync(settingsPath, settingsXml, 'utf-8');
return settingsPath;
}
/**
* Create a minimal Maven project for testing
*/
function createMavenProject(
projectDir: string,
groupId: string,
artifactId: string,
version: string,
registryUrl: string
): void {
fs.mkdirSync(projectDir, { recursive: true });
const pomXml = `<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
<packaging>jar</packaging>
<name>${artifactId}</name>
<description>Test Maven project for SmartRegistry CLI tests</description>
<distributionManagement>
<repository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</repository>
<snapshotRepository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</snapshotRepository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>
`;
fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8');
// Create minimal Java source
const srcDir = path.join(projectDir, 'src', 'main', 'java', 'com', 'test');
fs.mkdirSync(srcDir, { recursive: true });
const javaSource = `package com.test;
public class Main {
public static void main(String[] args) {
System.out.println("Hello from SmartRegistry test!");
}
}
`;
fs.writeFileSync(path.join(srcDir, 'Main.java'), javaSource, 'utf-8');
}
/**
* Run Maven command with custom settings
*/
async function runMavenCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const settingsPath = path.join(m2Dir, 'settings.xml');
const fullCommand = `cd "${cwd}" && mvn -s "${settingsPath}" ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('Maven CLI: should verify mvn is installed', async () => {
try {
const result = await tapNodeTools.runCommand('mvn -version');
console.log('Maven version output:', result.stdout.substring(0, 200));
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('Maven CLI not available, skipping native CLI tests');
// Skip remaining tests if Maven is not installed
tap.skip.test('Maven CLI: remaining tests skipped - mvn not available');
return;
}
});
tap.test('Maven CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
mavenToken = tokens.mavenToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(mavenToken).toBeTypeOf('string');
// Use port 37000 (avoids conflicts with other tests)
registryPort = 37000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-maven-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup .m2 directory
m2Dir = path.join(testDir, '.m2');
fs.mkdirSync(m2Dir, { recursive: true });
// Setup Maven settings
const settingsPath = setupMavenSettings(mavenToken, m2Dir, registryUrl);
expect(fs.existsSync(settingsPath)).toEqual(true);
});
tap.test('Maven CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request
const response = await fetch(`${registryUrl}/maven/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('Maven CLI: should deploy a JAR artifact', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
const projectDir = path.join(testDir, 'test-project');
createMavenProject(projectDir, groupId, artifactId, version, registryUrl);
// Build and deploy
const result = await runMavenCommand('clean package deploy -DskipTests', projectDir);
console.log('mvn deploy output:', result.stdout.substring(0, 500));
console.log('mvn deploy stderr:', result.stderr.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Maven CLI: should verify artifact in registry via API', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
// Maven path: /maven/{groupId path}/{artifactId}/{version}/{artifactId}-{version}.jar
const jarPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`;
const response = await fetch(`${registryUrl}${jarPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const jarData = await response.arrayBuffer();
expect(jarData.byteLength).toBeGreaterThan(0);
});
tap.test('Maven CLI: should verify POM in registry', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '1.0.0';
const pomPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.pom`;
const response = await fetch(`${registryUrl}${pomPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const pomContent = await response.text();
expect(pomContent).toContain(groupId);
expect(pomContent).toContain(artifactId);
expect(pomContent).toContain(version);
});
tap.test('Maven CLI: should verify checksums exist', async () => {
const artifactId = 'test-artifact';
const version = '1.0.0';
// Check JAR checksums
const basePath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`;
// MD5
const md5Response = await fetch(`${registryUrl}${basePath}.md5`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(md5Response.status).toEqual(200);
// SHA1
const sha1Response = await fetch(`${registryUrl}${basePath}.sha1`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(sha1Response.status).toEqual(200);
});
tap.test('Maven CLI: should deploy second version', async () => {
const groupId = 'com.test';
const artifactId = 'test-artifact';
const version = '2.0.0';
const projectDir = path.join(testDir, 'test-project-v2');
createMavenProject(projectDir, groupId, artifactId, version, registryUrl);
const result = await runMavenCommand('clean package deploy -DskipTests', projectDir);
console.log('mvn deploy v2 output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.test('Maven CLI: should verify metadata.xml exists', async () => {
const artifactId = 'test-artifact';
// Maven metadata is stored at /maven/{groupId path}/{artifactId}/maven-metadata.xml
const metadataPath = `/maven/com/test/${artifactId}/maven-metadata.xml`;
const response = await fetch(`${registryUrl}${metadataPath}`, {
headers: { Authorization: `Bearer ${mavenToken}` },
});
expect(response.status).toEqual(200);
const metadataXml = await response.text();
expect(metadataXml).toContain(artifactId);
expect(metadataXml).toContain('1.0.0');
expect(metadataXml).toContain('2.0.0');
});
tap.test('Maven CLI: should resolve dependency from registry', async () => {
const groupId = 'com.consumer';
const artifactId = 'consumer-app';
const version = '1.0.0';
const projectDir = path.join(testDir, 'consumer-project');
fs.mkdirSync(projectDir, { recursive: true });
// Create a consumer project that depends on our test artifact
const pomXml = `<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>${groupId}</groupId>
<artifactId>${artifactId}</artifactId>
<version>${version}</version>
<packaging>jar</packaging>
<repositories>
<repository>
<id>test-registry</id>
<url>${registryUrl}/maven</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>com.test</groupId>
<artifactId>test-artifact</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
</project>
`;
fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8');
// Try to resolve dependencies
const result = await runMavenCommand('dependency:resolve', projectDir);
console.log('mvn dependency:resolve output:', result.stdout.substring(0, 500));
expect(result.exitCode).toEqual(0);
});
tap.postTask('cleanup maven cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -0,0 +1,522 @@
/**
* Native PyPI CLI Testing
* Tests the PyPI registry implementation using pip and twine CLI tools
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createPythonWheel, createPythonSdist } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let pypiToken: string;
let testDir: string;
let pipHome: string;
let hasPip = false;
let hasTwine = false;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else if (contentType.includes('multipart/form-data')) {
// For multipart, pass raw buffer
body = bodyBuffer;
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
rawBody: bodyBuffer,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup .pypirc for twine authentication
*/
function setupPypirc(
token: string,
pipHomeArg: string,
serverUrl: string
): string {
fs.mkdirSync(pipHomeArg, { recursive: true });
const pypircContent = `[distutils]
index-servers = testpypi
[testpypi]
repository = ${serverUrl}/pypi
username = testuser
password = ${token}
`;
const pypircPath = path.join(pipHomeArg, '.pypirc');
fs.writeFileSync(pypircPath, pypircContent, 'utf-8');
// Set restrictive permissions
fs.chmodSync(pypircPath, 0o600);
return pypircPath;
}
/**
* Setup pip.conf for pip to use our registry
*/
function setupPipConf(
token: string,
pipHomeArg: string,
serverUrl: string,
port: number
): string {
fs.mkdirSync(pipHomeArg, { recursive: true });
// pip.conf with authentication
const pipConfContent = `[global]
index-url = ${serverUrl}/pypi/simple/
trusted-host = localhost
extra-index-url = https://pypi.org/simple/
`;
const pipDir = path.join(pipHomeArg, 'pip');
fs.mkdirSync(pipDir, { recursive: true });
const pipConfPath = path.join(pipDir, 'pip.conf');
fs.writeFileSync(pipConfPath, pipConfContent, 'utf-8');
return pipConfPath;
}
/**
* Create a test Python package wheel file
*/
async function createTestWheelFile(
packageName: string,
version: string,
targetDir: string
): Promise<string> {
const wheelData = await createPythonWheel(packageName, version);
const normalizedName = packageName.replace(/-/g, '_');
const wheelFilename = `${normalizedName}-${version}-py3-none-any.whl`;
const wheelPath = path.join(targetDir, wheelFilename);
fs.writeFileSync(wheelPath, wheelData);
return wheelPath;
}
/**
* Create a test Python package sdist file
*/
async function createTestSdistFile(
packageName: string,
version: string,
targetDir: string
): Promise<string> {
const sdistData = await createPythonSdist(packageName, version);
const sdistFilename = `${packageName}-${version}.tar.gz`;
const sdistPath = path.join(targetDir, sdistFilename);
fs.writeFileSync(sdistPath, sdistData);
return sdistPath;
}
/**
* Run pip command with custom config
*/
async function runPipCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const pipConfDir = path.join(pipHome, 'pip');
const fullCommand = `cd "${cwd}" && PIP_CONFIG_FILE="${path.join(pipConfDir, 'pip.conf')}" pip ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Run twine command with custom config
*/
async function runTwineCommand(
command: string,
cwd: string
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const pypircPath = path.join(pipHome, '.pypirc');
const fullCommand = `cd "${cwd}" && twine ${command} --config-file "${pypircPath}"`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('PyPI CLI: should verify pip is installed', async () => {
try {
const result = await tapNodeTools.runCommand('pip --version');
console.log('pip version output:', result.stdout.substring(0, 200));
hasPip = result.exitCode === 0;
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('pip CLI not available');
}
});
tap.test('PyPI CLI: should verify twine is installed', async () => {
try {
const result = await tapNodeTools.runCommand('twine --version');
console.log('twine version output:', result.stdout.substring(0, 200));
hasTwine = result.exitCode === 0;
expect(result.exitCode).toEqual(0);
} catch (error) {
console.log('twine CLI not available');
}
if (!hasPip && !hasTwine) {
console.log('Neither pip nor twine available, skipping native CLI tests');
tap.skip.test('PyPI CLI: remaining tests skipped - no CLI tools available');
return;
}
});
tap.test('PyPI CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
pypiToken = tokens.pypiToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(pypiToken).toBeTypeOf('string');
// Use port 39000 (avoids conflicts with other tests)
registryPort = 39000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-pypi-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup pip/pypi home directory
pipHome = path.join(testDir, '.pip');
fs.mkdirSync(pipHome, { recursive: true });
// Setup .pypirc for twine
const pypircPath = setupPypirc(pypiToken, pipHome, registryUrl);
expect(fs.existsSync(pypircPath)).toEqual(true);
// Setup pip.conf
const pipConfPath = setupPipConf(pypiToken, pipHome, registryUrl, registryPort);
expect(fs.existsSync(pipConfPath)).toEqual(true);
});
tap.test('PyPI CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to simple index
const response = await fetch(`${registryUrl}/pypi/simple/`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('PyPI CLI: should upload wheel with twine', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const packageName = 'test-pypi-pkg';
const version = '1.0.0';
const wheelPath = await createTestWheelFile(packageName, version, testDir);
expect(fs.existsSync(wheelPath)).toEqual(true);
const result = await runTwineCommand(
`upload --repository testpypi "${wheelPath}"`,
testDir
);
console.log('twine upload output:', result.stdout);
console.log('twine upload stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('PyPI CLI: should verify package in simple index', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const packageName = 'test-pypi-pkg';
const response = await fetch(`${registryUrl}/pypi/simple/${packageName}/`);
expect(response.status).toEqual(200);
const html = await response.text();
expect(html).toContain('1.0.0');
});
tap.test('PyPI CLI: should upload sdist with twine', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const packageName = 'test-pypi-pkg';
const version = '1.1.0';
const sdistPath = await createTestSdistFile(packageName, version, testDir);
expect(fs.existsSync(sdistPath)).toEqual(true);
const result = await runTwineCommand(
`upload --repository testpypi "${sdistPath}"`,
testDir
);
console.log('twine upload sdist output:', result.stdout);
console.log('twine upload sdist stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
});
tap.test('PyPI CLI: should list all versions in simple index', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const packageName = 'test-pypi-pkg';
const response = await fetch(`${registryUrl}/pypi/simple/${packageName}/`);
expect(response.status).toEqual(200);
const html = await response.text();
expect(html).toContain('1.0.0');
expect(html).toContain('1.1.0');
});
tap.test('PyPI CLI: should get JSON metadata', async () => {
if (!hasTwine) {
console.log('Skipping - twine not available');
return;
}
const packageName = 'test-pypi-pkg';
const response = await fetch(`${registryUrl}/pypi/pypi/${packageName}/json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
expect(metadata.info).toBeDefined();
expect(metadata.info.name).toEqual(packageName);
expect(metadata.releases).toBeDefined();
expect(metadata.releases['1.0.0']).toBeDefined();
});
tap.test('PyPI CLI: should download package with pip', async () => {
if (!hasPip || !hasTwine) {
console.log('Skipping pip download test - pip or twine not available');
return;
}
const downloadDir = path.join(testDir, 'downloads');
fs.mkdirSync(downloadDir, { recursive: true });
// Download (not install) the package
const result = await runPipCommand(
`download test-pypi-pkg==1.0.0 --dest "${downloadDir}" --no-deps`,
testDir
);
console.log('pip download output:', result.stdout);
console.log('pip download stderr:', result.stderr);
// pip download may fail if the package doesn't meet pip's requirements
// Just check it doesn't crash
expect(result.exitCode).toBeLessThanOrEqual(1);
});
tap.test('PyPI CLI: should search for packages via API', async () => {
const packageName = 'test-pypi-pkg';
// Use the JSON API to search/list
const response = await fetch(`${registryUrl}/pypi/pypi/${packageName}/json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
expect(metadata.info.name).toEqual(packageName);
});
tap.test('PyPI CLI: should fail upload without auth', async () => {
if (!hasTwine) {
console.log('Skipping twine test - twine not available');
return;
}
const packageName = 'unauth-pkg';
const version = '1.0.0';
const wheelPath = await createTestWheelFile(packageName, version, testDir);
// Create a pypirc without proper credentials
const badPypircPath = path.join(testDir, '.bad-pypirc');
fs.writeFileSync(badPypircPath, `[distutils]
index-servers = badpypi
[badpypi]
repository = ${registryUrl}/pypi
username = baduser
password = badtoken
`, 'utf-8');
const fullCommand = `cd "${testDir}" && twine upload --repository badpypi "${wheelPath}" --config-file "${badPypircPath}"`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
// Should fail
expect(result.exitCode).not.toEqual(0);
} catch (error: any) {
// Expected to fail
expect(error.exitCode || 1).not.toEqual(0);
}
});
tap.postTask('cleanup pypi cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartregistry', name: '@push.rocks/smartregistry',
version: '2.2.0', version: '2.2.1',
description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries'
} }

View File

@@ -7,6 +7,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js'; import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type { import type {
IComposerPackage, IComposerPackage,
IComposerPackageMetadata, IComposerPackageMetadata,
@@ -255,7 +256,7 @@ export class ComposerRegistry extends BaseRegistry {
}; };
} }
if (!body || !Buffer.isBuffer(body)) { if (!body || !isBinaryData(body)) {
return { return {
status: 400, status: 400,
headers: {}, headers: {},
@@ -263,8 +264,11 @@ export class ComposerRegistry extends BaseRegistry {
}; };
} }
// Convert to Buffer for ZIP processing
const zipData = toBuffer(body);
// Extract and validate composer.json from ZIP // Extract and validate composer.json from ZIP
const composerJson = await extractComposerJsonFromZip(body); const composerJson = await extractComposerJsonFromZip(zipData);
if (!composerJson || !validateComposerJson(composerJson)) { if (!composerJson || !validateComposerJson(composerJson)) {
return { return {
status: 400, status: 400,
@@ -292,13 +296,13 @@ export class ComposerRegistry extends BaseRegistry {
} }
// Calculate SHA-1 hash // Calculate SHA-1 hash
const shasum = await calculateSha1(body); const shasum = await calculateSha1(zipData);
// Generate reference (use version or commit hash) // Generate reference (use version or commit hash)
const reference = composerJson.source?.reference || version.replace(/[^a-zA-Z0-9.-]/g, '-'); const reference = composerJson.source?.reference || version.replace(/[^a-zA-Z0-9.-]/g, '-');
// Store ZIP file // Store ZIP file
await this.storage.putComposerPackageZip(vendorPackage, reference, body); await this.storage.putComposerPackageZip(vendorPackage, reference, zipData);
// Get or create metadata // Get or create metadata
let metadata = await this.storage.getComposerPackageMetadata(vendorPackage); let metadata = await this.storage.getComposerPackageMetadata(vendorPackage);

34
ts/core/helpers.buffer.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* Shared buffer utilities for consistent binary data handling across all registry types.
*
* This module addresses the common issue where `Buffer.isBuffer(Uint8Array)` returns `false`,
* which can cause data handling bugs when binary data arrives as Uint8Array instead of Buffer.
*/
/**
* Check if value is binary data (Buffer or Uint8Array)
*/
export function isBinaryData(value: unknown): value is Buffer | Uint8Array {
return Buffer.isBuffer(value) || value instanceof Uint8Array;
}
/**
* Convert any binary-like data to Buffer.
* Handles Buffer, Uint8Array, string, and objects.
*
* @param data - The data to convert to Buffer
* @returns A Buffer containing the data
*/
export function toBuffer(data: unknown): Buffer {
if (Buffer.isBuffer(data)) {
return data;
}
if (data instanceof Uint8Array) {
return Buffer.from(data);
}
if (typeof data === 'string') {
return Buffer.from(data, 'utf-8');
}
// Fallback: serialize object to JSON
return Buffer.from(JSON.stringify(data));
}

View File

@@ -7,6 +7,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js';
import type { AuthManager } from '../core/classes.authmanager.js'; import type { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import { toBuffer } from '../core/helpers.buffer.js';
import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js'; import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js';
import { import {
pathToGAV, pathToGAV,
@@ -296,7 +297,7 @@ export class MavenRegistry extends BaseRegistry {
coordinate: IMavenCoordinate, coordinate: IMavenCoordinate,
body: Buffer | any body: Buffer | any
): Promise<IResponse> { ): Promise<IResponse> {
const data = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body)); const data = toBuffer(body);
// Validate POM if uploading .pom file // Validate POM if uploading .pom file
if (coordinate.extension === 'pom') { if (coordinate.extension === 'pom') {

View File

@@ -738,7 +738,7 @@ export class OciRegistry extends BaseRegistry {
} }
private generateUploadId(): string { private generateUploadId(): string {
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
} }
private async calculateDigest(data: Buffer): Promise<string> { private async calculateDigest(data: Buffer): Promise<string> {

View File

@@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js';
import { RegistryStorage } from '../core/classes.registrystorage.js'; import { RegistryStorage } from '../core/classes.registrystorage.js';
import { AuthManager } from '../core/classes.authmanager.js'; import { AuthManager } from '../core/classes.authmanager.js';
import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js';
import { isBinaryData, toBuffer } from '../core/helpers.buffer.js';
import type { import type {
IPypiPackageMetadata, IPypiPackageMetadata,
IPypiFile, IPypiFile,
@@ -328,8 +329,9 @@ export class PypiRegistry extends BaseRegistry {
const version = formData.version; const version = formData.version;
// Support both: formData.content.filename (multipart parsed) and formData.filename (flat) // Support both: formData.content.filename (multipart parsed) and formData.filename (flat)
const filename = formData.content?.filename || formData.filename; const filename = formData.content?.filename || formData.filename;
// Support both: formData.content.data (multipart parsed) and formData.content (Buffer directly) // Support both: formData.content.data (multipart parsed) and formData.content (Buffer/Uint8Array directly)
const fileData = (formData.content?.data || (Buffer.isBuffer(formData.content) ? formData.content : null)) as Buffer; const rawContent = formData.content?.data || (isBinaryData(formData.content) ? formData.content : null);
const fileData = rawContent ? toBuffer(rawContent) : null;
const filetype = formData.filetype; // 'bdist_wheel' or 'sdist' const filetype = formData.filetype; // 'bdist_wheel' or 'sdist'
const pyversion = formData.pyversion; const pyversion = formData.pyversion;