diff --git a/changelog.md b/changelog.md index ca4c570..f107af8 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,14 @@ # Changelog +## 2025-11-25 - 2.2.1 - fix(core) +Normalize binary data handling across registries and add buffer helpers + +- Add core/helpers.buffer.ts with isBinaryData and toBuffer utilities to consistently handle Buffer, Uint8Array, string and object inputs. +- Composer: accept Uint8Array uploads, convert to Buffer before ZIP extraction, SHA-1 calculation and storage. +- PyPI: accept multipart file content as Buffer or Uint8Array and normalize to Buffer before processing and storage. +- Maven: normalize artifact body input with toBuffer before validation and storage. +- OCI: improve upload id generation by using substring for correct random length. + ## 2025-11-25 - 2.2.0 - feat(core/registrystorage) Persist OCI manifest content-type in sidecar and normalize manifest body handling diff --git a/test/test.composer.nativecli.node.ts b/test/test.composer.nativecli.node.ts new file mode 100644 index 0000000..8404d40 --- /dev/null +++ b/test/test.composer.nativecli.node.ts @@ -0,0 +1,425 @@ +/** + * Native Composer CLI Testing + * Tests the Composer registry implementation using the actual composer CLI + */ + +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside'; +import { SmartRegistry } from '../ts/index.js'; +import { createTestRegistry, createTestTokens, createComposerZip } from './helpers/registry.js'; +import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js'; +import * as http from 'http'; +import * as url from 'url'; +import * as fs from 'fs'; +import * as path from 'path'; + +// Test context +let registry: SmartRegistry; +let server: http.Server; +let registryUrl: string; +let registryPort: number; +let composerToken: string; +let testDir: string; +let composerHome: string; + +/** + * Create HTTP server wrapper around SmartRegistry + */ +async function createHttpServer( + registryInstance: SmartRegistry, + port: number +): Promise<{ server: http.Server; url: string }> { + return new Promise((resolve, reject) => { + const httpServer = http.createServer(async (req, res) => { + try { + // Parse request + const parsedUrl = url.parse(req.url || '', true); + const pathname = parsedUrl.pathname || '/'; + const query = parsedUrl.query; + + // Read body + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const bodyBuffer = Buffer.concat(chunks); + + // Parse body based on content type + let body: any; + if (bodyBuffer.length > 0) { + const contentType = req.headers['content-type'] || ''; + if (contentType.includes('application/json')) { + try { + body = JSON.parse(bodyBuffer.toString('utf-8')); + } catch (error) { + body = bodyBuffer; + } + } else { + body = bodyBuffer; + } + } + + // Convert to IRequestContext + const context: IRequestContext = { + method: req.method || 'GET', + path: pathname, + headers: req.headers as Record, + query: query as Record, + body: body, + }; + + // Handle request + const response: IResponse = await registryInstance.handleRequest(context); + + // Convert IResponse to HTTP response + res.statusCode = response.status; + + // Set headers + for (const [key, value] of Object.entries(response.headers || {})) { + res.setHeader(key, value); + } + + // Send body + if (response.body) { + if (Buffer.isBuffer(response.body)) { + res.end(response.body); + } else if (typeof response.body === 'string') { + res.end(response.body); + } else { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(response.body)); + } + } else { + res.end(); + } + } catch (error) { + console.error('Server error:', error); + res.statusCode = 500; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) })); + } + }); + + httpServer.listen(port, () => { + const serverUrl = `http://localhost:${port}`; + resolve({ server: httpServer, url: serverUrl }); + }); + + httpServer.on('error', reject); + }); +} + +/** + * Setup Composer auth.json for authentication + */ +function setupComposerAuth( + token: string, + composerHomeArg: string, + serverUrl: string, + port: number +): string { + fs.mkdirSync(composerHomeArg, { recursive: true }); + + const authJson = { + 'http-basic': { + [`localhost:${port}`]: { + username: 'testuser', + password: token, + }, + }, + }; + + const authPath = path.join(composerHomeArg, 'auth.json'); + fs.writeFileSync(authPath, JSON.stringify(authJson, null, 2), 'utf-8'); + + return authPath; +} + +/** + * Create a Composer project that uses our registry + */ +function createComposerProject( + projectDir: string, + serverUrl: string +): void { + fs.mkdirSync(projectDir, { recursive: true }); + + const composerJson = { + name: 'test/consumer-project', + description: 'Test consumer project for Composer CLI tests', + type: 'project', + require: {}, + repositories: [ + { + type: 'composer', + url: `${serverUrl}/composer`, + }, + ], + config: { + 'secure-http': false, + }, + }; + + fs.writeFileSync( + path.join(projectDir, 'composer.json'), + JSON.stringify(composerJson, null, 2), + 'utf-8' + ); +} + +/** + * Run Composer command with custom home directory + */ +async function runComposerCommand( + command: string, + cwd: string +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const fullCommand = `cd "${cwd}" && COMPOSER_HOME="${composerHome}" composer ${command}`; + + try { + const result = await tapNodeTools.runCommand(fullCommand); + return { + stdout: result.stdout || '', + stderr: result.stderr || '', + exitCode: result.exitCode || 0, + }; + } catch (error: any) { + return { + stdout: error.stdout || '', + stderr: error.stderr || String(error), + exitCode: error.exitCode || 1, + }; + } +} + +/** + * Upload a Composer package via HTTP API + */ +async function uploadComposerPackage( + vendorPackage: string, + version: string, + token: string, + serverUrl: string +): Promise { + const zipData = await createComposerZip(vendorPackage, version); + + const response = await fetch(`${serverUrl}/composer/packages/${vendorPackage}`, { + method: 'PUT', + headers: { + 'Content-Type': 'application/zip', + Authorization: `Bearer ${token}`, + }, + body: zipData, + }); + + if (!response.ok) { + const body = await response.text(); + throw new Error(`Failed to upload package: ${response.status} ${body}`); + } +} + +/** + * Cleanup test directory + */ +function cleanupTestDir(dir: string): void { + if (fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +// ======================================================================== +// TESTS +// ======================================================================== + +tap.test('Composer CLI: should verify composer is installed', async () => { + try { + const result = await tapNodeTools.runCommand('composer --version'); + console.log('Composer version output:', result.stdout.substring(0, 200)); + expect(result.exitCode).toEqual(0); + } catch (error) { + console.log('Composer CLI not available, skipping native CLI tests'); + // Skip remaining tests if Composer is not installed + tap.skip.test('Composer CLI: remaining tests skipped - composer not available'); + return; + } +}); + +tap.test('Composer CLI: should setup registry and HTTP server', async () => { + // Create registry + registry = await createTestRegistry(); + const tokens = await createTestTokens(registry); + composerToken = tokens.composerToken; + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(composerToken).toBeTypeOf('string'); + + // Use port 38000 (avoids conflicts with other tests) + registryPort = 38000; + const serverSetup = await createHttpServer(registry, registryPort); + server = serverSetup.server; + registryUrl = serverSetup.url; + + expect(server).toBeDefined(); + expect(registryUrl).toEqual(`http://localhost:${registryPort}`); + + // Setup test directory + testDir = path.join(process.cwd(), '.nogit', 'test-composer-cli'); + cleanupTestDir(testDir); + fs.mkdirSync(testDir, { recursive: true }); + + // Setup COMPOSER_HOME directory + composerHome = path.join(testDir, '.composer'); + fs.mkdirSync(composerHome, { recursive: true }); + + // Setup Composer auth + const authPath = setupComposerAuth(composerToken, composerHome, registryUrl, registryPort); + expect(fs.existsSync(authPath)).toEqual(true); +}); + +tap.test('Composer CLI: should verify server is responding', async () => { + // Check server is up by doing a direct HTTP request + const response = await fetch(`${registryUrl}/composer/packages.json`); + expect(response.status).toBeGreaterThanOrEqual(200); + expect(response.status).toBeLessThan(500); +}); + +tap.test('Composer CLI: should upload a package via API', async () => { + const vendorPackage = 'testvendor/test-package'; + const version = '1.0.0'; + + await uploadComposerPackage(vendorPackage, version, composerToken, registryUrl); + + // Verify package exists via packages.json + const response = await fetch(`${registryUrl}/composer/packages.json`); + expect(response.status).toEqual(200); + + const packagesJson = await response.json(); + expect(packagesJson.packages).toBeDefined(); + expect(packagesJson.packages[vendorPackage]).toBeDefined(); +}); + +tap.test('Composer CLI: should require package from registry', async () => { + const projectDir = path.join(testDir, 'consumer-project'); + createComposerProject(projectDir, registryUrl); + + // Try to require the package we uploaded + const result = await runComposerCommand( + 'require testvendor/test-package:1.0.0 --no-interaction', + projectDir + ); + console.log('composer require output:', result.stdout); + console.log('composer require stderr:', result.stderr); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('Composer CLI: should verify package in vendor directory', async () => { + const projectDir = path.join(testDir, 'consumer-project'); + const packageDir = path.join(projectDir, 'vendor', 'testvendor', 'test-package'); + + expect(fs.existsSync(packageDir)).toEqual(true); + + // Check composer.json exists in package + const packageComposerPath = path.join(packageDir, 'composer.json'); + expect(fs.existsSync(packageComposerPath)).toEqual(true); +}); + +tap.test('Composer CLI: should upload second version', async () => { + const vendorPackage = 'testvendor/test-package'; + const version = '2.0.0'; + + await uploadComposerPackage(vendorPackage, version, composerToken, registryUrl); + + // Verify both versions exist + const response = await fetch(`${registryUrl}/composer/packages.json`); + const packagesJson = await response.json(); + + expect(packagesJson.packages[vendorPackage]['1.0.0']).toBeDefined(); + expect(packagesJson.packages[vendorPackage]['2.0.0']).toBeDefined(); +}); + +tap.test('Composer CLI: should update to new version', async () => { + const projectDir = path.join(testDir, 'consumer-project'); + + // Update to version 2.0.0 + const result = await runComposerCommand( + 'require testvendor/test-package:2.0.0 --no-interaction', + projectDir + ); + console.log('composer update output:', result.stdout); + + expect(result.exitCode).toEqual(0); + + // Verify composer.lock has the new version + const lockPath = path.join(projectDir, 'composer.lock'); + expect(fs.existsSync(lockPath)).toEqual(true); + + const lockContent = JSON.parse(fs.readFileSync(lockPath, 'utf-8')); + const pkg = lockContent.packages.find((p: any) => p.name === 'testvendor/test-package'); + expect(pkg?.version).toEqual('2.0.0'); +}); + +tap.test('Composer CLI: should search for packages', async () => { + const projectDir = path.join(testDir, 'consumer-project'); + + // Search for packages (may not work on all Composer versions) + const result = await runComposerCommand( + 'search testvendor --no-interaction 2>&1 || true', + projectDir + ); + console.log('composer search output:', result.stdout); + + // Search may or may not work depending on registry implementation + // Just verify it doesn't crash + expect(result.exitCode).toBeLessThanOrEqual(1); +}); + +tap.test('Composer CLI: should show package info', async () => { + const projectDir = path.join(testDir, 'consumer-project'); + + const result = await runComposerCommand( + 'show testvendor/test-package --no-interaction', + projectDir + ); + console.log('composer show output:', result.stdout); + + expect(result.exitCode).toEqual(0); + expect(result.stdout).toContain('testvendor/test-package'); +}); + +tap.test('Composer CLI: should remove package', async () => { + const projectDir = path.join(testDir, 'consumer-project'); + + const result = await runComposerCommand( + 'remove testvendor/test-package --no-interaction', + projectDir + ); + console.log('composer remove output:', result.stdout); + + expect(result.exitCode).toEqual(0); + + // Verify package is removed from vendor + const packageDir = path.join(projectDir, 'vendor', 'testvendor', 'test-package'); + expect(fs.existsSync(packageDir)).toEqual(false); +}); + +tap.postTask('cleanup composer cli tests', async () => { + // Stop server + if (server) { + await new Promise((resolve) => { + server.close(() => resolve()); + }); + } + + // Cleanup test directory + if (testDir) { + cleanupTestDir(testDir); + } + + // Destroy registry + if (registry) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/test/test.maven.nativecli.node.ts b/test/test.maven.nativecli.node.ts new file mode 100644 index 0000000..f32e3d6 --- /dev/null +++ b/test/test.maven.nativecli.node.ts @@ -0,0 +1,490 @@ +/** + * Native Maven CLI Testing + * Tests the Maven registry implementation using the actual mvn CLI + */ + +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside'; +import { SmartRegistry } from '../ts/index.js'; +import { createTestRegistry, createTestTokens, createTestPom, createTestJar } from './helpers/registry.js'; +import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js'; +import * as http from 'http'; +import * as url from 'url'; +import * as fs from 'fs'; +import * as path from 'path'; + +// Test context +let registry: SmartRegistry; +let server: http.Server; +let registryUrl: string; +let registryPort: number; +let mavenToken: string; +let testDir: string; +let m2Dir: string; + +/** + * Create HTTP server wrapper around SmartRegistry + */ +async function createHttpServer( + registryInstance: SmartRegistry, + port: number +): Promise<{ server: http.Server; url: string }> { + return new Promise((resolve, reject) => { + const httpServer = http.createServer(async (req, res) => { + try { + // Parse request + const parsedUrl = url.parse(req.url || '', true); + const pathname = parsedUrl.pathname || '/'; + const query = parsedUrl.query; + + // Read body + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const bodyBuffer = Buffer.concat(chunks); + + // Parse body based on content type + let body: any; + if (bodyBuffer.length > 0) { + const contentType = req.headers['content-type'] || ''; + if (contentType.includes('application/json')) { + try { + body = JSON.parse(bodyBuffer.toString('utf-8')); + } catch (error) { + body = bodyBuffer; + } + } else { + body = bodyBuffer; + } + } + + // Convert to IRequestContext + const context: IRequestContext = { + method: req.method || 'GET', + path: pathname, + headers: req.headers as Record, + query: query as Record, + body: body, + }; + + // Handle request + const response: IResponse = await registryInstance.handleRequest(context); + + // Convert IResponse to HTTP response + res.statusCode = response.status; + + // Set headers + for (const [key, value] of Object.entries(response.headers || {})) { + res.setHeader(key, value); + } + + // Send body + if (response.body) { + if (Buffer.isBuffer(response.body)) { + res.end(response.body); + } else if (typeof response.body === 'string') { + res.end(response.body); + } else { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(response.body)); + } + } else { + res.end(); + } + } catch (error) { + console.error('Server error:', error); + res.statusCode = 500; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) })); + } + }); + + httpServer.listen(port, () => { + const serverUrl = `http://localhost:${port}`; + resolve({ server: httpServer, url: serverUrl }); + }); + + httpServer.on('error', reject); + }); +} + +/** + * Setup Maven settings.xml for authentication + */ +function setupMavenSettings( + token: string, + m2DirArg: string, + serverUrl: string +): string { + fs.mkdirSync(m2DirArg, { recursive: true }); + + const settingsXml = ` + + + + test-registry + testuser + ${token} + + + + + test-registry + + + test-registry + ${serverUrl}/maven + + true + + + true + + + + + + + test-registry + + +`; + + const settingsPath = path.join(m2DirArg, 'settings.xml'); + fs.writeFileSync(settingsPath, settingsXml, 'utf-8'); + + return settingsPath; +} + +/** + * Create a minimal Maven project for testing + */ +function createMavenProject( + projectDir: string, + groupId: string, + artifactId: string, + version: string, + registryUrl: string +): void { + fs.mkdirSync(projectDir, { recursive: true }); + + const pomXml = ` + + 4.0.0 + ${groupId} + ${artifactId} + ${version} + jar + ${artifactId} + Test Maven project for SmartRegistry CLI tests + + + + test-registry + ${registryUrl}/maven + + + test-registry + ${registryUrl}/maven + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + + + + + +`; + + fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8'); + + // Create minimal Java source + const srcDir = path.join(projectDir, 'src', 'main', 'java', 'com', 'test'); + fs.mkdirSync(srcDir, { recursive: true }); + + const javaSource = `package com.test; + +public class Main { + public static void main(String[] args) { + System.out.println("Hello from SmartRegistry test!"); + } +} +`; + fs.writeFileSync(path.join(srcDir, 'Main.java'), javaSource, 'utf-8'); +} + +/** + * Run Maven command with custom settings + */ +async function runMavenCommand( + command: string, + cwd: string +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const settingsPath = path.join(m2Dir, 'settings.xml'); + const fullCommand = `cd "${cwd}" && mvn -s "${settingsPath}" ${command}`; + + try { + const result = await tapNodeTools.runCommand(fullCommand); + return { + stdout: result.stdout || '', + stderr: result.stderr || '', + exitCode: result.exitCode || 0, + }; + } catch (error: any) { + return { + stdout: error.stdout || '', + stderr: error.stderr || String(error), + exitCode: error.exitCode || 1, + }; + } +} + +/** + * Cleanup test directory + */ +function cleanupTestDir(dir: string): void { + if (fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +// ======================================================================== +// TESTS +// ======================================================================== + +tap.test('Maven CLI: should verify mvn is installed', async () => { + try { + const result = await tapNodeTools.runCommand('mvn -version'); + console.log('Maven version output:', result.stdout.substring(0, 200)); + expect(result.exitCode).toEqual(0); + } catch (error) { + console.log('Maven CLI not available, skipping native CLI tests'); + // Skip remaining tests if Maven is not installed + tap.skip.test('Maven CLI: remaining tests skipped - mvn not available'); + return; + } +}); + +tap.test('Maven CLI: should setup registry and HTTP server', async () => { + // Create registry + registry = await createTestRegistry(); + const tokens = await createTestTokens(registry); + mavenToken = tokens.mavenToken; + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(mavenToken).toBeTypeOf('string'); + + // Use port 37000 (avoids conflicts with other tests) + registryPort = 37000; + const serverSetup = await createHttpServer(registry, registryPort); + server = serverSetup.server; + registryUrl = serverSetup.url; + + expect(server).toBeDefined(); + expect(registryUrl).toEqual(`http://localhost:${registryPort}`); + + // Setup test directory + testDir = path.join(process.cwd(), '.nogit', 'test-maven-cli'); + cleanupTestDir(testDir); + fs.mkdirSync(testDir, { recursive: true }); + + // Setup .m2 directory + m2Dir = path.join(testDir, '.m2'); + fs.mkdirSync(m2Dir, { recursive: true }); + + // Setup Maven settings + const settingsPath = setupMavenSettings(mavenToken, m2Dir, registryUrl); + expect(fs.existsSync(settingsPath)).toEqual(true); +}); + +tap.test('Maven CLI: should verify server is responding', async () => { + // Check server is up by doing a direct HTTP request + const response = await fetch(`${registryUrl}/maven/`); + expect(response.status).toBeGreaterThanOrEqual(200); + expect(response.status).toBeLessThan(500); +}); + +tap.test('Maven CLI: should deploy a JAR artifact', async () => { + const groupId = 'com.test'; + const artifactId = 'test-artifact'; + const version = '1.0.0'; + + const projectDir = path.join(testDir, 'test-project'); + createMavenProject(projectDir, groupId, artifactId, version, registryUrl); + + // Build and deploy + const result = await runMavenCommand('clean package deploy -DskipTests', projectDir); + console.log('mvn deploy output:', result.stdout.substring(0, 500)); + console.log('mvn deploy stderr:', result.stderr.substring(0, 500)); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('Maven CLI: should verify artifact in registry via API', async () => { + const groupId = 'com.test'; + const artifactId = 'test-artifact'; + const version = '1.0.0'; + + // Maven path: /maven/{groupId path}/{artifactId}/{version}/{artifactId}-{version}.jar + const jarPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`; + const response = await fetch(`${registryUrl}${jarPath}`, { + headers: { Authorization: `Bearer ${mavenToken}` }, + }); + + expect(response.status).toEqual(200); + + const jarData = await response.arrayBuffer(); + expect(jarData.byteLength).toBeGreaterThan(0); +}); + +tap.test('Maven CLI: should verify POM in registry', async () => { + const groupId = 'com.test'; + const artifactId = 'test-artifact'; + const version = '1.0.0'; + + const pomPath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.pom`; + const response = await fetch(`${registryUrl}${pomPath}`, { + headers: { Authorization: `Bearer ${mavenToken}` }, + }); + + expect(response.status).toEqual(200); + + const pomContent = await response.text(); + expect(pomContent).toContain(groupId); + expect(pomContent).toContain(artifactId); + expect(pomContent).toContain(version); +}); + +tap.test('Maven CLI: should verify checksums exist', async () => { + const artifactId = 'test-artifact'; + const version = '1.0.0'; + + // Check JAR checksums + const basePath = `/maven/com/test/${artifactId}/${version}/${artifactId}-${version}.jar`; + + // MD5 + const md5Response = await fetch(`${registryUrl}${basePath}.md5`, { + headers: { Authorization: `Bearer ${mavenToken}` }, + }); + expect(md5Response.status).toEqual(200); + + // SHA1 + const sha1Response = await fetch(`${registryUrl}${basePath}.sha1`, { + headers: { Authorization: `Bearer ${mavenToken}` }, + }); + expect(sha1Response.status).toEqual(200); +}); + +tap.test('Maven CLI: should deploy second version', async () => { + const groupId = 'com.test'; + const artifactId = 'test-artifact'; + const version = '2.0.0'; + + const projectDir = path.join(testDir, 'test-project-v2'); + createMavenProject(projectDir, groupId, artifactId, version, registryUrl); + + const result = await runMavenCommand('clean package deploy -DskipTests', projectDir); + console.log('mvn deploy v2 output:', result.stdout.substring(0, 500)); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('Maven CLI: should verify metadata.xml exists', async () => { + const artifactId = 'test-artifact'; + + // Maven metadata is stored at /maven/{groupId path}/{artifactId}/maven-metadata.xml + const metadataPath = `/maven/com/test/${artifactId}/maven-metadata.xml`; + const response = await fetch(`${registryUrl}${metadataPath}`, { + headers: { Authorization: `Bearer ${mavenToken}` }, + }); + + expect(response.status).toEqual(200); + + const metadataXml = await response.text(); + expect(metadataXml).toContain(artifactId); + expect(metadataXml).toContain('1.0.0'); + expect(metadataXml).toContain('2.0.0'); +}); + +tap.test('Maven CLI: should resolve dependency from registry', async () => { + const groupId = 'com.consumer'; + const artifactId = 'consumer-app'; + const version = '1.0.0'; + + const projectDir = path.join(testDir, 'consumer-project'); + fs.mkdirSync(projectDir, { recursive: true }); + + // Create a consumer project that depends on our test artifact + const pomXml = ` + + 4.0.0 + ${groupId} + ${artifactId} + ${version} + jar + + + + test-registry + ${registryUrl}/maven + + + + + + com.test + test-artifact + 1.0.0 + + + +`; + + fs.writeFileSync(path.join(projectDir, 'pom.xml'), pomXml, 'utf-8'); + + // Try to resolve dependencies + const result = await runMavenCommand('dependency:resolve', projectDir); + console.log('mvn dependency:resolve output:', result.stdout.substring(0, 500)); + + expect(result.exitCode).toEqual(0); +}); + +tap.postTask('cleanup maven cli tests', async () => { + // Stop server + if (server) { + await new Promise((resolve) => { + server.close(() => resolve()); + }); + } + + // Cleanup test directory + if (testDir) { + cleanupTestDir(testDir); + } + + // Destroy registry + if (registry) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/test/test.pypi.nativecli.node.ts b/test/test.pypi.nativecli.node.ts new file mode 100644 index 0000000..73a0d03 --- /dev/null +++ b/test/test.pypi.nativecli.node.ts @@ -0,0 +1,522 @@ +/** + * Native PyPI CLI Testing + * Tests the PyPI registry implementation using pip and twine CLI tools + */ + +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside'; +import { SmartRegistry } from '../ts/index.js'; +import { createTestRegistry, createTestTokens, createPythonWheel, createPythonSdist } from './helpers/registry.js'; +import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js'; +import * as http from 'http'; +import * as url from 'url'; +import * as fs from 'fs'; +import * as path from 'path'; + +// Test context +let registry: SmartRegistry; +let server: http.Server; +let registryUrl: string; +let registryPort: number; +let pypiToken: string; +let testDir: string; +let pipHome: string; +let hasPip = false; +let hasTwine = false; + +/** + * Create HTTP server wrapper around SmartRegistry + */ +async function createHttpServer( + registryInstance: SmartRegistry, + port: number +): Promise<{ server: http.Server; url: string }> { + return new Promise((resolve, reject) => { + const httpServer = http.createServer(async (req, res) => { + try { + // Parse request + const parsedUrl = url.parse(req.url || '', true); + const pathname = parsedUrl.pathname || '/'; + const query = parsedUrl.query; + + // Read body + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const bodyBuffer = Buffer.concat(chunks); + + // Parse body based on content type + let body: any; + if (bodyBuffer.length > 0) { + const contentType = req.headers['content-type'] || ''; + if (contentType.includes('application/json')) { + try { + body = JSON.parse(bodyBuffer.toString('utf-8')); + } catch (error) { + body = bodyBuffer; + } + } else if (contentType.includes('multipart/form-data')) { + // For multipart, pass raw buffer + body = bodyBuffer; + } else { + body = bodyBuffer; + } + } + + // Convert to IRequestContext + const context: IRequestContext = { + method: req.method || 'GET', + path: pathname, + headers: req.headers as Record, + query: query as Record, + body: body, + rawBody: bodyBuffer, + }; + + // Handle request + const response: IResponse = await registryInstance.handleRequest(context); + + // Convert IResponse to HTTP response + res.statusCode = response.status; + + // Set headers + for (const [key, value] of Object.entries(response.headers || {})) { + res.setHeader(key, value); + } + + // Send body + if (response.body) { + if (Buffer.isBuffer(response.body)) { + res.end(response.body); + } else if (typeof response.body === 'string') { + res.end(response.body); + } else { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(response.body)); + } + } else { + res.end(); + } + } catch (error) { + console.error('Server error:', error); + res.statusCode = 500; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) })); + } + }); + + httpServer.listen(port, () => { + const serverUrl = `http://localhost:${port}`; + resolve({ server: httpServer, url: serverUrl }); + }); + + httpServer.on('error', reject); + }); +} + +/** + * Setup .pypirc for twine authentication + */ +function setupPypirc( + token: string, + pipHomeArg: string, + serverUrl: string +): string { + fs.mkdirSync(pipHomeArg, { recursive: true }); + + const pypircContent = `[distutils] +index-servers = testpypi + +[testpypi] +repository = ${serverUrl}/pypi +username = testuser +password = ${token} +`; + + const pypircPath = path.join(pipHomeArg, '.pypirc'); + fs.writeFileSync(pypircPath, pypircContent, 'utf-8'); + + // Set restrictive permissions + fs.chmodSync(pypircPath, 0o600); + + return pypircPath; +} + +/** + * Setup pip.conf for pip to use our registry + */ +function setupPipConf( + token: string, + pipHomeArg: string, + serverUrl: string, + port: number +): string { + fs.mkdirSync(pipHomeArg, { recursive: true }); + + // pip.conf with authentication + const pipConfContent = `[global] +index-url = ${serverUrl}/pypi/simple/ +trusted-host = localhost +extra-index-url = https://pypi.org/simple/ +`; + + const pipDir = path.join(pipHomeArg, 'pip'); + fs.mkdirSync(pipDir, { recursive: true }); + + const pipConfPath = path.join(pipDir, 'pip.conf'); + fs.writeFileSync(pipConfPath, pipConfContent, 'utf-8'); + + return pipConfPath; +} + +/** + * Create a test Python package wheel file + */ +async function createTestWheelFile( + packageName: string, + version: string, + targetDir: string +): Promise { + const wheelData = await createPythonWheel(packageName, version); + const normalizedName = packageName.replace(/-/g, '_'); + const wheelFilename = `${normalizedName}-${version}-py3-none-any.whl`; + const wheelPath = path.join(targetDir, wheelFilename); + + fs.writeFileSync(wheelPath, wheelData); + + return wheelPath; +} + +/** + * Create a test Python package sdist file + */ +async function createTestSdistFile( + packageName: string, + version: string, + targetDir: string +): Promise { + const sdistData = await createPythonSdist(packageName, version); + const sdistFilename = `${packageName}-${version}.tar.gz`; + const sdistPath = path.join(targetDir, sdistFilename); + + fs.writeFileSync(sdistPath, sdistData); + + return sdistPath; +} + +/** + * Run pip command with custom config + */ +async function runPipCommand( + command: string, + cwd: string +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const pipConfDir = path.join(pipHome, 'pip'); + const fullCommand = `cd "${cwd}" && PIP_CONFIG_FILE="${path.join(pipConfDir, 'pip.conf')}" pip ${command}`; + + try { + const result = await tapNodeTools.runCommand(fullCommand); + return { + stdout: result.stdout || '', + stderr: result.stderr || '', + exitCode: result.exitCode || 0, + }; + } catch (error: any) { + return { + stdout: error.stdout || '', + stderr: error.stderr || String(error), + exitCode: error.exitCode || 1, + }; + } +} + +/** + * Run twine command with custom config + */ +async function runTwineCommand( + command: string, + cwd: string +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const pypircPath = path.join(pipHome, '.pypirc'); + const fullCommand = `cd "${cwd}" && twine ${command} --config-file "${pypircPath}"`; + + try { + const result = await tapNodeTools.runCommand(fullCommand); + return { + stdout: result.stdout || '', + stderr: result.stderr || '', + exitCode: result.exitCode || 0, + }; + } catch (error: any) { + return { + stdout: error.stdout || '', + stderr: error.stderr || String(error), + exitCode: error.exitCode || 1, + }; + } +} + +/** + * Cleanup test directory + */ +function cleanupTestDir(dir: string): void { + if (fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +// ======================================================================== +// TESTS +// ======================================================================== + +tap.test('PyPI CLI: should verify pip is installed', async () => { + try { + const result = await tapNodeTools.runCommand('pip --version'); + console.log('pip version output:', result.stdout.substring(0, 200)); + hasPip = result.exitCode === 0; + expect(result.exitCode).toEqual(0); + } catch (error) { + console.log('pip CLI not available'); + } +}); + +tap.test('PyPI CLI: should verify twine is installed', async () => { + try { + const result = await tapNodeTools.runCommand('twine --version'); + console.log('twine version output:', result.stdout.substring(0, 200)); + hasTwine = result.exitCode === 0; + expect(result.exitCode).toEqual(0); + } catch (error) { + console.log('twine CLI not available'); + } + + if (!hasPip && !hasTwine) { + console.log('Neither pip nor twine available, skipping native CLI tests'); + tap.skip.test('PyPI CLI: remaining tests skipped - no CLI tools available'); + return; + } +}); + +tap.test('PyPI CLI: should setup registry and HTTP server', async () => { + // Create registry + registry = await createTestRegistry(); + const tokens = await createTestTokens(registry); + pypiToken = tokens.pypiToken; + + expect(registry).toBeInstanceOf(SmartRegistry); + expect(pypiToken).toBeTypeOf('string'); + + // Use port 39000 (avoids conflicts with other tests) + registryPort = 39000; + const serverSetup = await createHttpServer(registry, registryPort); + server = serverSetup.server; + registryUrl = serverSetup.url; + + expect(server).toBeDefined(); + expect(registryUrl).toEqual(`http://localhost:${registryPort}`); + + // Setup test directory + testDir = path.join(process.cwd(), '.nogit', 'test-pypi-cli'); + cleanupTestDir(testDir); + fs.mkdirSync(testDir, { recursive: true }); + + // Setup pip/pypi home directory + pipHome = path.join(testDir, '.pip'); + fs.mkdirSync(pipHome, { recursive: true }); + + // Setup .pypirc for twine + const pypircPath = setupPypirc(pypiToken, pipHome, registryUrl); + expect(fs.existsSync(pypircPath)).toEqual(true); + + // Setup pip.conf + const pipConfPath = setupPipConf(pypiToken, pipHome, registryUrl, registryPort); + expect(fs.existsSync(pipConfPath)).toEqual(true); +}); + +tap.test('PyPI CLI: should verify server is responding', async () => { + // Check server is up by doing a direct HTTP request to simple index + const response = await fetch(`${registryUrl}/pypi/simple/`); + expect(response.status).toBeGreaterThanOrEqual(200); + expect(response.status).toBeLessThan(500); +}); + +tap.test('PyPI CLI: should upload wheel with twine', async () => { + if (!hasTwine) { + console.log('Skipping twine test - twine not available'); + return; + } + + const packageName = 'test-pypi-pkg'; + const version = '1.0.0'; + const wheelPath = await createTestWheelFile(packageName, version, testDir); + + expect(fs.existsSync(wheelPath)).toEqual(true); + + const result = await runTwineCommand( + `upload --repository testpypi "${wheelPath}"`, + testDir + ); + console.log('twine upload output:', result.stdout); + console.log('twine upload stderr:', result.stderr); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('PyPI CLI: should verify package in simple index', async () => { + if (!hasTwine) { + console.log('Skipping - twine not available'); + return; + } + + const packageName = 'test-pypi-pkg'; + + const response = await fetch(`${registryUrl}/pypi/simple/${packageName}/`); + expect(response.status).toEqual(200); + + const html = await response.text(); + expect(html).toContain('1.0.0'); +}); + +tap.test('PyPI CLI: should upload sdist with twine', async () => { + if (!hasTwine) { + console.log('Skipping twine test - twine not available'); + return; + } + + const packageName = 'test-pypi-pkg'; + const version = '1.1.0'; + const sdistPath = await createTestSdistFile(packageName, version, testDir); + + expect(fs.existsSync(sdistPath)).toEqual(true); + + const result = await runTwineCommand( + `upload --repository testpypi "${sdistPath}"`, + testDir + ); + console.log('twine upload sdist output:', result.stdout); + console.log('twine upload sdist stderr:', result.stderr); + + expect(result.exitCode).toEqual(0); +}); + +tap.test('PyPI CLI: should list all versions in simple index', async () => { + if (!hasTwine) { + console.log('Skipping - twine not available'); + return; + } + + const packageName = 'test-pypi-pkg'; + + const response = await fetch(`${registryUrl}/pypi/simple/${packageName}/`); + expect(response.status).toEqual(200); + + const html = await response.text(); + expect(html).toContain('1.0.0'); + expect(html).toContain('1.1.0'); +}); + +tap.test('PyPI CLI: should get JSON metadata', async () => { + if (!hasTwine) { + console.log('Skipping - twine not available'); + return; + } + + const packageName = 'test-pypi-pkg'; + + const response = await fetch(`${registryUrl}/pypi/pypi/${packageName}/json`); + expect(response.status).toEqual(200); + + const metadata = await response.json(); + expect(metadata.info).toBeDefined(); + expect(metadata.info.name).toEqual(packageName); + expect(metadata.releases).toBeDefined(); + expect(metadata.releases['1.0.0']).toBeDefined(); +}); + +tap.test('PyPI CLI: should download package with pip', async () => { + if (!hasPip || !hasTwine) { + console.log('Skipping pip download test - pip or twine not available'); + return; + } + + const downloadDir = path.join(testDir, 'downloads'); + fs.mkdirSync(downloadDir, { recursive: true }); + + // Download (not install) the package + const result = await runPipCommand( + `download test-pypi-pkg==1.0.0 --dest "${downloadDir}" --no-deps`, + testDir + ); + console.log('pip download output:', result.stdout); + console.log('pip download stderr:', result.stderr); + + // pip download may fail if the package doesn't meet pip's requirements + // Just check it doesn't crash + expect(result.exitCode).toBeLessThanOrEqual(1); +}); + +tap.test('PyPI CLI: should search for packages via API', async () => { + const packageName = 'test-pypi-pkg'; + + // Use the JSON API to search/list + const response = await fetch(`${registryUrl}/pypi/pypi/${packageName}/json`); + expect(response.status).toEqual(200); + + const metadata = await response.json(); + expect(metadata.info.name).toEqual(packageName); +}); + +tap.test('PyPI CLI: should fail upload without auth', async () => { + if (!hasTwine) { + console.log('Skipping twine test - twine not available'); + return; + } + + const packageName = 'unauth-pkg'; + const version = '1.0.0'; + const wheelPath = await createTestWheelFile(packageName, version, testDir); + + // Create a pypirc without proper credentials + const badPypircPath = path.join(testDir, '.bad-pypirc'); + fs.writeFileSync(badPypircPath, `[distutils] +index-servers = badpypi + +[badpypi] +repository = ${registryUrl}/pypi +username = baduser +password = badtoken +`, 'utf-8'); + + const fullCommand = `cd "${testDir}" && twine upload --repository badpypi "${wheelPath}" --config-file "${badPypircPath}"`; + + try { + const result = await tapNodeTools.runCommand(fullCommand); + // Should fail + expect(result.exitCode).not.toEqual(0); + } catch (error: any) { + // Expected to fail + expect(error.exitCode || 1).not.toEqual(0); + } +}); + +tap.postTask('cleanup pypi cli tests', async () => { + // Stop server + if (server) { + await new Promise((resolve) => { + server.close(() => resolve()); + }); + } + + // Cleanup test directory + if (testDir) { + cleanupTestDir(testDir); + } + + // Destroy registry + if (registry) { + registry.destroy(); + } +}); + +export default tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 597d64f..cf76dca 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartregistry', - version: '2.2.0', + version: '2.2.1', description: 'A composable TypeScript library implementing OCI, NPM, Maven, Cargo, Composer, PyPI, and RubyGems registries for building unified container and package registries' } diff --git a/ts/composer/classes.composerregistry.ts b/ts/composer/classes.composerregistry.ts index 34f524f..fdb657f 100644 --- a/ts/composer/classes.composerregistry.ts +++ b/ts/composer/classes.composerregistry.ts @@ -7,6 +7,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { AuthManager } from '../core/classes.authmanager.js'; import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; +import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import type { IComposerPackage, IComposerPackageMetadata, @@ -255,7 +256,7 @@ export class ComposerRegistry extends BaseRegistry { }; } - if (!body || !Buffer.isBuffer(body)) { + if (!body || !isBinaryData(body)) { return { status: 400, headers: {}, @@ -263,8 +264,11 @@ export class ComposerRegistry extends BaseRegistry { }; } + // Convert to Buffer for ZIP processing + const zipData = toBuffer(body); + // Extract and validate composer.json from ZIP - const composerJson = await extractComposerJsonFromZip(body); + const composerJson = await extractComposerJsonFromZip(zipData); if (!composerJson || !validateComposerJson(composerJson)) { return { status: 400, @@ -292,13 +296,13 @@ export class ComposerRegistry extends BaseRegistry { } // Calculate SHA-1 hash - const shasum = await calculateSha1(body); + const shasum = await calculateSha1(zipData); // Generate reference (use version or commit hash) const reference = composerJson.source?.reference || version.replace(/[^a-zA-Z0-9.-]/g, '-'); // Store ZIP file - await this.storage.putComposerPackageZip(vendorPackage, reference, body); + await this.storage.putComposerPackageZip(vendorPackage, reference, zipData); // Get or create metadata let metadata = await this.storage.getComposerPackageMetadata(vendorPackage); diff --git a/ts/core/helpers.buffer.ts b/ts/core/helpers.buffer.ts new file mode 100644 index 0000000..45911a9 --- /dev/null +++ b/ts/core/helpers.buffer.ts @@ -0,0 +1,34 @@ +/** + * Shared buffer utilities for consistent binary data handling across all registry types. + * + * This module addresses the common issue where `Buffer.isBuffer(Uint8Array)` returns `false`, + * which can cause data handling bugs when binary data arrives as Uint8Array instead of Buffer. + */ + +/** + * Check if value is binary data (Buffer or Uint8Array) + */ +export function isBinaryData(value: unknown): value is Buffer | Uint8Array { + return Buffer.isBuffer(value) || value instanceof Uint8Array; +} + +/** + * Convert any binary-like data to Buffer. + * Handles Buffer, Uint8Array, string, and objects. + * + * @param data - The data to convert to Buffer + * @returns A Buffer containing the data + */ +export function toBuffer(data: unknown): Buffer { + if (Buffer.isBuffer(data)) { + return data; + } + if (data instanceof Uint8Array) { + return Buffer.from(data); + } + if (typeof data === 'string') { + return Buffer.from(data, 'utf-8'); + } + // Fallback: serialize object to JSON + return Buffer.from(JSON.stringify(data)); +} diff --git a/ts/maven/classes.mavenregistry.ts b/ts/maven/classes.mavenregistry.ts index f717b53..221a592 100644 --- a/ts/maven/classes.mavenregistry.ts +++ b/ts/maven/classes.mavenregistry.ts @@ -7,6 +7,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js'; import type { RegistryStorage } from '../core/classes.registrystorage.js'; import type { AuthManager } from '../core/classes.authmanager.js'; import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; +import { toBuffer } from '../core/helpers.buffer.js'; import type { IMavenCoordinate, IMavenMetadata, IChecksums } from './interfaces.maven.js'; import { pathToGAV, @@ -296,7 +297,7 @@ export class MavenRegistry extends BaseRegistry { coordinate: IMavenCoordinate, body: Buffer | any ): Promise { - const data = Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body)); + const data = toBuffer(body); // Validate POM if uploading .pom file if (coordinate.extension === 'pom') { diff --git a/ts/oci/classes.ociregistry.ts b/ts/oci/classes.ociregistry.ts index 2ed5676..ac8d9ee 100644 --- a/ts/oci/classes.ociregistry.ts +++ b/ts/oci/classes.ociregistry.ts @@ -738,7 +738,7 @@ export class OciRegistry extends BaseRegistry { } private generateUploadId(): string { - return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`; } private async calculateDigest(data: Buffer): Promise { diff --git a/ts/pypi/classes.pypiregistry.ts b/ts/pypi/classes.pypiregistry.ts index b20b668..94edf87 100644 --- a/ts/pypi/classes.pypiregistry.ts +++ b/ts/pypi/classes.pypiregistry.ts @@ -3,6 +3,7 @@ import { BaseRegistry } from '../core/classes.baseregistry.js'; import { RegistryStorage } from '../core/classes.registrystorage.js'; import { AuthManager } from '../core/classes.authmanager.js'; import type { IRequestContext, IResponse, IAuthToken } from '../core/interfaces.core.js'; +import { isBinaryData, toBuffer } from '../core/helpers.buffer.js'; import type { IPypiPackageMetadata, IPypiFile, @@ -328,8 +329,9 @@ export class PypiRegistry extends BaseRegistry { const version = formData.version; // Support both: formData.content.filename (multipart parsed) and formData.filename (flat) const filename = formData.content?.filename || formData.filename; - // Support both: formData.content.data (multipart parsed) and formData.content (Buffer directly) - const fileData = (formData.content?.data || (Buffer.isBuffer(formData.content) ? formData.content : null)) as Buffer; + // Support both: formData.content.data (multipart parsed) and formData.content (Buffer/Uint8Array directly) + const rawContent = formData.content?.data || (isBinaryData(formData.content) ? formData.content : null); + const fileData = rawContent ? toBuffer(rawContent) : null; const filetype = formData.filetype; // 'bdist_wheel' or 'sdist' const pyversion = formData.pyversion;