feat(auth): Implement HMAC-SHA256 OCI JWTs; enhance PyPI & RubyGems uploads and normalize responses

This commit is contained in:
2025-11-25 14:28:19 +00:00
parent 2d6059ba7f
commit 547c262578
14 changed files with 765 additions and 158 deletions

View File

@@ -1,5 +1,6 @@
import * as qenv from '@push.rocks/qenv';
import * as crypto from 'crypto';
import * as smartarchive from '@push.rocks/smartarchive';
import { SmartRegistry } from '../../ts/classes.smartregistry.js';
import type { IRegistryConfig } from '../../ts/core/interfaces.core.js';
@@ -241,7 +242,7 @@ export function calculateMavenChecksums(data: Buffer) {
}
/**
* Helper to create a Composer package ZIP
* Helper to create a Composer package ZIP using smartarchive
*/
export async function createComposerZip(
vendorPackage: string,
@@ -252,8 +253,7 @@ export async function createComposerZip(
authors?: Array<{ name: string; email?: string }>;
}
): Promise<Buffer> {
const AdmZip = (await import('adm-zip')).default;
const zip = new AdmZip();
const zipTools = new smartarchive.ZipTools();
const composerJson = {
name: vendorPackage,
@@ -272,9 +272,6 @@ export async function createComposerZip(
},
};
// Add composer.json
zip.addFile('composer.json', Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'));
// Add a test PHP file
const [vendor, pkg] = vendorPackage.split('/');
const namespace = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}\\${pkg.charAt(0).toUpperCase() + pkg.slice(1).replace(/-/g, '')}`;
@@ -290,24 +287,33 @@ class TestClass
}
`;
zip.addFile('src/TestClass.php', Buffer.from(testPhpContent, 'utf-8'));
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'composer.json',
content: Buffer.from(JSON.stringify(composerJson, null, 2), 'utf-8'),
},
{
archivePath: 'src/TestClass.php',
content: Buffer.from(testPhpContent, 'utf-8'),
},
{
archivePath: 'README.md',
content: Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'),
},
];
// Add README
zip.addFile('README.md', Buffer.from(`# ${vendorPackage}\n\nTest package`, 'utf-8'));
return zip.toBuffer();
return zipTools.createZip(entries);
}
/**
* Helper to create a test Python wheel file (minimal ZIP structure)
* Helper to create a test Python wheel file (minimal ZIP structure) using smartarchive
*/
export async function createPythonWheel(
packageName: string,
version: string,
pyVersion: string = 'py3'
): Promise<Buffer> {
const AdmZip = (await import('adm-zip')).default;
const zip = new AdmZip();
const zipTools = new smartarchive.ZipTools();
const normalizedName = packageName.replace(/-/g, '_');
const distInfoDir = `${normalizedName}-${version}.dist-info`;
@@ -331,8 +337,6 @@ Description-Content-Type: text/markdown
Test package for SmartRegistry
`;
zip.addFile(`${distInfoDir}/METADATA`, Buffer.from(metadata, 'utf-8'));
// Create WHEEL file
const wheelContent = `Wheel-Version: 1.0
Generator: test 1.0.0
@@ -340,14 +344,6 @@ Root-Is-Purelib: true
Tag: ${pyVersion}-none-any
`;
zip.addFile(`${distInfoDir}/WHEEL`, Buffer.from(wheelContent, 'utf-8'));
// Create RECORD file (empty for test)
zip.addFile(`${distInfoDir}/RECORD`, Buffer.from('', 'utf-8'));
// Create top_level.txt
zip.addFile(`${distInfoDir}/top_level.txt`, Buffer.from(normalizedName, 'utf-8'));
// Create a simple Python module
const moduleContent = `"""${packageName} module"""
@@ -357,27 +353,44 @@ def hello():
return "Hello from ${packageName}!"
`;
zip.addFile(`${normalizedName}/__init__.py`, Buffer.from(moduleContent, 'utf-8'));
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${distInfoDir}/METADATA`,
content: Buffer.from(metadata, 'utf-8'),
},
{
archivePath: `${distInfoDir}/WHEEL`,
content: Buffer.from(wheelContent, 'utf-8'),
},
{
archivePath: `${distInfoDir}/RECORD`,
content: Buffer.from('', 'utf-8'),
},
{
archivePath: `${distInfoDir}/top_level.txt`,
content: Buffer.from(normalizedName, 'utf-8'),
},
{
archivePath: `${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
return zip.toBuffer();
return zipTools.createZip(entries);
}
/**
* Helper to create a test Python source distribution (sdist)
* Helper to create a test Python source distribution (sdist) using smartarchive
*/
export async function createPythonSdist(
packageName: string,
version: string
): Promise<Buffer> {
const tar = await import('tar-stream');
const zlib = await import('zlib');
const { Readable } = await import('stream');
const tarTools = new smartarchive.TarTools();
const normalizedName = packageName.replace(/-/g, '_');
const dirPrefix = `${packageName}-${version}`;
const pack = tar.pack();
// PKG-INFO
const pkgInfo = `Metadata-Version: 2.1
Name: ${packageName}
@@ -389,8 +402,6 @@ Author-email: test@example.com
License: MIT
`;
pack.entry({ name: `${dirPrefix}/PKG-INFO` }, pkgInfo);
// setup.py
const setupPy = `from setuptools import setup, find_packages
@@ -402,8 +413,6 @@ setup(
)
`;
pack.entry({ name: `${dirPrefix}/setup.py` }, setupPy);
// Module file
const moduleContent = `"""${packageName} module"""
@@ -413,20 +422,22 @@ def hello():
return "Hello from ${packageName}!"
`;
pack.entry({ name: `${dirPrefix}/${normalizedName}/__init__.py` }, moduleContent);
const entries: smartarchive.IArchiveEntry[] = [
{
archivePath: `${dirPrefix}/PKG-INFO`,
content: Buffer.from(pkgInfo, 'utf-8'),
},
{
archivePath: `${dirPrefix}/setup.py`,
content: Buffer.from(setupPy, 'utf-8'),
},
{
archivePath: `${dirPrefix}/${normalizedName}/__init__.py`,
content: Buffer.from(moduleContent, 'utf-8'),
},
];
pack.finalize();
// Convert to gzipped tar
const chunks: Buffer[] = [];
const gzip = zlib.createGzip();
return new Promise((resolve, reject) => {
pack.pipe(gzip);
gzip.on('data', (chunk) => chunks.push(chunk));
gzip.on('end', () => resolve(Buffer.concat(chunks)));
gzip.on('error', reject);
});
return tarTools.packFilesToTarGz(entries);
}
/**
@@ -441,17 +452,15 @@ export function calculatePypiHashes(data: Buffer) {
}
/**
* Helper to create a test RubyGem file (minimal tar.gz structure)
* Helper to create a test RubyGem file (minimal tar.gz structure) using smartarchive
*/
export async function createRubyGem(
gemName: string,
version: string,
platform: string = 'ruby'
): Promise<Buffer> {
const tar = await import('tar-stream');
const zlib = await import('zlib');
const pack = tar.pack();
const tarTools = new smartarchive.TarTools();
const gzipTools = new smartarchive.GzipTools();
// Create metadata.gz (simplified)
const metadataYaml = `--- !ruby/object:Gem::Specification
@@ -499,10 +508,9 @@ summary: Test gem for SmartRegistry
test_files: []
`;
pack.entry({ name: 'metadata.gz' }, zlib.gzipSync(Buffer.from(metadataYaml, 'utf-8')));
const metadataGz = await gzipTools.compress(Buffer.from(metadataYaml, 'utf-8'));
// Create data.tar.gz (simplified)
const dataPack = tar.pack();
// Create data.tar.gz content
const libContent = `# ${gemName}
module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
@@ -514,32 +522,28 @@ module ${gemName.charAt(0).toUpperCase() + gemName.slice(1).replace(/-/g, '')}
end
`;
dataPack.entry({ name: `lib/${gemName}.rb` }, libContent);
dataPack.finalize();
const dataEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: `lib/${gemName}.rb`,
content: Buffer.from(libContent, 'utf-8'),
},
];
const dataChunks: Buffer[] = [];
const dataGzip = zlib.createGzip();
dataPack.pipe(dataGzip);
const dataTarGz = await tarTools.packFilesToTarGz(dataEntries);
await new Promise((resolve) => {
dataGzip.on('data', (chunk) => dataChunks.push(chunk));
dataGzip.on('end', resolve);
});
// Create the outer gem (tar.gz containing metadata.gz and data.tar.gz)
const gemEntries: smartarchive.IArchiveEntry[] = [
{
archivePath: 'metadata.gz',
content: metadataGz,
},
{
archivePath: 'data.tar.gz',
content: dataTarGz,
},
];
pack.entry({ name: 'data.tar.gz' }, Buffer.concat(dataChunks));
pack.finalize();
// Convert to gzipped tar
const chunks: Buffer[] = [];
const gzip = zlib.createGzip();
return new Promise((resolve, reject) => {
pack.pipe(gzip);
gzip.on('data', (chunk) => chunks.push(chunk));
gzip.on('end', () => resolve(Buffer.concat(chunks)));
gzip.on('error', reject);
});
return tarTools.packFilesToTarGz(gemEntries);
}
/**

View File

@@ -78,7 +78,7 @@ tap.test('Integration: should handle /simple path for PyPI', async () => {
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/html');
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toContain('integration-test-py');
});

View File

@@ -99,7 +99,7 @@ tap.test('PyPI: should retrieve Simple API root index HTML (GET /simple/)', asyn
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/html');
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string');
const html = response.body as string;
@@ -125,8 +125,10 @@ tap.test('PyPI: should retrieve Simple API root index JSON (GET /simple/ with Ac
const json = response.body as any;
expect(json).toHaveProperty('meta');
expect(json).toHaveProperty('projects');
expect(json.projects).toBeTypeOf('object');
expect(json.projects).toHaveProperty(normalizedPackageName);
expect(json.projects).toBeInstanceOf(Array);
// Check that the package is in the projects list (PEP 691 format: array of { name } objects)
const packageNames = json.projects.map((p: any) => p.name);
expect(packageNames).toContain(normalizedPackageName);
});
tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)', async () => {
@@ -140,7 +142,7 @@ tap.test('PyPI: should retrieve Simple API package HTML (GET /simple/{package}/)
});
expect(response.status).toEqual(200);
expect(response.headers['Content-Type']).toEqual('text/html');
expect(response.headers['Content-Type']).toStartWith('text/html');
expect(response.body).toBeTypeOf('string');
const html = response.body as string;

View File

@@ -0,0 +1,448 @@
/**
* Native gem CLI Testing
* Tests the RubyGems registry implementation using the actual gem CLI
*/
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { tapNodeTools } from '@git.zone/tstest/tapbundle_serverside';
import { SmartRegistry } from '../ts/index.js';
import { createTestRegistry, createTestTokens, createRubyGem } from './helpers/registry.js';
import type { IRequestContext, IResponse } from '../ts/core/interfaces.core.js';
import * as http from 'http';
import * as url from 'url';
import * as fs from 'fs';
import * as path from 'path';
// Test context
let registry: SmartRegistry;
let server: http.Server;
let registryUrl: string;
let registryPort: number;
let rubygemsToken: string;
let testDir: string;
let gemHome: string;
/**
* Create HTTP server wrapper around SmartRegistry
*/
async function createHttpServer(
registryInstance: SmartRegistry,
port: number
): Promise<{ server: http.Server; url: string }> {
return new Promise((resolve, reject) => {
const httpServer = http.createServer(async (req, res) => {
try {
// Parse request
const parsedUrl = url.parse(req.url || '', true);
const pathname = parsedUrl.pathname || '/';
const query = parsedUrl.query;
// Read body
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(chunk);
}
const bodyBuffer = Buffer.concat(chunks);
// Parse body based on content type
let body: any;
if (bodyBuffer.length > 0) {
const contentType = req.headers['content-type'] || '';
if (contentType.includes('application/json')) {
try {
body = JSON.parse(bodyBuffer.toString('utf-8'));
} catch (error) {
body = bodyBuffer;
}
} else {
body = bodyBuffer;
}
}
// Convert to IRequestContext
const context: IRequestContext = {
method: req.method || 'GET',
path: pathname,
headers: req.headers as Record<string, string>,
query: query as Record<string, string>,
body: body,
};
// Handle request
const response: IResponse = await registryInstance.handleRequest(context);
// Convert IResponse to HTTP response
res.statusCode = response.status;
// Set headers
for (const [key, value] of Object.entries(response.headers || {})) {
res.setHeader(key, value);
}
// Send body
if (response.body) {
if (Buffer.isBuffer(response.body)) {
res.end(response.body);
} else if (typeof response.body === 'string') {
res.end(response.body);
} else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(response.body));
}
} else {
res.end();
}
} catch (error) {
console.error('Server error:', error);
res.statusCode = 500;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'INTERNAL_ERROR', message: String(error) }));
}
});
httpServer.listen(port, () => {
const serverUrl = `http://localhost:${port}`;
resolve({ server: httpServer, url: serverUrl });
});
httpServer.on('error', reject);
});
}
/**
* Setup gem credentials file
* Format: YAML with :rubygems_api_key: TOKEN
*/
function setupGemCredentials(token: string, gemHomeArg: string): string {
const gemDir = path.join(gemHomeArg, '.gem');
fs.mkdirSync(gemDir, { recursive: true });
// Create credentials file in YAML format
const credentialsContent = `:rubygems_api_key: ${token}\n`;
const credentialsPath = path.join(gemDir, 'credentials');
fs.writeFileSync(credentialsPath, credentialsContent, 'utf-8');
// Set restrictive permissions (gem requires 0600)
fs.chmodSync(credentialsPath, 0o600);
return credentialsPath;
}
/**
* Create a test gem file
*/
async function createTestGemFile(
gemName: string,
version: string,
targetDir: string
): Promise<string> {
const gemData = await createRubyGem(gemName, version);
const gemFilename = `${gemName}-${version}.gem`;
const gemPath = path.join(targetDir, gemFilename);
fs.writeFileSync(gemPath, gemData);
return gemPath;
}
/**
* Run gem command with proper environment
*/
async function runGemCommand(
command: string,
cwd: string,
includeAuth: boolean = true
): Promise<{ stdout: string; stderr: string; exitCode: number }> {
// Prepare environment variables
const envVars = [
`HOME="${gemHome}"`,
`GEM_HOME="${gemHome}"`,
includeAuth ? '' : 'RUBYGEMS_API_KEY=""',
].filter(Boolean).join(' ');
// Build command with cd to correct directory and environment variables
const fullCommand = `cd "${cwd}" && ${envVars} ${command}`;
try {
const result = await tapNodeTools.runCommand(fullCommand);
return {
stdout: result.stdout || '',
stderr: result.stderr || '',
exitCode: result.exitCode || 0,
};
} catch (error: any) {
return {
stdout: error.stdout || '',
stderr: error.stderr || String(error),
exitCode: error.exitCode || 1,
};
}
}
/**
* Cleanup test directory
*/
function cleanupTestDir(dir: string): void {
if (fs.existsSync(dir)) {
fs.rmSync(dir, { recursive: true, force: true });
}
}
// ========================================================================
// TESTS
// ========================================================================
tap.test('RubyGems CLI: should setup registry and HTTP server', async () => {
// Create registry
registry = await createTestRegistry();
const tokens = await createTestTokens(registry);
rubygemsToken = tokens.rubygemsToken;
expect(registry).toBeInstanceOf(SmartRegistry);
expect(rubygemsToken).toBeTypeOf('string');
// Use port 36000 (avoids npm:35000, cargo:5000 conflicts)
registryPort = 36000;
const serverSetup = await createHttpServer(registry, registryPort);
server = serverSetup.server;
registryUrl = serverSetup.url;
expect(server).toBeDefined();
expect(registryUrl).toEqual(`http://localhost:${registryPort}`);
// Setup test directory
testDir = path.join(process.cwd(), '.nogit', 'test-rubygems-cli');
cleanupTestDir(testDir);
fs.mkdirSync(testDir, { recursive: true });
// Setup GEM_HOME
gemHome = path.join(testDir, '.gem-home');
fs.mkdirSync(gemHome, { recursive: true });
// Setup gem credentials
const credentialsPath = setupGemCredentials(rubygemsToken, gemHome);
expect(fs.existsSync(credentialsPath)).toEqual(true);
// Verify credentials file has correct permissions
const stats = fs.statSync(credentialsPath);
const mode = stats.mode & 0o777;
expect(mode).toEqual(0o600);
});
tap.test('RubyGems CLI: should verify server is responding', async () => {
// Check server is up by doing a direct HTTP request to the Compact Index
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toBeGreaterThanOrEqual(200);
expect(response.status).toBeLessThan(500);
});
tap.test('RubyGems CLI: should build and push a gem', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
expect(fs.existsSync(gemPath)).toEqual(true);
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem push output:', result.stdout);
console.log('gem push stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
expect(result.stdout || result.stderr).toContain(gemName);
});
tap.test('RubyGems CLI: should verify gem in Compact Index /versions', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toEqual(200);
const versionsData = await response.text();
console.log('Versions data:', versionsData);
// Format: GEMNAME VERSION[,VERSION...] MD5
expect(versionsData).toContain(gemName);
expect(versionsData).toContain('1.0.0');
});
tap.test('RubyGems CLI: should verify gem in Compact Index /info file', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/info/${gemName}`);
expect(response.status).toEqual(200);
const infoData = await response.text();
console.log('Info data:', infoData);
// Format: VERSION [DEPS]|REQS
expect(infoData).toContain('1.0.0');
});
tap.test('RubyGems CLI: should download gem file', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const response = await fetch(`${registryUrl}/rubygems/gems/${gemName}-${version}.gem`);
expect(response.status).toEqual(200);
const gemData = await response.arrayBuffer();
expect(gemData.byteLength).toBeGreaterThan(0);
// Verify content type
expect(response.headers.get('content-type')).toContain('application/octet-stream');
});
tap.test('RubyGems CLI: should fetch gem metadata JSON', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/api/v1/versions/${gemName}.json`);
expect(response.status).toEqual(200);
const metadata = await response.json();
console.log('Metadata:', metadata);
expect(metadata).toBeInstanceOf(Array);
expect(metadata.length).toBeGreaterThan(0);
expect(metadata[0].number).toEqual('1.0.0');
});
tap.test('RubyGems CLI: should push second version', async () => {
const gemName = 'test-gem-cli';
const version = '2.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem push v2.0.0 output:', result.stdout);
expect(result.exitCode).toEqual(0);
});
tap.test('RubyGems CLI: should list all versions in /versions file', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/versions`);
expect(response.status).toEqual(200);
const versionsData = await response.text();
console.log('All versions data:', versionsData);
// Should contain both versions
expect(versionsData).toContain(gemName);
expect(versionsData).toContain('1.0.0');
expect(versionsData).toContain('2.0.0');
});
tap.test('RubyGems CLI: should yank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const result = await runGemCommand(
`gem yank ${gemName} -v ${version} --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem yank output:', result.stdout);
console.log('gem yank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is yanked in /versions file
// Yanked versions are prefixed with '-'
const response = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text();
console.log('Versions after yank:', versionsData);
// Yanked version should have '-' prefix
expect(versionsData).toContain('-1.0.0');
});
tap.test('RubyGems CLI: should unyank a version', async () => {
const gemName = 'test-gem-cli';
const version = '1.0.0';
const result = await runGemCommand(
`gem yank ${gemName} -v ${version} --undo --host ${registryUrl}/rubygems`,
testDir
);
console.log('gem unyank output:', result.stdout);
console.log('gem unyank stderr:', result.stderr);
expect(result.exitCode).toEqual(0);
// Verify version is not yanked in /versions file
const response = await fetch(`${registryUrl}/rubygems/versions`);
const versionsData = await response.text();
console.log('Versions after unyank:', versionsData);
// Should not have '-' prefix anymore (or have both without prefix)
// Check that we have the version without yank marker
const lines = versionsData.trim().split('\n');
const gemLine = lines.find(line => line.startsWith(gemName));
if (gemLine) {
// Parse format: "gemname version[,version...] md5"
const parts = gemLine.split(' ');
const versions = parts[1];
// Should have 1.0.0 without '-' prefix
expect(versions).toContain('1.0.0');
expect(versions).not.toContain('-1.0.0');
}
});
tap.test('RubyGems CLI: should fetch dependencies', async () => {
const gemName = 'test-gem-cli';
const response = await fetch(`${registryUrl}/rubygems/api/v1/dependencies?gems=${gemName}`);
expect(response.status).toEqual(200);
const dependencies = await response.json();
console.log('Dependencies:', dependencies);
expect(dependencies).toBeInstanceOf(Array);
});
tap.test('RubyGems CLI: should fail to push without auth', async () => {
const gemName = 'unauth-gem';
const version = '1.0.0';
const gemPath = await createTestGemFile(gemName, version, testDir);
// Run without auth
const result = await runGemCommand(
`gem push ${gemPath} --host ${registryUrl}/rubygems`,
testDir,
false
);
console.log('gem push unauth output:', result.stdout);
console.log('gem push unauth stderr:', result.stderr);
// Should fail with auth error
expect(result.exitCode).not.toEqual(0);
});
tap.postTask('cleanup rubygems cli tests', async () => {
// Stop server
if (server) {
await new Promise<void>((resolve) => {
server.close(() => resolve());
});
}
// Cleanup test directory
if (testDir) {
cleanupTestDir(testDir);
}
// Destroy registry
if (registry) {
registry.destroy();
}
});
export default tap.start();