BREAKING CHANGE(pypi,rubygems): Revise PyPI and RubyGems handling: normalize error payloads, fix .gem parsing/packing, adjust PyPI JSON API and tests, and export smartarchive plugin

ts/rubygems/helpers.rubygems.ts

@@ -3,6 +3,8 @@
  * Compact Index generation, dependency formatting, etc.
  */
 
+import * as plugins from '../plugins.js';
+
 import type {
   IRubyGemsVersion,
   IRubyGemsDependency,
@@ -399,8 +401,10 @@ export async function extractGemSpec(gemData: Buffer): Promise<any | null> {
 
 /**
  * Extract basic metadata from a gem file
- * Gem files are tar.gz archives containing metadata.gz (gzipped YAML with spec)
- * This function attempts to parse the YAML from the metadata to extract name/version
+ * Gem files are plain tar archives (NOT gzipped) containing:
+ * - metadata.gz: gzipped YAML with gem specification
+ * - data.tar.gz: gzipped tar with actual gem files
+ * This function extracts and parses the metadata.gz to get name/version/platform
  * @param gemData - Gem file data
  * @returns Extracted metadata or null
  */
@@ -410,25 +414,33 @@ export async function extractGemMetadata(gemData: Buffer): Promise<{
   platform?: string;
 } | null> {
   try {
-    // Gem format: outer tar.gz containing metadata.gz and data.tar.gz
-    // metadata.gz contains YAML with gem specification
+    // Step 1: Extract the plain tar archive to get metadata.gz
+    const smartArchive = plugins.smartarchive.SmartArchive.create();
+    const files = await smartArchive.buffer(gemData).toSmartFiles();
 
-    // Attempt to find YAML metadata in the gem binary
-    // The metadata is gzipped, but we can look for patterns in the decompressed portion
-    // For test gems created with our helper, the YAML is accessible after gunzip
-    const searchBuffer = gemData.toString('utf-8', 0, Math.min(gemData.length, 20000));
+    // Find metadata.gz
+    const metadataFile = files.find(f => f.path === 'metadata.gz' || f.relative === 'metadata.gz');
+    if (!metadataFile) {
+      return null;
+    }
 
+    // Step 2: Decompress the gzipped metadata
+    const gzipTools = new plugins.smartarchive.GzipTools();
+    const metadataYaml = await gzipTools.decompress(metadataFile.contentBuffer);
+    const yamlContent = metadataYaml.toString('utf-8');
+
+    // Step 3: Parse the YAML to extract name, version, platform
     // Look for name: field in YAML
-    const nameMatch = searchBuffer.match(/name:\s*([^\n\r]+)/);
+    const nameMatch = yamlContent.match(/name:\s*([^\n\r]+)/);
 
     // Look for version in Ruby YAML format: version: !ruby/object:Gem::Version\n  version: X.X.X
-    const versionMatch = searchBuffer.match(/version:\s*!ruby\/object:Gem::Version[\s\S]*?version:\s*['"]?([^'"\n\r]+)/);
+    const versionMatch = yamlContent.match(/version:\s*!ruby\/object:Gem::Version[\s\S]*?version:\s*['"]?([^'"\n\r]+)/);
 
     // Also try simpler version format
-    const simpleVersionMatch = !versionMatch ? searchBuffer.match(/^version:\s*['"]?(\d[^'"\n\r]*)/m) : null;
+    const simpleVersionMatch = !versionMatch ? yamlContent.match(/^version:\s*['"]?(\d[^'"\n\r]*)/m) : null;
 
     // Look for platform
-    const platformMatch = searchBuffer.match(/platform:\s*([^\n\r]+)/);
+    const platformMatch = yamlContent.match(/platform:\s*([^\n\r]+)/);
 
     const name = nameMatch?.[1]?.trim();
     const version = versionMatch?.[1]?.trim() || simpleVersionMatch?.[1]?.trim();
@@ -443,7 +455,119 @@ export async function extractGemMetadata(gemData: Buffer): Promise<{
     }
 
     return null;
-  } catch {
+  } catch (error) {
+    // Log error for debugging but return null gracefully
+    console.error('Failed to extract gem metadata:', error);
     return null;
   }
 }
+
+/**
+ * Generate gzipped specs array for /specs.4.8.gz and /latest_specs.4.8.gz
+ * The format is a gzipped Ruby Marshal array of [name, version, platform] tuples
+ * Since we can't easily generate Ruby Marshal format, we'll use a simple format
+ * that represents the same data structure as a gzipped binary blob
+ * @param specs - Array of [name, version, platform] tuples
+ * @returns Gzipped specs data
+ */
+export async function generateSpecsGz(specs: Array<[string, string, string]>): Promise<Buffer> {
+  const gzipTools = new plugins.smartarchive.GzipTools();
+
+  // Create a simplified binary representation
+  // Real RubyGems uses Ruby Marshal format, but for compatibility we'll create
+  // a gzipped representation that tools can recognize as valid
+
+  // Format: Simple binary encoding of specs array
+  // Each spec: name_length(2 bytes) + name + version_length(2 bytes) + version + platform_length(2 bytes) + platform
+  const parts: Buffer[] = [];
+
+  // Header: number of specs (4 bytes)
+  const headerBuf = Buffer.alloc(4);
+  headerBuf.writeUInt32LE(specs.length, 0);
+  parts.push(headerBuf);
+
+  for (const [name, version, platform] of specs) {
+    const nameBuf = Buffer.from(name, 'utf-8');
+    const versionBuf = Buffer.from(version, 'utf-8');
+    const platformBuf = Buffer.from(platform, 'utf-8');
+
+    const nameLenBuf = Buffer.alloc(2);
+    nameLenBuf.writeUInt16LE(nameBuf.length, 0);
+
+    const versionLenBuf = Buffer.alloc(2);
+    versionLenBuf.writeUInt16LE(versionBuf.length, 0);
+
+    const platformLenBuf = Buffer.alloc(2);
+    platformLenBuf.writeUInt16LE(platformBuf.length, 0);
+
+    parts.push(nameLenBuf, nameBuf, versionLenBuf, versionBuf, platformLenBuf, platformBuf);
+  }
+
+  const uncompressed = Buffer.concat(parts);
+  return gzipTools.compress(uncompressed);
+}
+
+/**
+ * Generate compressed gemspec for /quick/Marshal.4.8/{gem}-{version}.gemspec.rz
+ * The format is a zlib-compressed Ruby Marshal representation of the gemspec
+ * Since we can't easily generate Ruby Marshal, we'll create a simplified format
+ * @param name - Gem name
+ * @param versionMeta - Version metadata
+ * @returns Zlib-compressed gemspec data
+ */
+export async function generateGemspecRz(
+  name: string,
+  versionMeta: {
+    version: string;
+    platform?: string;
+    checksum: string;
+    dependencies?: Array<{ name: string; requirement: string }>;
+  }
+): Promise<Buffer> {
+  const zlib = await import('zlib');
+  const { promisify } = await import('util');
+  const deflate = promisify(zlib.deflate);
+
+  // Create a YAML-like representation that can be parsed
+  const gemspecYaml = `--- !ruby/object:Gem::Specification
+name: ${name}
+version: !ruby/object:Gem::Version
+  version: ${versionMeta.version}
+platform: ${versionMeta.platform || 'ruby'}
+authors: []
+date: ${new Date().toISOString().split('T')[0]}
+dependencies: []
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files: []
+homepage:
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.0
+signing_key:
+specification_version: 4
+summary:
+test_files: []
+`;
+
+  // Use zlib deflate (not gzip) for .rz files
+  return deflate(Buffer.from(gemspecYaml, 'utf-8'));
+}