fix(Smarts3): Allow overriding S3 descriptor; update dependencies, test config and documentation
This commit is contained in:
68
.serena/project.yml
Normal file
68
.serena/project.yml
Normal file
@@ -0,0 +1,68 @@
|
||||
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
|
||||
# * For C, use cpp
|
||||
# * For JavaScript, use typescript
|
||||
# Special requirements:
|
||||
# * csharp: Requires the presence of a .sln file in the project folder.
|
||||
language: typescript
|
||||
|
||||
# whether to use the project's gitignore file to ignore files
|
||||
# Added on 2025-04-07
|
||||
ignore_all_files_in_gitignore: true
|
||||
# list of additional paths to ignore
|
||||
# same syntax as gitignore, so you can use * and **
|
||||
# Was previously called `ignored_dirs`, please update your config if you are using that.
|
||||
# Added (renamed) on 2025-04-07
|
||||
ignored_paths: []
|
||||
|
||||
# whether the project is in read-only mode
|
||||
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||
# Added on 2025-04-18
|
||||
read_only: false
|
||||
|
||||
|
||||
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
|
||||
# Below is the complete list of tools for convenience.
|
||||
# To make sure you have the latest list of tools, and to view their descriptions,
|
||||
# execute `uv run scripts/print_tool_overview.py`.
|
||||
#
|
||||
# * `activate_project`: Activates a project by name.
|
||||
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
|
||||
# * `create_text_file`: Creates/overwrites a file in the project directory.
|
||||
# * `delete_lines`: Deletes a range of lines within a file.
|
||||
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
|
||||
# * `execute_shell_command`: Executes a shell command.
|
||||
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
|
||||
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
|
||||
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
|
||||
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
|
||||
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
|
||||
# * `initial_instructions`: Gets the initial instructions for the current project.
|
||||
# Should only be used in settings where the system prompt cannot be set,
|
||||
# e.g. in clients you have no control over, like Claude Desktop.
|
||||
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
|
||||
# * `insert_at_line`: Inserts content at a given line in a file.
|
||||
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
|
||||
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
|
||||
# * `list_memories`: Lists memories in Serena's project-specific memory store.
|
||||
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
|
||||
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
|
||||
# * `read_file`: Reads a file within the project directory.
|
||||
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
|
||||
# * `remove_project`: Removes a project from the Serena configuration.
|
||||
# * `replace_lines`: Replaces a range of lines within a file with new content.
|
||||
# * `replace_symbol_body`: Replaces the full definition of a symbol.
|
||||
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
|
||||
# * `search_for_pattern`: Performs a search for a pattern in the project.
|
||||
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
|
||||
# * `switch_modes`: Activates modes by providing a list of their names
|
||||
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
|
||||
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
|
||||
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
|
||||
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
|
||||
excluded_tools: []
|
||||
|
||||
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||
# (contrary to the memories, which are loaded on demand).
|
||||
initial_prompt: ""
|
||||
|
||||
project_name: "smarts3"
|
10
changelog.md
10
changelog.md
@@ -1,5 +1,15 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-08-16 - 2.2.6 - fix(Smarts3)
|
||||
Allow overriding S3 descriptor; update dependencies, test config and documentation
|
||||
|
||||
- ts/index.ts: getS3Descriptor now accepts an optional Partial<IS3Descriptor> to override defaults (backwards compatible)
|
||||
- package.json: updated devDependencies and runtime dependency versions (tstest, smartpath, tsclass, s3rver, etc.) and added packageManager field
|
||||
- package.json: expanded test script to run tstest with --web --verbose --logfile --timeout 60
|
||||
- test/test.ts: test instance port changed to 3333
|
||||
- readme.md: major rewrite and expansion of usage examples, API reference and guides
|
||||
- added project config files: .claude/settings.local.json and .serena/project.yml
|
||||
|
||||
## 2024-11-06 - 2.2.5 - fix(ci)
|
||||
Corrected docker image URLs in Gitea workflows to match the correct domain format.
|
||||
|
||||
|
12
package.json
12
package.json
@@ -9,7 +9,7 @@
|
||||
"author": "Lossless GmbH",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "(tstest test/ --web)",
|
||||
"test": "(tstest test/ --web --verbose --logfile --timeout 60)",
|
||||
"build": "(tsbuild --web --allowimplicitany)",
|
||||
"buildDocs": "tsdoc"
|
||||
},
|
||||
@@ -17,8 +17,7 @@
|
||||
"@git.zone/tsbuild": "^2.2.0",
|
||||
"@git.zone/tsbundle": "^2.1.0",
|
||||
"@git.zone/tsrun": "^1.3.3",
|
||||
"@git.zone/tstest": "^1.0.72",
|
||||
"@push.rocks/tapbundle": "^5.4.3",
|
||||
"@git.zone/tstest": "^2.3.2",
|
||||
"@types/node": "^22.9.0"
|
||||
},
|
||||
"browserslist": [
|
||||
@@ -39,8 +38,8 @@
|
||||
"dependencies": {
|
||||
"@push.rocks/smartbucket": "^3.0.23",
|
||||
"@push.rocks/smartfile": "^11.0.21",
|
||||
"@push.rocks/smartpath": "^5.0.5",
|
||||
"@tsclass/tsclass": "^4.1.2",
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@tsclass/tsclass": "^9.2.0",
|
||||
"@types/s3rver": "^3.7.0",
|
||||
"s3rver": "^3.7.1"
|
||||
},
|
||||
@@ -69,5 +68,6 @@
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://code.foss.global/push.rocks/smarts3/issues"
|
||||
}
|
||||
},
|
||||
"packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748"
|
||||
}
|
||||
|
13358
pnpm-lock.yaml
generated
13358
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
501
readme.md
501
readme.md
@@ -1,206 +1,414 @@
|
||||
````markdown
|
||||
# @push.rocks/smarts3
|
||||
# @push.rocks/smarts3 🚀
|
||||
|
||||
A Node.js TypeScript package to create a local S3 endpoint for development and testing using mapped local directories, simulating AWS S3.
|
||||
**Mock S3 made simple** - A powerful Node.js TypeScript package for creating a local S3 endpoint that simulates AWS S3 operations using mapped local directories. Perfect for development and testing!
|
||||
|
||||
## Install
|
||||
## 🌟 Features
|
||||
|
||||
To integrate `@push.rocks/smarts3` with your project, you need to install it via npm. Execute the following command within your project's root directory:
|
||||
- 🏃 **Lightning-fast local S3 simulation** - No more waiting for cloud operations during development
|
||||
- 🔄 **Full AWS S3 API compatibility** - Drop-in replacement for S3 in your tests
|
||||
- 📂 **Local directory mapping** - Your buckets live right on your filesystem
|
||||
- 🧪 **Perfect for testing** - Reliable, repeatable tests without cloud dependencies
|
||||
- 🎯 **TypeScript-first** - Built with TypeScript for excellent type safety and IDE support
|
||||
- 🔧 **Zero configuration** - Works out of the box with sensible defaults
|
||||
- 🧹 **Clean slate mode** - Start fresh on every test run
|
||||
|
||||
```sh
|
||||
npm install @push.rocks/smarts3 --save
|
||||
```
|
||||
````
|
||||
## 📦 Installation
|
||||
|
||||
This command will add `@push.rocks/smarts3` as a dependency in your project's `package.json` file and download the package into the `node_modules` directory.
|
||||
Install using your favorite package manager:
|
||||
|
||||
## Usage
|
||||
```bash
|
||||
# Using npm
|
||||
npm install @push.rocks/smarts3 --save-dev
|
||||
|
||||
### Overview
|
||||
# Using pnpm (recommended)
|
||||
pnpm add @push.rocks/smarts3 -D
|
||||
|
||||
The `@push.rocks/smarts3` module allows users to create a mock S3 endpoint that maps to a local directory using `s3rver`. This simulation of AWS S3 operations facilitates development and testing by enabling file uploads, bucket creation, and other interactions locally. This local setup is ideal for developers looking to test cloud file storage operations without requiring access to a real AWS S3 instance.
|
||||
|
||||
In this comprehensive guide, we will explore setting up a local S3 server, performing operations like creating buckets and uploading files, and how to effectively integrate this into your development workflow.
|
||||
|
||||
### Setting Up the Environment
|
||||
|
||||
To begin any operations, your environment must be configured correctly. Here’s a simple setup procedure:
|
||||
|
||||
```typescript
|
||||
import * as path from 'path';
|
||||
import { promises as fs } from 'fs';
|
||||
|
||||
async function setupEnvironment() {
|
||||
const packageDir = path.resolve();
|
||||
const nogitDir = path.join(packageDir, './.nogit');
|
||||
const bucketsDir = path.join(nogitDir, 'bucketsDir');
|
||||
|
||||
try {
|
||||
await fs.mkdir(bucketsDir, { recursive: true });
|
||||
} catch (error) {
|
||||
console.error('Failed to create buckets directory!', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
console.log('Environment setup complete.');
|
||||
}
|
||||
|
||||
setupEnvironment().catch(console.error);
|
||||
# Using yarn
|
||||
yarn add @push.rocks/smarts3 --dev
|
||||
```
|
||||
|
||||
This script sets up a directory structure required for the `smarts3` server, ensuring that the directories needed for bucket storage exist before starting the server.
|
||||
## 🚀 Quick Start
|
||||
|
||||
### Starting the S3 Server
|
||||
|
||||
Once your environment is set up, start an instance of the `smarts3` server. This acts as your local mock S3 endpoint:
|
||||
Get up and running in seconds:
|
||||
|
||||
```typescript
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
|
||||
async function startServer() {
|
||||
const smarts3Instance = await Smarts3.createAndStart({
|
||||
port: 3000,
|
||||
cleanSlate: true,
|
||||
});
|
||||
// Start your local S3 server
|
||||
const s3Server = await Smarts3.createAndStart({
|
||||
port: 3000,
|
||||
cleanSlate: true // Start with empty buckets
|
||||
});
|
||||
|
||||
console.log('S3 server is up and running at http://localhost:3000');
|
||||
return smarts3Instance;
|
||||
}
|
||||
// Create a bucket
|
||||
const bucket = await s3Server.createBucket('my-awesome-bucket');
|
||||
|
||||
startServer().catch(console.error);
|
||||
// Get S3 connection details for use with AWS SDK or other S3 clients
|
||||
const s3Config = await s3Server.getS3Descriptor();
|
||||
|
||||
// When you're done
|
||||
await s3Server.stop();
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
## 📖 Detailed Usage Guide
|
||||
|
||||
- **Port**: Specify the port for the local S3 server. Defaults to `3000`.
|
||||
- **CleanSlate**: If `true`, clears the storage directory each time the server starts, providing a fresh test state.
|
||||
### 🏗️ Setting Up Your S3 Server
|
||||
|
||||
### Creating and Managing Buckets
|
||||
|
||||
With your server running, create buckets for storing files. A bucket in S3 acts similarly to a root directory.
|
||||
The `Smarts3` class provides a simple interface for managing your local S3 server:
|
||||
|
||||
```typescript
|
||||
async function createBucket(smarts3Instance: Smarts3, bucketName: string) {
|
||||
const bucket = await smarts3Instance.createBucket(bucketName);
|
||||
console.log(`Bucket created: ${bucket.name}`);
|
||||
}
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
|
||||
startServer()
|
||||
.then((smarts3Instance) => createBucket(smarts3Instance, 'my-awesome-bucket'))
|
||||
.catch(console.error);
|
||||
// Configuration options
|
||||
const config = {
|
||||
port: 3000, // Port to run the server on (default: 3000)
|
||||
cleanSlate: true // Clear all data on start (default: false)
|
||||
};
|
||||
|
||||
// Create and start in one go
|
||||
const s3Server = await Smarts3.createAndStart(config);
|
||||
|
||||
// Or create and start separately
|
||||
const s3Server = new Smarts3(config);
|
||||
await s3Server.start();
|
||||
```
|
||||
|
||||
### Uploading and Managing Files
|
||||
### 🪣 Working with Buckets
|
||||
|
||||
Uploading files to a bucket uses the `SmartBucket` module, part of the `@push.rocks/smartbucket` ecosystem:
|
||||
Creating and managing buckets is straightforward:
|
||||
|
||||
```typescript
|
||||
// Create a new bucket
|
||||
const bucket = await s3Server.createBucket('my-bucket');
|
||||
|
||||
// The bucket is now ready to use!
|
||||
console.log(`Created bucket: ${bucket.name}`);
|
||||
```
|
||||
|
||||
### 📤 Uploading Files
|
||||
|
||||
Use the powerful `SmartBucket` integration for file operations:
|
||||
|
||||
```typescript
|
||||
import { SmartBucket } from '@push.rocks/smartbucket';
|
||||
|
||||
async function uploadFile(
|
||||
smarts3Instance: Smarts3,
|
||||
bucketName: string,
|
||||
filePath: string,
|
||||
fileContent: string,
|
||||
) {
|
||||
const s3Descriptor = await smarts3Instance.getS3Descriptor();
|
||||
const smartbucketInstance = new SmartBucket(s3Descriptor);
|
||||
const bucket = await smartbucketInstance.getBucket(bucketName);
|
||||
// Get connection configuration
|
||||
const s3Config = await s3Server.getS3Descriptor();
|
||||
|
||||
await bucket.getBaseDirectory().fastStore(filePath, fileContent);
|
||||
console.log(`File "${filePath}" uploaded successfully to bucket "${bucketName}".`);
|
||||
}
|
||||
// Create a SmartBucket instance
|
||||
const smartbucket = new SmartBucket(s3Config);
|
||||
|
||||
startServer()
|
||||
.then(async (smarts3Instance) => {
|
||||
await createBucket(smarts3Instance, 'my-awesome-bucket');
|
||||
await uploadFile(smarts3Instance, 'my-awesome-bucket', 'hello.txt', 'Hello, world!');
|
||||
})
|
||||
.catch(console.error);
|
||||
// Get your bucket
|
||||
const bucket = await smartbucket.getBucket('my-bucket');
|
||||
|
||||
// Upload a file
|
||||
const baseDir = await bucket.getBaseDirectory();
|
||||
await baseDir.fastStore('path/to/file.txt', 'Hello, S3! 🎉');
|
||||
|
||||
// Upload with more control
|
||||
await baseDir.fastPut({
|
||||
path: 'documents/important.pdf',
|
||||
contents: Buffer.from(yourPdfData)
|
||||
});
|
||||
```
|
||||
|
||||
### Listing Files in a Bucket
|
||||
### 📥 Downloading Files
|
||||
|
||||
Listing files within a bucket allows you to manage its contents conveniently:
|
||||
Retrieve your files easily:
|
||||
|
||||
```typescript
|
||||
async function listFiles(smarts3Instance: Smarts3, bucketName: string) {
|
||||
const s3Descriptor = await smarts3Instance.getS3Descriptor();
|
||||
const smartbucketInstance = new SmartBucket(s3Descriptor);
|
||||
const bucket = await smartbucketInstance.getBucket(bucketName);
|
||||
// Get file contents as string
|
||||
const content = await baseDir.fastGet('path/to/file.txt');
|
||||
console.log(content); // "Hello, S3! 🎉"
|
||||
|
||||
const baseDirectory = await bucket.getBaseDirectory();
|
||||
const files = await baseDirectory.listFiles();
|
||||
|
||||
console.log(`Files in bucket "${bucketName}":`, files);
|
||||
}
|
||||
|
||||
startServer()
|
||||
.then(async (smarts3Instance) => {
|
||||
await createBucket(smarts3Instance, 'my-awesome-bucket');
|
||||
await listFiles(smarts3Instance, 'my-awesome-bucket');
|
||||
})
|
||||
.catch(console.error);
|
||||
// Get file as Buffer
|
||||
const buffer = await baseDir.fastGetBuffer('documents/important.pdf');
|
||||
```
|
||||
|
||||
### Deleting a File
|
||||
### 📋 Listing Files
|
||||
|
||||
Managing storage efficiently involves deleting files when necessary:
|
||||
Browse your bucket contents:
|
||||
|
||||
```typescript
|
||||
async function deleteFile(smarts3Instance: Smarts3, bucketName: string, filePath: string) {
|
||||
const s3Descriptor = await smarts3Instance.getS3Descriptor();
|
||||
const smartbucketInstance = new SmartBucket(s3Descriptor);
|
||||
const bucket = await smartbucketInstance.getBucket(bucketName);
|
||||
// List all files in the bucket
|
||||
const files = await baseDir.listFiles();
|
||||
|
||||
await bucket.getBaseDirectory().fastDelete(filePath);
|
||||
console.log(`File "${filePath}" deleted from bucket "${bucketName}".`);
|
||||
}
|
||||
files.forEach(file => {
|
||||
console.log(`📄 ${file.name} (${file.size} bytes)`);
|
||||
});
|
||||
|
||||
startServer()
|
||||
.then(async (smarts3Instance) => {
|
||||
await createBucket(smarts3Instance, 'my-awesome-bucket');
|
||||
await deleteFile(smarts3Instance, 'my-awesome-bucket', 'hello.txt');
|
||||
})
|
||||
.catch(console.error);
|
||||
// List files with a specific prefix
|
||||
const docs = await baseDir.listFiles('documents/');
|
||||
```
|
||||
|
||||
### Scenario Integrations
|
||||
### 🗑️ Deleting Files
|
||||
|
||||
#### Development and Testing
|
||||
|
||||
1. **Feature Development:** Use `@push.rocks/smarts3` to simulate file upload endpoints, ensuring your application handles file operations correctly before going live.
|
||||
2. **Continuous Integration/Continuous Deployment (CI/CD):** Integrate with CI/CD pipelines to automatically test file interactions.
|
||||
|
||||
3. **Data Migration Testing:** Simulate data migrations between buckets to perfect processes before implementation on actual S3.
|
||||
|
||||
4. **Onboarding New Developers:** Offer new team members hands-on practice with mock setups to improve their understanding without real-world consequences.
|
||||
|
||||
### Stopping the Server
|
||||
|
||||
Safely shutting down the server when tasks are complete ensures system resources are managed well:
|
||||
Clean up when needed:
|
||||
|
||||
```typescript
|
||||
async function stopServer(smarts3Instance: Smarts3) {
|
||||
await smarts3Instance.stop();
|
||||
console.log('S3 server has been stopped.');
|
||||
// Delete a single file
|
||||
await baseDir.fastDelete('old-file.txt');
|
||||
|
||||
// Delete multiple files
|
||||
const filesToDelete = ['temp1.txt', 'temp2.txt', 'temp3.txt'];
|
||||
for (const file of filesToDelete) {
|
||||
await baseDir.fastDelete(file);
|
||||
}
|
||||
|
||||
startServer()
|
||||
.then(async (smarts3Instance) => {
|
||||
await createBucket(smarts3Instance, 'my-awesome-bucket');
|
||||
await stopServer(smarts3Instance);
|
||||
})
|
||||
.catch(console.error);
|
||||
```
|
||||
|
||||
In this guide, we walked through setting up and fully utilizing the `@push.rocks/smarts3` package for local development and testing. The package simulates AWS S3 operations, reducing dependency on remote services and allowing efficient development iteration cycles. By implementing the practices and scripts shared here, you can ensure a seamless and productive development experience using the local S3 simulation capabilities of `@push.rocks/smarts3`.
|
||||
## 🧪 Testing Integration
|
||||
|
||||
### Using with Jest
|
||||
|
||||
```typescript
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
|
||||
describe('S3 Operations', () => {
|
||||
let s3Server: Smarts3;
|
||||
|
||||
beforeAll(async () => {
|
||||
s3Server = await Smarts3.createAndStart({
|
||||
port: 9999,
|
||||
cleanSlate: true
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await s3Server.stop();
|
||||
});
|
||||
|
||||
test('should upload and retrieve a file', async () => {
|
||||
const bucket = await s3Server.createBucket('test-bucket');
|
||||
// Your test logic here
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Using with Mocha
|
||||
|
||||
```typescript
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
import { expect } from 'chai';
|
||||
|
||||
describe('S3 Operations', () => {
|
||||
let s3Server: Smarts3;
|
||||
|
||||
before(async () => {
|
||||
s3Server = await Smarts3.createAndStart({
|
||||
port: 9999,
|
||||
cleanSlate: true
|
||||
});
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await s3Server.stop();
|
||||
});
|
||||
|
||||
it('should upload and retrieve a file', async () => {
|
||||
const bucket = await s3Server.createBucket('test-bucket');
|
||||
// Your test logic here
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## 🔌 AWS SDK Integration
|
||||
|
||||
Use `smarts3` with the official AWS SDK:
|
||||
|
||||
```typescript
|
||||
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
|
||||
// Start local S3
|
||||
const s3Server = await Smarts3.createAndStart({ port: 3000 });
|
||||
const config = await s3Server.getS3Descriptor();
|
||||
|
||||
// Configure AWS SDK
|
||||
const s3Client = new S3Client({
|
||||
endpoint: `http://${config.endpoint}:${config.port}`,
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: config.accessKey,
|
||||
secretAccessKey: config.accessSecret
|
||||
},
|
||||
forcePathStyle: true
|
||||
});
|
||||
|
||||
// Use AWS SDK as normal
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: 'my-bucket',
|
||||
Key: 'test-file.txt',
|
||||
Body: 'Hello from AWS SDK!'
|
||||
});
|
||||
|
||||
await s3Client.send(command);
|
||||
```
|
||||
|
||||
## 🎯 Real-World Examples
|
||||
|
||||
### CI/CD Pipeline Testing
|
||||
|
||||
```typescript
|
||||
// ci-test.ts
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
|
||||
export async function setupTestEnvironment() {
|
||||
// Start S3 server for CI tests
|
||||
const s3 = await Smarts3.createAndStart({
|
||||
port: process.env.S3_PORT || 3000,
|
||||
cleanSlate: true
|
||||
});
|
||||
|
||||
// Create test buckets
|
||||
await s3.createBucket('uploads');
|
||||
await s3.createBucket('processed');
|
||||
await s3.createBucket('archive');
|
||||
|
||||
return s3;
|
||||
}
|
||||
```
|
||||
|
||||
### Microservice Development
|
||||
|
||||
```typescript
|
||||
// dev-server.ts
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
import express from 'express';
|
||||
|
||||
async function startDevelopmentServer() {
|
||||
// Start local S3
|
||||
const s3 = await Smarts3.createAndStart({ port: 3000 });
|
||||
await s3.createBucket('user-uploads');
|
||||
|
||||
// Start your API server
|
||||
const app = express();
|
||||
|
||||
app.post('/upload', async (req, res) => {
|
||||
// Your upload logic using local S3
|
||||
});
|
||||
|
||||
app.listen(8080, () => {
|
||||
console.log('🚀 Dev server running with local S3!');
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### Data Migration Testing
|
||||
|
||||
```typescript
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
|
||||
async function testDataMigration() {
|
||||
const s3 = await Smarts3.createAndStart({ cleanSlate: true });
|
||||
|
||||
// Create source and destination buckets
|
||||
const sourceBucket = await s3.createBucket('legacy-data');
|
||||
const destBucket = await s3.createBucket('new-data');
|
||||
|
||||
// Populate source with test data
|
||||
const config = await s3.getS3Descriptor();
|
||||
const smartbucket = new SmartBucket(config);
|
||||
const source = await smartbucket.getBucket('legacy-data');
|
||||
const sourceDir = await source.getBaseDirectory();
|
||||
|
||||
// Add test files
|
||||
await sourceDir.fastStore('user-1.json', JSON.stringify({ id: 1, name: 'Alice' }));
|
||||
await sourceDir.fastStore('user-2.json', JSON.stringify({ id: 2, name: 'Bob' }));
|
||||
|
||||
// Run your migration logic
|
||||
await runMigration(config);
|
||||
|
||||
// Verify migration results
|
||||
const dest = await smartbucket.getBucket('new-data');
|
||||
const destDir = await dest.getBaseDirectory();
|
||||
const migratedFiles = await destDir.listFiles();
|
||||
|
||||
console.log(`✅ Migrated ${migratedFiles.length} files successfully!`);
|
||||
}
|
||||
```
|
||||
|
||||
## 🛠️ Advanced Configuration
|
||||
|
||||
### Custom S3 Descriptor Options
|
||||
|
||||
When integrating with different S3 clients, you can customize the connection details:
|
||||
|
||||
```typescript
|
||||
const customDescriptor = await s3Server.getS3Descriptor({
|
||||
endpoint: 'localhost', // Custom endpoint
|
||||
port: 3001, // Different port
|
||||
useSsl: false, // SSL configuration
|
||||
// Add any additional options your S3 client needs
|
||||
});
|
||||
```
|
||||
|
||||
### Environment-Based Configuration
|
||||
|
||||
```typescript
|
||||
const config = {
|
||||
port: parseInt(process.env.S3_PORT || '3000'),
|
||||
cleanSlate: process.env.NODE_ENV === 'test'
|
||||
};
|
||||
|
||||
const s3Server = await Smarts3.createAndStart(config);
|
||||
```
|
||||
|
||||
## 🤝 Use Cases
|
||||
|
||||
- **🧪 Unit & Integration Testing** - Test S3 operations without AWS credentials or internet
|
||||
- **🏗️ Local Development** - Develop cloud features offline with full S3 compatibility
|
||||
- **📚 Teaching & Demos** - Perfect for workshops and tutorials without AWS setup
|
||||
- **🔄 CI/CD Pipelines** - Reliable S3 operations in containerized test environments
|
||||
- **🎭 Mocking & Stubbing** - Replace real S3 calls in test suites
|
||||
- **📊 Data Migration Testing** - Safely test data migrations locally before production
|
||||
|
||||
## 🔧 API Reference
|
||||
|
||||
### Smarts3 Class
|
||||
|
||||
#### Constructor Options
|
||||
|
||||
```typescript
|
||||
interface ISmarts3ContructorOptions {
|
||||
port?: number; // Server port (default: 3000)
|
||||
cleanSlate?: boolean; // Clear storage on start (default: false)
|
||||
}
|
||||
```
|
||||
|
||||
#### Methods
|
||||
|
||||
- `static createAndStart(options)` - Create and start server in one call
|
||||
- `start()` - Start the S3 server
|
||||
- `stop()` - Stop the S3 server
|
||||
- `createBucket(name)` - Create a new bucket
|
||||
- `getS3Descriptor(options?)` - Get S3 connection configuration
|
||||
|
||||
## 🐛 Debugging Tips
|
||||
|
||||
1. **Enable verbose logging** - The server logs all operations by default
|
||||
2. **Check the buckets directory** - Find your data in `.nogit/bucketsDir/`
|
||||
3. **Use the correct endpoint** - Remember to use `127.0.0.1` or `localhost`
|
||||
4. **Force path style** - Always use path-style URLs with local S3
|
||||
|
||||
## 📈 Performance
|
||||
|
||||
`@push.rocks/smarts3` is optimized for development and testing:
|
||||
|
||||
- ⚡ **Instant operations** - No network latency
|
||||
- 💾 **Low memory footprint** - Efficient file system usage
|
||||
- 🔄 **Fast cleanup** - Clean slate mode for quick test resets
|
||||
- 🚀 **Parallel operations** - Handle multiple requests simultaneously
|
||||
|
||||
## 🔗 Related Packages
|
||||
|
||||
- [`@push.rocks/smartbucket`](https://www.npmjs.com/package/@push.rocks/smartbucket) - Powerful S3 abstraction layer
|
||||
- [`@push.rocks/smartfile`](https://www.npmjs.com/package/@push.rocks/smartfile) - Advanced file system operations
|
||||
- [`@tsclass/tsclass`](https://www.npmjs.com/package/@tsclass/tsclass) - TypeScript class helpers
|
||||
- [`s3rver`](https://www.npmjs.com/package/s3rver) - The underlying S3 server implementation
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
@@ -210,10 +418,9 @@ This project is owned and maintained by Task Venture Capital GmbH. The names and
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
```
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
@@ -1,4 +1,4 @@
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
import * as smarts3 from '../ts/index.js';
|
||||
@@ -7,7 +7,7 @@ let testSmarts3Instance: smarts3.Smarts3;
|
||||
|
||||
tap.test('should create a smarts3 instance and run it', async (toolsArg) => {
|
||||
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
|
||||
port: 3000,
|
||||
port: 3333,
|
||||
cleanSlate: true,
|
||||
});
|
||||
console.log(`Let the instance run for 2 seconds`);
|
||||
|
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smarts3',
|
||||
version: '2.2.5',
|
||||
version: '2.2.6',
|
||||
description: 'A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.'
|
||||
}
|
||||
|
17
ts/index.ts
17
ts/index.ts
@@ -42,13 +42,18 @@ export class Smarts3 {
|
||||
console.log('s3 server is running');
|
||||
}
|
||||
|
||||
public async getS3Descriptor(): Promise<plugins.tsclass.storage.IS3Descriptor> {
|
||||
public async getS3Descriptor(
|
||||
optionsArg?: Partial<plugins.tsclass.storage.IS3Descriptor>
|
||||
): Promise<plugins.tsclass.storage.IS3Descriptor> {
|
||||
return {
|
||||
accessKey: 'S3RVER',
|
||||
accessSecret: 'S3RVER',
|
||||
endpoint: '127.0.0.1',
|
||||
port: this.options.port,
|
||||
useSsl: false,
|
||||
...{
|
||||
accessKey: 'S3RVER',
|
||||
accessSecret: 'S3RVER',
|
||||
endpoint: '127.0.0.1',
|
||||
port: this.options.port,
|
||||
useSsl: false,
|
||||
},
|
||||
...(optionsArg? optionsArg : {})
|
||||
};
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user