fix(tstest): Fix test timing display issue and update TAP protocol documentation
This commit is contained in:
234
readme.plan.md
234
readme.plan.md
@@ -2,6 +2,81 @@
|
||||
|
||||
!! FIRST: Reread /home/philkunz/.claude/CLAUDE.md to ensure following all guidelines !!
|
||||
|
||||
## Improved Internal Protocol (NEW - Critical)
|
||||
|
||||
### Current Issues
|
||||
- TAP protocol uses `#` for metadata which conflicts with test descriptions containing `#`
|
||||
- Fragile regex parsing that breaks with special characters
|
||||
- Limited extensibility for new metadata types
|
||||
|
||||
### Proposed Solution: Protocol V2
|
||||
- Use Unicode delimiters `⟦TSTEST:META:{}⟧` that won't appear in test names
|
||||
- Structured JSON metadata format
|
||||
- Separate protocol blocks for complex data (errors, snapshots)
|
||||
- Backwards compatible with gradual migration
|
||||
|
||||
### Implementation
|
||||
- Phase 1: Add protocol v2 parser alongside v1
|
||||
- Phase 2: Generate v2 by default with --legacy flag for v1
|
||||
- Phase 3: Full migration to v2 in next major version
|
||||
|
||||
See `readme.protocol.md` for detailed specification.
|
||||
|
||||
## Test Configuration System (NEW)
|
||||
|
||||
### Global Test Configuration via 00init.ts
|
||||
- **Discovery**: Check for `test/00init.ts` before running tests
|
||||
- **Execution**: Import and execute before any test files if found
|
||||
- **Purpose**: Define project-wide default test settings
|
||||
|
||||
### tap.settings() API
|
||||
```typescript
|
||||
interface TapSettings {
|
||||
// Timing
|
||||
timeout?: number; // Default timeout for all tests (ms)
|
||||
slowThreshold?: number; // Mark tests as slow if they exceed this (ms)
|
||||
|
||||
// Execution Control
|
||||
bail?: boolean; // Stop on first test failure
|
||||
retries?: number; // Number of retries for failed tests
|
||||
retryDelay?: number; // Delay between retries (ms)
|
||||
|
||||
// Output Control
|
||||
suppressConsole?: boolean; // Suppress console output in passing tests
|
||||
verboseErrors?: boolean; // Show full stack traces
|
||||
showTestDuration?: boolean; // Show duration for each test
|
||||
|
||||
// Parallel Execution
|
||||
maxConcurrency?: number; // Max parallel tests (for .para files)
|
||||
isolateTests?: boolean; // Run each test in fresh context
|
||||
|
||||
// Lifecycle Hooks
|
||||
beforeAll?: () => Promise<void> | void;
|
||||
afterAll?: () => Promise<void> | void;
|
||||
beforeEach?: (testName: string) => Promise<void> | void;
|
||||
afterEach?: (testName: string, passed: boolean) => Promise<void> | void;
|
||||
|
||||
// Environment
|
||||
env?: Record<string, string>; // Additional environment variables
|
||||
|
||||
// Features
|
||||
enableSnapshots?: boolean; // Enable snapshot testing
|
||||
snapshotDirectory?: string; // Custom snapshot directory
|
||||
updateSnapshots?: boolean; // Update snapshots instead of comparing
|
||||
}
|
||||
```
|
||||
|
||||
### Settings Inheritance
|
||||
- Global (00init.ts) → File level → Test level
|
||||
- More specific settings override less specific ones
|
||||
- Arrays/objects are merged, primitives are replaced
|
||||
|
||||
### Implementation Phases
|
||||
1. **Core Infrastructure**: Settings storage and merge logic
|
||||
2. **Discovery**: 00init.ts loading mechanism
|
||||
3. **Application**: Apply settings to test execution
|
||||
4. **Advanced**: Parallel execution and snapshot configuration
|
||||
|
||||
## 1. Enhanced Communication Between tapbundle and tstest
|
||||
|
||||
### 1.1 Real-time Test Progress API
|
||||
@@ -18,45 +93,9 @@
|
||||
|
||||
## 2. Enhanced toolsArg Functionality
|
||||
|
||||
### 2.1 Test Flow Control ✅
|
||||
```typescript
|
||||
tap.test('conditional test', async (toolsArg) => {
|
||||
const result = await someOperation();
|
||||
|
||||
// Skip the rest of the test
|
||||
if (!result) {
|
||||
return toolsArg.skip('Precondition not met');
|
||||
}
|
||||
|
||||
// Conditional skipping
|
||||
await toolsArg.skipIf(condition, 'Reason for skipping');
|
||||
|
||||
// Mark test as todo
|
||||
await toolsArg.todo('Not implemented yet');
|
||||
});
|
||||
```
|
||||
|
||||
### 2.2 Test Metadata and Configuration ✅
|
||||
```typescript
|
||||
// Fluent syntax ✅
|
||||
tap.tags('slow', 'integration')
|
||||
.priority('high')
|
||||
.timeout(5000)
|
||||
.retry(3)
|
||||
.test('configurable test', async (toolsArg) => {
|
||||
// Test implementation
|
||||
});
|
||||
```
|
||||
|
||||
### 2.3 Test Data and Context Sharing ✅
|
||||
### 2.3 Test Data and Context Sharing (Partial)
|
||||
```typescript
|
||||
tap.test('data-driven test', async (toolsArg) => {
|
||||
// Access shared context ✅
|
||||
const sharedData = toolsArg.context.get('sharedData');
|
||||
|
||||
// Set data for other tests ✅
|
||||
toolsArg.context.set('resultData', computedValue);
|
||||
|
||||
// Parameterized test data (not yet implemented)
|
||||
const testData = toolsArg.data<TestInput>();
|
||||
expect(processData(testData)).toEqual(expected);
|
||||
@@ -65,32 +104,7 @@ tap.test('data-driven test', async (toolsArg) => {
|
||||
|
||||
## 3. Nested Tests and Test Suites
|
||||
|
||||
### 3.1 Test Grouping with describe() ✅
|
||||
```typescript
|
||||
tap.describe('User Authentication', () => {
|
||||
tap.beforeEach(async (toolsArg) => {
|
||||
// Setup for each test in this suite
|
||||
await toolsArg.context.set('db', await createTestDatabase());
|
||||
});
|
||||
|
||||
tap.afterEach(async (toolsArg) => {
|
||||
// Cleanup after each test
|
||||
await toolsArg.context.get('db').cleanup();
|
||||
});
|
||||
|
||||
tap.test('should login with valid credentials', async (toolsArg) => {
|
||||
// Test implementation
|
||||
});
|
||||
|
||||
tap.describe('Password Reset', () => {
|
||||
tap.test('should send reset email', async (toolsArg) => {
|
||||
// Nested test
|
||||
});
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### 3.2 Hierarchical Test Organization
|
||||
### 3.2 Hierarchical Test Organization (Not yet implemented)
|
||||
- Support for multiple levels of nesting
|
||||
- Inherited context and configuration from parent suites
|
||||
- Aggregated reporting for test suites
|
||||
@@ -98,15 +112,7 @@ tap.describe('User Authentication', () => {
|
||||
|
||||
## 4. Advanced Test Features
|
||||
|
||||
### 4.1 Snapshot Testing
|
||||
```typescript
|
||||
tap.test('component render', async (toolsArg) => {
|
||||
const output = renderComponent(props);
|
||||
|
||||
// Compare with stored snapshot
|
||||
await toolsArg.matchSnapshot(output, 'component-output');
|
||||
});
|
||||
```
|
||||
### 4.1 Snapshot Testing ✅ (Basic implementation complete)
|
||||
|
||||
### 4.2 Performance Benchmarking
|
||||
```typescript
|
||||
@@ -124,30 +130,9 @@ tap.test('performance test', async (toolsArg) => {
|
||||
});
|
||||
```
|
||||
|
||||
### 4.3 Test Fixtures and Factories ✅
|
||||
```typescript
|
||||
tap.test('with fixtures', async (toolsArg) => {
|
||||
// Create test fixtures
|
||||
const user = await toolsArg.fixture('user', { name: 'Test User' });
|
||||
const post = await toolsArg.fixture('post', { author: user });
|
||||
|
||||
// Use factory functions
|
||||
const users = await toolsArg.factory('user').createMany(5);
|
||||
});
|
||||
```
|
||||
|
||||
## 5. Test Execution Improvements
|
||||
|
||||
### 5.1 Parallel Test Execution ✅
|
||||
- Run independent tests concurrently ✅
|
||||
- Configurable concurrency limits (via file naming convention)
|
||||
- Resource pooling for shared resources
|
||||
- Proper isolation between parallel tests ✅
|
||||
|
||||
Implementation:
|
||||
- Tests with `para__<groupNumber>` in filename run in parallel
|
||||
- Different groups run sequentially
|
||||
- Tests without `para__` run serially
|
||||
|
||||
### 5.2 Watch Mode
|
||||
- Automatically re-run tests on file changes
|
||||
@@ -155,11 +140,8 @@ Implementation:
|
||||
- Fast feedback loop for development
|
||||
- Integration with IDE/editor plugins
|
||||
|
||||
### 5.3 Advanced Test Filtering ✅ (partially)
|
||||
### 5.3 Advanced Test Filtering (Partial)
|
||||
```typescript
|
||||
// Run tests by tags ✅
|
||||
tstest --tags "unit,fast"
|
||||
|
||||
// Exclude tests by pattern (not yet implemented)
|
||||
tstest --exclude "**/slow/**"
|
||||
|
||||
@@ -198,50 +180,36 @@ tstest --changed
|
||||
- Links to documentation
|
||||
- Code examples in error output
|
||||
|
||||
### 7.2 Interactive Mode (Needs Detailed Specification)
|
||||
- REPL for exploring test failures
|
||||
- Need to define: How to enter interactive mode? When tests fail?
|
||||
- What commands/features should be available in the REPL?
|
||||
- Debugging integration
|
||||
- Node.js inspector protocol integration?
|
||||
- Breakpoint support?
|
||||
- Step-through test execution
|
||||
- Pause between tests?
|
||||
- Step into/over/out functionality?
|
||||
- Interactive test data manipulation
|
||||
- Modify test inputs on the fly?
|
||||
- Inspect intermediate values?
|
||||
|
||||
### 7.3 ~~VS Code Extension~~ (Scratched)
|
||||
- ~~Test explorer integration~~
|
||||
- ~~Inline test results~~
|
||||
- ~~CodeLens for running individual tests~~
|
||||
- ~~Debugging support~~
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Core Enhancements (Priority: High) ✅
|
||||
1. Implement enhanced toolsArg methods (skip, skipIf, timeout, retry) ✅
|
||||
2. Add basic test grouping with describe() ✅
|
||||
3. Improve error reporting between tapbundle and tstest ✅
|
||||
### Phase 1: Improved Internal Protocol (Priority: Critical) (NEW)
|
||||
1. Implement Protocol V2 parser in tstest
|
||||
2. Add protocol version negotiation
|
||||
3. Update tapbundle to generate V2 format with feature flag
|
||||
4. Test with real-world test suites containing special characters
|
||||
|
||||
### Phase 2: Advanced Features (Priority: Medium)
|
||||
1. Implement nested test suites ✅ (basic describe support)
|
||||
2. Add snapshot testing ✅
|
||||
3. Create test fixture system ✅
|
||||
4. Implement parallel test execution ✅
|
||||
### Phase 2: Test Configuration System (Priority: High)
|
||||
1. Implement tap.settings() API with TypeScript interfaces
|
||||
2. Add 00init.ts discovery and loading mechanism
|
||||
3. Implement settings inheritance and merge logic
|
||||
4. Apply settings to test execution (timeouts, retries, etc.)
|
||||
|
||||
### Phase 3: Developer Experience (Priority: Medium)
|
||||
### Phase 3: Enhanced Communication (Priority: High)
|
||||
1. Build on Protocol V2 for richer communication
|
||||
2. Implement real-time test progress API
|
||||
3. Add structured error reporting with diffs and traces
|
||||
|
||||
### Phase 4: Developer Experience (Priority: Medium)
|
||||
1. Add watch mode
|
||||
2. Implement custom reporters
|
||||
3. ~~Create VS Code extension~~ (Scratched)
|
||||
4. Add interactive debugging (Needs detailed spec first)
|
||||
3. Complete advanced test filtering options
|
||||
4. Add performance benchmarking API
|
||||
|
||||
### Phase 4: Analytics and Performance (Priority: Low)
|
||||
### Phase 5: Analytics and Performance (Priority: Low)
|
||||
1. Build test analytics dashboard
|
||||
2. Add performance benchmarking
|
||||
3. Implement coverage integration
|
||||
4. Create trend analysis tools
|
||||
2. Implement coverage integration
|
||||
3. Create trend analysis tools
|
||||
4. Add test impact analysis
|
||||
|
||||
## Technical Considerations
|
||||
|
||||
|
||||
Reference in New Issue
Block a user