diff --git a/changelog.md b/changelog.md index 4103d21..e3176ac 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,15 @@ # Changelog +## 2025-05-23 - 1.9.3 - fix(tstest) +Fix test timing display issue and update TAP protocol documentation + +- Changed TAP parser regex to non-greedy pattern to correctly separate test timing metadata +- Enhanced readme.hints.md with detailed explanation of test timing fix and planned protocol upgrades +- Updated readme.md with improved usage examples for tapbundle and comprehensive test framework documentation +- Added new protocol design document (readme.protocol.md) and improvement plan (readme.plan.md) outlining future changes +- Introduced .claude/settings.local.json update for npm and CLI permissions +- Exported protocol utilities and added tapbundle protocol implementation for future enhancements + ## 2025-05-23 - 1.9.2 - fix(logging) Fix log file naming to prevent collisions and update logging system documentation. diff --git a/readme.hints.md b/readme.hints.md index 3a9e7a9..fd64266 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -74,4 +74,34 @@ This fix ensures that test files with the same basename in different directories 1. Takes the relative path from the current working directory 2. Replaces path separators (`/`) with double underscores (`__`) 3. Removes the `.ts` extension -4. Creates a flat filename that preserves the directory structure \ No newline at end of file +4. Creates a flat filename that preserves the directory structure + +### Test Timing Display (Fixed in v1.9.2) + +Fixed an issue where test timing was displayed incorrectly with duplicate values like: +- Before: `✅ test name # time=133ms (0ms)` +- After: `✅ test name (133ms)` + +The issue was in the TAP parser regex which was greedily capturing the entire line including the TAP timing comment. Changed the regex from `(.*)` to `(.*?)` to make it non-greedy, properly separating the test name from the timing metadata. + +## Protocol Limitations and Improvements + +### Current TAP Protocol Issues +The current implementation uses standard TAP format with metadata in comments: +``` +ok 1 - test name # time=123ms +``` + +This has several limitations: +1. **Delimiter Conflict**: Test descriptions containing `#` can break parsing +2. **Regex Fragility**: Complex regex patterns that are hard to maintain +3. **Limited Metadata**: Difficult to add rich error information or custom data + +### Planned Protocol V2 +A new internal protocol is being designed that will: +- Use Unicode delimiters `⟦TSTEST:⟧` that won't conflict with test content +- Support structured JSON metadata +- Allow rich error reporting with stack traces and diffs +- Maintain backwards compatibility during migration + +See `readme.protocol.md` for the full specification and `tapbundle.protocols.ts` for the implementation utilities. \ No newline at end of file diff --git a/readme.md b/readme.md index 23dda78..003d472 100644 --- a/readme.md +++ b/readme.md @@ -141,9 +141,9 @@ tstest supports different test environments through file naming: | `*.browser.ts` | Browser environment | `test.ui.browser.ts` | | `*.both.ts` | Both Node.js and browser | `test.isomorphic.both.ts` | -### Writing Tests +### Writing Tests with tapbundle -tstest includes a built-in TAP (Test Anything Protocol) test framework. Import it from the embedded tapbundle: +tstest includes tapbundle, a powerful TAP-based test framework. Import it from the embedded tapbundle: ```typescript import { expect, tap } from '@git.zone/tstest/tapbundle'; @@ -164,100 +164,392 @@ tstest provides multiple exports for different use cases: - `@git.zone/tstest/tapbundle` - Browser-compatible test framework - `@git.zone/tstest/tapbundle_node` - Node.js-specific test utilities -#### Test Features +## tapbundle Test Framework + +### Basic Test Syntax -**Tag-based Test Filtering** ```typescript -tap.tags('unit', 'api') - .test('should handle API requests', async () => { - // Test code - }); +import { tap, expect } from '@git.zone/tstest/tapbundle'; -// Run with: tstest test/ --tags unit,api -``` - -**Test Lifecycle Hooks** -```typescript -tap.describe('User API Tests', () => { - let testUser; - - tap.beforeEach(async () => { - testUser = await createTestUser(); - }); - - tap.afterEach(async () => { - await deleteTestUser(testUser.id); - }); - - tap.test('should update user profile', async () => { - // Test code using testUser - }); +// Basic test +tap.test('should perform basic arithmetic', async () => { + expect(2 + 2).toEqual(4); }); -``` -**Parallel Test Execution** -```typescript -// Files with matching parallel group names run concurrently -// test.auth.para__1.ts -tap.test('authentication test', async () => { /* ... */ }); - -// test.user.para__1.ts -tap.test('user operations test', async () => { /* ... */ }); -``` - -**Test Timeouts and Retries** -```typescript -tap.timeout(5000) - .retry(3) - .test('flaky network test', async (tools) => { - // This test has 5 seconds to complete and will retry up to 3 times - }); -``` - -**Snapshot Testing** -```typescript -tap.test('should match snapshot', async (tools) => { - const result = await generateReport(); - await tools.matchSnapshot(result); +// Async test with tools +tap.test('async operations', async (tools) => { + await tools.delayFor(100); // delay for 100ms + const result = await fetchData(); + expect(result).toBeDefined(); }); + +// Start test execution +tap.start(); ``` -**Test Fixtures** -```typescript -// Define a reusable fixture -tap.defineFixture('testUser', async () => ({ - id: 1, - name: 'Test User', - email: 'test@example.com' -})); +### Test Modifiers and Chaining -tap.test('user test', async (tools) => { - const user = tools.fixture('testUser'); - expect(user.name).toEqual('Test User'); -}); -``` - -**Skipping and Todo Tests** ```typescript -tap.skip.test('work in progress', async () => { +// Skip a test +tap.skip.test('not ready yet', async () => { // This test will be skipped }); -tap.todo('implement user deletion', async () => { - // This marks a test as todo +// Run only this test (exclusive) +tap.only.test('focus on this', async () => { + // Only this test will run +}); + +// Todo test +tap.todo('implement later', async () => { + // Marked as todo +}); + +// Chaining modifiers +tap.timeout(5000) + .retry(3) + .tags('api', 'integration') + .test('complex test', async (tools) => { + // Test with 5s timeout, 3 retries, and tags + }); +``` + +### Test Organization with describe() + +```typescript +tap.describe('User Management', () => { + let testDatabase; + + tap.beforeEach(async () => { + testDatabase = await createTestDB(); + }); + + tap.afterEach(async () => { + await testDatabase.cleanup(); + }); + + tap.test('should create user', async () => { + const user = await testDatabase.createUser({ name: 'John' }); + expect(user.id).toBeDefined(); + }); + + tap.describe('User Permissions', () => { + tap.test('should set admin role', async () => { + // Nested describe blocks + }); + }); }); ``` -**Browser Testing** +### Test Tools (Available in Test Function) + +Every test function receives a `tools` parameter with utilities: + +```typescript +tap.test('using test tools', async (tools) => { + // Delay utilities + await tools.delayFor(1000); // delay for 1000ms + await tools.delayForRandom(100, 500); // random delay between 100-500ms + + // Skip test conditionally + tools.skipIf(process.env.CI === 'true', 'Skipping in CI'); + + // Skip test unconditionally + if (!apiKeyAvailable) { + tools.skip('API key not available'); + } + + // Mark as todo + tools.todo('Needs implementation'); + + // Retry configuration + tools.retry(3); // Set retry count + + // Timeout configuration + tools.timeout(10000); // Set timeout to 10s + + // Context sharing between tests + tools.context.set('userId', 12345); + const userId = tools.context.get('userId'); + + // Deferred promises + const deferred = tools.defer(); + setTimeout(() => deferred.resolve('done'), 100); + await deferred.promise; + + // Colored console output + const coloredString = await tools.coloredString('Success!', 'green'); + console.log(coloredString); + + // Error handling helper + const error = await tools.returnError(async () => { + throw new Error('Expected error'); + }); + expect(error).toBeInstanceOf(Error); +}); +``` + +### Snapshot Testing + +```typescript +tap.test('snapshot test', async (tools) => { + const output = generateComplexOutput(); + + // Compare with saved snapshot + await tools.matchSnapshot(output); + + // Named snapshots for multiple checks in one test + await tools.matchSnapshot(output.header, 'header'); + await tools.matchSnapshot(output.body, 'body'); +}); + +// Update snapshots with: UPDATE_SNAPSHOTS=true tstest test/ +``` + +### Test Fixtures + +```typescript +// Define reusable fixtures +tap.defineFixture('testUser', async (data) => ({ + id: Date.now(), + name: data?.name || 'Test User', + email: data?.email || 'test@example.com', + created: new Date() +})); + +tap.defineFixture('testPost', async (data) => ({ + id: Date.now(), + title: data?.title || 'Test Post', + authorId: data?.authorId || 1 +})); + +// Use fixtures in tests +tap.test('fixture test', async (tools) => { + const user = await tools.fixture('testUser', { name: 'John' }); + const post = await tools.fixture('testPost', { authorId: user.id }); + + expect(post.authorId).toEqual(user.id); + + // Factory pattern for multiple instances + const users = await tools.factory('testUser').createMany(5); + expect(users).toHaveLength(5); +}); +``` + +### Parallel Test Execution + +```typescript +// Parallel tests within a file +tap.testParallel('parallel test 1', async () => { + await heavyOperation(); +}); + +tap.testParallel('parallel test 2', async () => { + await anotherHeavyOperation(); +}); + +// File naming for parallel groups +// test.api.para__1.ts - runs in parallel with other para__1 files +// test.db.para__1.ts - runs in parallel with other para__1 files +// test.auth.para__2.ts - runs after para__1 group completes +``` + +### Assertions with expect() + +tapbundle uses @push.rocks/smartexpect for assertions: + +```typescript +// Basic assertions +expect(value).toEqual(5); +expect(value).not.toEqual(10); +expect(obj).toDeepEqual({ a: 1, b: 2 }); + +// Type assertions +expect('hello').toBeTypeofString(); +expect(42).toBeTypeofNumber(); +expect(true).toBeTypeofBoolean(); +expect([]).toBeArray(); +expect({}).toBeTypeOf('object'); + +// Comparison assertions +expect(5).toBeGreaterThan(3); +expect(3).toBeLessThan(5); +expect(5).toBeGreaterThanOrEqual(5); +expect(5).toBeLessThanOrEqual(5); +expect(0.1 + 0.2).toBeCloseTo(0.3, 10); + +// Truthiness +expect(true).toBeTrue(); +expect(false).toBeFalse(); +expect('text').toBeTruthy(); +expect(0).toBeFalsy(); +expect(null).toBeNull(); +expect(undefined).toBeUndefined(); +expect(null).toBeNullOrUndefined(); + +// String assertions +expect('hello world').toStartWith('hello'); +expect('hello world').toEndWith('world'); +expect('hello world').toInclude('lo wo'); +expect('hello world').toMatch(/^hello/); +expect('option').toBeOneOf(['choice', 'option', 'alternative']); + +// Array assertions +expect([1, 2, 3]).toContain(2); +expect([1, 2, 3]).toContainAll([1, 3]); +expect([1, 2, 3]).toExclude(4); +expect([1, 2, 3]).toHaveLength(3); +expect([]).toBeEmptyArray(); +expect([{ id: 1 }]).toContainEqual({ id: 1 }); + +// Object assertions +expect(obj).toHaveProperty('name'); +expect(obj).toHaveProperty('user.email', 'test@example.com'); +expect(obj).toHaveDeepProperty(['level1', 'level2']); +expect(obj).toMatchObject({ name: 'John' }); + +// Function assertions +expect(() => { throw new Error('test'); }).toThrow(); +expect(() => { throw new Error('test'); }).toThrow(Error); +expect(() => { throw new Error('test error'); }).toThrowErrorMatching(/test/); +expect(myFunction).not.toThrow(); + +// Promise assertions +await expect(Promise.resolve('value')).resolves.toEqual('value'); +await expect(Promise.reject(new Error('fail'))).rejects.toThrow(); + +// Custom assertions +expect(7).customAssertion( + value => value % 2 === 1, + 'Value is not odd' +); +``` + +### Pre-tasks + +Run setup tasks before tests start: + +```typescript +tap.preTask('setup database', async () => { + await initializeTestDatabase(); + console.log('Database initialized'); +}); + +tap.preTask('load environment', async () => { + await loadTestEnvironment(); +}); + +// Pre-tasks run in order before any tests +``` + +### Tag-based Test Filtering + +```typescript +// Tag individual tests +tap.tags('unit', 'api') + .test('api unit test', async () => { + // Test code + }); + +tap.tags('integration', 'slow') + .test('database integration', async () => { + // Test code + }); + +// Run only tests with specific tags +// tstest test/ --tags unit,api +``` + +### Context Sharing + +Share data between tests: + +```typescript +tap.test('first test', async (tools) => { + const sessionId = await createSession(); + tools.context.set('sessionId', sessionId); +}); + +tap.test('second test', async (tools) => { + const sessionId = tools.context.get('sessionId'); + expect(sessionId).toBeDefined(); + + // Cleanup + tools.context.delete('sessionId'); +}); +``` + +### Browser Testing with webhelpers + +For browser-specific tests: + ```typescript -// test.browser.ts import { tap, webhelpers } from '@git.zone/tstest/tapbundle'; tap.test('DOM manipulation', async () => { + // Create DOM elements from HTML strings const element = await webhelpers.fixture(webhelpers.html` -
Hello World
+
+

Test Title

+ +
`); - expect(element).toBeInstanceOf(HTMLElement); + + expect(element.querySelector('h1').textContent).toEqual('Test Title'); + + // Simulate interactions + const button = element.querySelector('#test-btn'); + button.click(); +}); + +tap.test('CSS testing', async () => { + const styles = webhelpers.css` + .test-class { + color: red; + font-size: 16px; + } + `; + + // styles is a string that can be injected into the page + expect(styles).toInclude('color: red'); +}); +``` + +### Advanced Error Handling + +```typescript +tap.test('error handling', async (tools) => { + // Capture errors without failing the test + const error = await tools.returnError(async () => { + await functionThatThrows(); + }); + + expect(error).toBeInstanceOf(Error); + expect(error.message).toEqual('Expected error message'); +}); +``` + +### Test Wrap + +Create wrapped test environments: + +```typescript +import { TapWrap } from '@git.zone/tstest/tapbundle'; + +const tapWrap = new TapWrap({ + before: async () => { + console.log('Before all tests'); + await globalSetup(); + }, + after: async () => { + console.log('After all tests'); + await globalCleanup(); + } +}); + +// Tests registered here will have the wrap lifecycle +tapWrap.tap.test('wrapped test', async () => { + // This test runs with the wrap setup/teardown }); ``` @@ -330,6 +622,20 @@ tstest test/ --quiet ## Changelog +### Version 1.9.2 +- 🐛 Fixed test timing display issue (removed duplicate timing in output) +- 📝 Improved internal protocol design documentation +- 🔧 Added protocol v2 utilities for future improvements + +### Version 1.9.1 +- 🐛 Fixed log file naming to preserve directory structure +- 📁 Log files now prevent collisions: `test__dir__file.log` + +### Version 1.9.0 +- 📚 Comprehensive documentation update +- 🏗️ Embedded tapbundle for better integration +- 🌐 Full browser compatibility + ### Version 1.8.0 - 📦 Embedded tapbundle directly into tstest project - 🌐 Made tapbundle fully browser-compatible diff --git a/readme.plan.md b/readme.plan.md index 74fb3ed..f6398c9 100644 --- a/readme.plan.md +++ b/readme.plan.md @@ -2,6 +2,81 @@ !! FIRST: Reread /home/philkunz/.claude/CLAUDE.md to ensure following all guidelines !! +## Improved Internal Protocol (NEW - Critical) + +### Current Issues +- TAP protocol uses `#` for metadata which conflicts with test descriptions containing `#` +- Fragile regex parsing that breaks with special characters +- Limited extensibility for new metadata types + +### Proposed Solution: Protocol V2 +- Use Unicode delimiters `⟦TSTEST:META:{}⟧` that won't appear in test names +- Structured JSON metadata format +- Separate protocol blocks for complex data (errors, snapshots) +- Backwards compatible with gradual migration + +### Implementation +- Phase 1: Add protocol v2 parser alongside v1 +- Phase 2: Generate v2 by default with --legacy flag for v1 +- Phase 3: Full migration to v2 in next major version + +See `readme.protocol.md` for detailed specification. + +## Test Configuration System (NEW) + +### Global Test Configuration via 00init.ts +- **Discovery**: Check for `test/00init.ts` before running tests +- **Execution**: Import and execute before any test files if found +- **Purpose**: Define project-wide default test settings + +### tap.settings() API +```typescript +interface TapSettings { + // Timing + timeout?: number; // Default timeout for all tests (ms) + slowThreshold?: number; // Mark tests as slow if they exceed this (ms) + + // Execution Control + bail?: boolean; // Stop on first test failure + retries?: number; // Number of retries for failed tests + retryDelay?: number; // Delay between retries (ms) + + // Output Control + suppressConsole?: boolean; // Suppress console output in passing tests + verboseErrors?: boolean; // Show full stack traces + showTestDuration?: boolean; // Show duration for each test + + // Parallel Execution + maxConcurrency?: number; // Max parallel tests (for .para files) + isolateTests?: boolean; // Run each test in fresh context + + // Lifecycle Hooks + beforeAll?: () => Promise | void; + afterAll?: () => Promise | void; + beforeEach?: (testName: string) => Promise | void; + afterEach?: (testName: string, passed: boolean) => Promise | void; + + // Environment + env?: Record; // Additional environment variables + + // Features + enableSnapshots?: boolean; // Enable snapshot testing + snapshotDirectory?: string; // Custom snapshot directory + updateSnapshots?: boolean; // Update snapshots instead of comparing +} +``` + +### Settings Inheritance +- Global (00init.ts) → File level → Test level +- More specific settings override less specific ones +- Arrays/objects are merged, primitives are replaced + +### Implementation Phases +1. **Core Infrastructure**: Settings storage and merge logic +2. **Discovery**: 00init.ts loading mechanism +3. **Application**: Apply settings to test execution +4. **Advanced**: Parallel execution and snapshot configuration + ## 1. Enhanced Communication Between tapbundle and tstest ### 1.1 Real-time Test Progress API @@ -18,45 +93,9 @@ ## 2. Enhanced toolsArg Functionality -### 2.1 Test Flow Control ✅ -```typescript -tap.test('conditional test', async (toolsArg) => { - const result = await someOperation(); - - // Skip the rest of the test - if (!result) { - return toolsArg.skip('Precondition not met'); - } - - // Conditional skipping - await toolsArg.skipIf(condition, 'Reason for skipping'); - - // Mark test as todo - await toolsArg.todo('Not implemented yet'); -}); -``` - -### 2.2 Test Metadata and Configuration ✅ -```typescript -// Fluent syntax ✅ -tap.tags('slow', 'integration') - .priority('high') - .timeout(5000) - .retry(3) - .test('configurable test', async (toolsArg) => { - // Test implementation - }); -``` - -### 2.3 Test Data and Context Sharing ✅ +### 2.3 Test Data and Context Sharing (Partial) ```typescript tap.test('data-driven test', async (toolsArg) => { - // Access shared context ✅ - const sharedData = toolsArg.context.get('sharedData'); - - // Set data for other tests ✅ - toolsArg.context.set('resultData', computedValue); - // Parameterized test data (not yet implemented) const testData = toolsArg.data(); expect(processData(testData)).toEqual(expected); @@ -65,32 +104,7 @@ tap.test('data-driven test', async (toolsArg) => { ## 3. Nested Tests and Test Suites -### 3.1 Test Grouping with describe() ✅ -```typescript -tap.describe('User Authentication', () => { - tap.beforeEach(async (toolsArg) => { - // Setup for each test in this suite - await toolsArg.context.set('db', await createTestDatabase()); - }); - - tap.afterEach(async (toolsArg) => { - // Cleanup after each test - await toolsArg.context.get('db').cleanup(); - }); - - tap.test('should login with valid credentials', async (toolsArg) => { - // Test implementation - }); - - tap.describe('Password Reset', () => { - tap.test('should send reset email', async (toolsArg) => { - // Nested test - }); - }); -}); -``` - -### 3.2 Hierarchical Test Organization +### 3.2 Hierarchical Test Organization (Not yet implemented) - Support for multiple levels of nesting - Inherited context and configuration from parent suites - Aggregated reporting for test suites @@ -98,15 +112,7 @@ tap.describe('User Authentication', () => { ## 4. Advanced Test Features -### 4.1 Snapshot Testing -```typescript -tap.test('component render', async (toolsArg) => { - const output = renderComponent(props); - - // Compare with stored snapshot - await toolsArg.matchSnapshot(output, 'component-output'); -}); -``` +### 4.1 Snapshot Testing ✅ (Basic implementation complete) ### 4.2 Performance Benchmarking ```typescript @@ -124,30 +130,9 @@ tap.test('performance test', async (toolsArg) => { }); ``` -### 4.3 Test Fixtures and Factories ✅ -```typescript -tap.test('with fixtures', async (toolsArg) => { - // Create test fixtures - const user = await toolsArg.fixture('user', { name: 'Test User' }); - const post = await toolsArg.fixture('post', { author: user }); - - // Use factory functions - const users = await toolsArg.factory('user').createMany(5); -}); -``` ## 5. Test Execution Improvements -### 5.1 Parallel Test Execution ✅ -- Run independent tests concurrently ✅ -- Configurable concurrency limits (via file naming convention) -- Resource pooling for shared resources -- Proper isolation between parallel tests ✅ - -Implementation: -- Tests with `para__` in filename run in parallel -- Different groups run sequentially -- Tests without `para__` run serially ### 5.2 Watch Mode - Automatically re-run tests on file changes @@ -155,11 +140,8 @@ Implementation: - Fast feedback loop for development - Integration with IDE/editor plugins -### 5.3 Advanced Test Filtering ✅ (partially) +### 5.3 Advanced Test Filtering (Partial) ```typescript -// Run tests by tags ✅ -tstest --tags "unit,fast" - // Exclude tests by pattern (not yet implemented) tstest --exclude "**/slow/**" @@ -198,50 +180,36 @@ tstest --changed - Links to documentation - Code examples in error output -### 7.2 Interactive Mode (Needs Detailed Specification) -- REPL for exploring test failures - - Need to define: How to enter interactive mode? When tests fail? - - What commands/features should be available in the REPL? -- Debugging integration - - Node.js inspector protocol integration? - - Breakpoint support? -- Step-through test execution - - Pause between tests? - - Step into/over/out functionality? -- Interactive test data manipulation - - Modify test inputs on the fly? - - Inspect intermediate values? - -### 7.3 ~~VS Code Extension~~ (Scratched) -- ~~Test explorer integration~~ -- ~~Inline test results~~ -- ~~CodeLens for running individual tests~~ -- ~~Debugging support~~ - ## Implementation Phases -### Phase 1: Core Enhancements (Priority: High) ✅ -1. Implement enhanced toolsArg methods (skip, skipIf, timeout, retry) ✅ -2. Add basic test grouping with describe() ✅ -3. Improve error reporting between tapbundle and tstest ✅ +### Phase 1: Improved Internal Protocol (Priority: Critical) (NEW) +1. Implement Protocol V2 parser in tstest +2. Add protocol version negotiation +3. Update tapbundle to generate V2 format with feature flag +4. Test with real-world test suites containing special characters -### Phase 2: Advanced Features (Priority: Medium) -1. Implement nested test suites ✅ (basic describe support) -2. Add snapshot testing ✅ -3. Create test fixture system ✅ -4. Implement parallel test execution ✅ +### Phase 2: Test Configuration System (Priority: High) +1. Implement tap.settings() API with TypeScript interfaces +2. Add 00init.ts discovery and loading mechanism +3. Implement settings inheritance and merge logic +4. Apply settings to test execution (timeouts, retries, etc.) -### Phase 3: Developer Experience (Priority: Medium) +### Phase 3: Enhanced Communication (Priority: High) +1. Build on Protocol V2 for richer communication +2. Implement real-time test progress API +3. Add structured error reporting with diffs and traces + +### Phase 4: Developer Experience (Priority: Medium) 1. Add watch mode 2. Implement custom reporters -3. ~~Create VS Code extension~~ (Scratched) -4. Add interactive debugging (Needs detailed spec first) +3. Complete advanced test filtering options +4. Add performance benchmarking API -### Phase 4: Analytics and Performance (Priority: Low) +### Phase 5: Analytics and Performance (Priority: Low) 1. Build test analytics dashboard -2. Add performance benchmarking -3. Implement coverage integration -4. Create trend analysis tools +2. Implement coverage integration +3. Create trend analysis tools +4. Add test impact analysis ## Technical Considerations diff --git a/readme.protocol.md b/readme.protocol.md new file mode 100644 index 0000000..5991526 --- /dev/null +++ b/readme.protocol.md @@ -0,0 +1,287 @@ +# Improved Internal Protocol Design + +## Current Issues with TAP Protocol + +1. **Delimiter Conflict**: Using `#` for metadata conflicts with test descriptions containing `#` +2. **Ambiguous Parsing**: No clear boundary between test name and metadata +3. **Limited Extensibility**: Adding new metadata requires regex changes +4. **Mixed Concerns**: Protocol data mixed with human-readable output + +## Proposed Internal Protocol v2 + +### Design Principles + +1. **Clear Separation**: Protocol data must be unambiguously separated from user content +2. **Extensibility**: Easy to add new metadata without breaking parsers +3. **Backwards Compatible**: Can coexist with standard TAP for gradual migration +4. **Machine Readable**: Structured format for reliable parsing +5. **Human Friendly**: Still readable in raw form + +### Protocol Options + +#### Option 1: Special Delimiters +``` +ok 1 - test description ::TSTEST:: {"time":123,"retry":0} +not ok 2 - another test ::TSTEST:: {"time":45,"error":"timeout"} +ok 3 - skipped test ::TSTEST:: {"time":0,"skip":"not ready"} +``` + +**Pros**: +- Simple to implement +- Backwards compatible with TAP parsers (they ignore the suffix) +- Easy to parse with split() + +**Cons**: +- Still could conflict if test name contains `::TSTEST::` +- Not standard TAP + +#### Option 2: Separate Metadata Lines +``` +ok 1 - test description +::METADATA:: {"test":1,"time":123,"retry":0} +not ok 2 - another test +::METADATA:: {"test":2,"time":45,"error":"timeout"} +``` + +**Pros**: +- Complete separation of concerns +- No chance of conflicts +- Can include arbitrary metadata + +**Cons**: +- Requires correlation between lines +- More complex parsing + +#### Option 3: YAML Blocks (TAP 13 Compatible) +``` +ok 1 - test description + --- + time: 123 + retry: 0 + ... +not ok 2 - another test + --- + time: 45 + error: timeout + stack: | + Error: timeout + at Test.run (test.js:10:5) + ... +``` + +**Pros**: +- Standard TAP 13 feature +- Structured data format +- Human readable +- Extensible + +**Cons**: +- More verbose +- YAML parsing overhead + +#### Option 4: Binary Protocol Markers (Recommended) +``` +ok 1 - test description +␛[TSTEST:eyJ0aW1lIjoxMjMsInJldHJ5IjowfQ==]␛ +not ok 2 - another test +␛[TSTEST:eyJ0aW1lIjo0NSwiZXJyb3IiOiJ0aW1lb3V0In0=]␛ +``` + +Using ASCII escape character (␛ = \x1B) with base64 encoded JSON. + +**Pros**: +- Zero chance of accidental conflicts +- Compact +- Fast to parse +- Invisible in most terminals + +**Cons**: +- Not human readable in raw form +- Requires base64 encoding/decoding + +### Recommended Implementation: Hybrid Approach + +Use multiple strategies based on context: + +1. **For timing and basic metadata**: Use structured delimiters + ``` + ok 1 - test name ⟦time:123,retry:0⟧ + ``` + +2. **For complex data (errors, snapshots)**: Use separate protocol lines + ``` + ok 1 - test failed + ⟦TSTEST:ERROR⟧ + {"message":"Assertion failed","stack":"...","diff":"..."} + ⟦/TSTEST:ERROR⟧ + ``` + +3. **For human-readable output**: Keep standard TAP comments + ``` + # Test suite: User Authentication + ok 1 - should login + ``` + +### Implementation Plan + +#### Phase 1: Parser Enhancement +1. Add new protocol parser alongside existing TAP parser +2. Support both old and new formats during transition +3. Add protocol version negotiation + +#### Phase 2: Metadata Structure +```typescript +interface TestMetadata { + // Timing + time: number; // milliseconds + startTime?: number; // Unix timestamp + endTime?: number; // Unix timestamp + + // Status + skip?: string; // skip reason + todo?: string; // todo reason + retry?: number; // retry attempt + maxRetries?: number; // max retries allowed + + // Error details + error?: { + message: string; + stack?: string; + diff?: string; + actual?: any; + expected?: any; + }; + + // Test context + file?: string; // source file + line?: number; // line number + column?: number; // column number + + // Custom data + tags?: string[]; // test tags + custom?: Record; +} +``` + +#### Phase 3: Protocol Messages + +##### Success Message +``` +ok 1 - user authentication works +⟦TSTEST:META:{"time":123,"tags":["auth","unit"]}⟧ +``` + +##### Failure Message +``` +not ok 2 - login fails with invalid password +⟦TSTEST:META:{"time":45,"retry":1,"maxRetries":3}⟧ +⟦TSTEST:ERROR⟧ +{ + "message": "Expected 401 but got 500", + "stack": "Error: Expected 401 but got 500\n at Test.run (auth.test.ts:25:10)", + "actual": 500, + "expected": 401 +} +⟦/TSTEST:ERROR⟧ +``` + +##### Skip Message +``` +ok 3 - database integration test ⟦TSTEST:SKIP:No database connection⟧ +``` + +##### Snapshot Communication +``` +⟦TSTEST:SNAPSHOT:user-profile⟧ +{ + "name": "John Doe", + "email": "john@example.com", + "roles": ["user", "admin"] +} +⟦/TSTEST:SNAPSHOT⟧ +``` + +### Migration Strategy + +1. **Version Detection**: First line indicates protocol version + ``` + ⟦TSTEST:PROTOCOL:2.0⟧ + TAP version 13 + ``` + +2. **Gradual Rollout**: + - v1.10: Add protocol v2 parser, keep v1 generator + - v1.11: Generate v2 by default, v1 with --legacy flag + - v2.0: Remove v1 support + +3. **Feature Flags**: + ```typescript + tap.settings({ + protocol: 'v2', // or 'v1', 'auto' + protocolFeatures: { + structuredErrors: true, + enhancedTiming: true, + binaryMarkers: false + } + }); + ``` + +### Benefits of New Protocol + +1. **Reliability**: No more regex fragility or description conflicts +2. **Performance**: Faster parsing with clear boundaries +3. **Extensibility**: Easy to add new metadata fields +4. **Debugging**: Rich error information with stack traces and diffs +5. **Integration**: Better IDE and CI/CD tool integration +6. **Forward Compatible**: Room for future enhancements + +### Example Parser Implementation + +```typescript +class ProtocolV2Parser { + private readonly MARKER_START = '⟦TSTEST:'; + private readonly MARKER_END = '⟧'; + + parseMetadata(line: string): TestMetadata | null { + const start = line.lastIndexOf(this.MARKER_START); + if (start === -1) return null; + + const end = line.indexOf(this.MARKER_END, start); + if (end === -1) return null; + + const content = line.substring(start + this.MARKER_START.length, end); + const [type, data] = content.split(':', 2); + + switch (type) { + case 'META': + return JSON.parse(data); + case 'SKIP': + return { skip: data }; + case 'TODO': + return { todo: data }; + default: + return null; + } + } + + parseTestLine(line: string): ParsedTest { + // First extract any metadata + const metadata = this.parseMetadata(line); + + // Then parse the TAP part (without metadata) + const cleanLine = this.removeMetadata(line); + const tapResult = this.parseTAP(cleanLine); + + return { ...tapResult, metadata }; + } +} +``` + +### Next Steps + +1. Implement proof of concept with basic metadata support +2. Test with real-world test suites for edge cases +3. Benchmark parsing performance +4. Get feedback from users +5. Finalize protocol specification +6. Implement in both tapbundle and tstest \ No newline at end of file diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 31459ac..08253b7 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@git.zone/tstest', - version: '1.9.2', + version: '1.9.3', description: 'a test utility to run tests that match test/**/*.ts' } diff --git a/ts/tstest.classes.tap.parser.ts b/ts/tstest.classes.tap.parser.ts index ee68730..cd2f8b2 100644 --- a/ts/tstest.classes.tap.parser.ts +++ b/ts/tstest.classes.tap.parser.ts @@ -16,7 +16,7 @@ export class TapParser { expectedTests: number; receivedTests: number; - testStatusRegex = /(ok|not\sok)\s([0-9]+)\s-\s(.*)(\s#\s(.*))?$/; + testStatusRegex = /(ok|not\sok)\s([0-9]+)\s-\s(.*?)(\s#\s(.*))?$/; activeTapTestResult: TapTestResult; collectingErrorDetails: boolean = false; currentTestError: string[] = []; @@ -77,7 +77,7 @@ export class TapParser { return false; })(); - const testSubject = regexResult[3]; + const testSubject = regexResult[3].trim(); const testMetadata = regexResult[5]; // This will be either "time=XXXms" or "SKIP reason" or "TODO reason" let testDuration = 0; diff --git a/ts_tapbundle/index.ts b/ts_tapbundle/index.ts index bebf64b..1bbcdbf 100644 --- a/ts_tapbundle/index.ts +++ b/ts_tapbundle/index.ts @@ -1,6 +1,9 @@ export { tap } from './tapbundle.classes.tap.js'; export { TapWrap } from './tapbundle.classes.tapwrap.js'; export { webhelpers } from './webhelpers.js'; + +// Protocol utilities (for future protocol v2) +export * from './tapbundle.protocols.js'; export { TapTools } from './tapbundle.classes.taptools.js'; import { expect } from '@push.rocks/smartexpect'; diff --git a/ts_tapbundle/tapbundle.protocols.ts b/ts_tapbundle/tapbundle.protocols.ts new file mode 100644 index 0000000..70d1208 --- /dev/null +++ b/ts_tapbundle/tapbundle.protocols.ts @@ -0,0 +1,226 @@ +/** + * Internal protocol constants and utilities for improved TAP communication + * between tapbundle and tstest + */ + +export const PROTOCOL = { + VERSION: '2.0', + MARKERS: { + START: '⟦TSTEST:', + END: '⟧', + BLOCK_END: '⟦/TSTEST:', + }, + TYPES: { + META: 'META', + ERROR: 'ERROR', + SKIP: 'SKIP', + TODO: 'TODO', + SNAPSHOT: 'SNAPSHOT', + PROTOCOL: 'PROTOCOL', + } +} as const; + +export interface TestMetadata { + // Timing + time?: number; // milliseconds + startTime?: number; // Unix timestamp + endTime?: number; // Unix timestamp + + // Status + skip?: string; // skip reason + todo?: string; // todo reason + retry?: number; // retry attempt + maxRetries?: number; // max retries allowed + + // Error details + error?: { + message: string; + stack?: string; + diff?: string; + actual?: any; + expected?: any; + }; + + // Test context + file?: string; // source file + line?: number; // line number + column?: number; // column number + + // Custom data + tags?: string[]; // test tags + custom?: Record; +} + +export class ProtocolEncoder { + /** + * Encode metadata for inline inclusion + */ + static encodeInline(type: string, data: any): string { + if (typeof data === 'string') { + return `${PROTOCOL.MARKERS.START}${type}:${data}${PROTOCOL.MARKERS.END}`; + } + return `${PROTOCOL.MARKERS.START}${type}:${JSON.stringify(data)}${PROTOCOL.MARKERS.END}`; + } + + /** + * Encode block data for multi-line content + */ + static encodeBlock(type: string, data: any): string[] { + const lines: string[] = []; + lines.push(`${PROTOCOL.MARKERS.START}${type}${PROTOCOL.MARKERS.END}`); + + if (typeof data === 'string') { + lines.push(data); + } else { + lines.push(JSON.stringify(data, null, 2)); + } + + lines.push(`${PROTOCOL.MARKERS.BLOCK_END}${type}${PROTOCOL.MARKERS.END}`); + return lines; + } + + /** + * Create a TAP line with metadata + */ + static createTestLine( + status: 'ok' | 'not ok', + number: number, + description: string, + metadata?: TestMetadata + ): string { + let line = `${status} ${number} - ${description}`; + + if (metadata) { + // For skip/todo, use inline format for compatibility + if (metadata.skip) { + line += ` ${this.encodeInline(PROTOCOL.TYPES.SKIP, metadata.skip)}`; + } else if (metadata.todo) { + line += ` ${this.encodeInline(PROTOCOL.TYPES.TODO, metadata.todo)}`; + } else { + // For other metadata, append inline + const metaCopy = { ...metadata }; + delete metaCopy.error; // Error details go in separate block + + if (Object.keys(metaCopy).length > 0) { + line += ` ${this.encodeInline(PROTOCOL.TYPES.META, metaCopy)}`; + } + } + } + + return line; + } +} + +export class ProtocolDecoder { + /** + * Extract all protocol markers from a line + */ + static extractMarkers(line: string): Array<{type: string, data: any, start: number, end: number}> { + const markers: Array<{type: string, data: any, start: number, end: number}> = []; + let searchFrom = 0; + + while (true) { + const start = line.indexOf(PROTOCOL.MARKERS.START, searchFrom); + if (start === -1) break; + + const end = line.indexOf(PROTOCOL.MARKERS.END, start); + if (end === -1) break; + + const content = line.substring(start + PROTOCOL.MARKERS.START.length, end); + const colonIndex = content.indexOf(':'); + + if (colonIndex !== -1) { + const type = content.substring(0, colonIndex); + const dataStr = content.substring(colonIndex + 1); + + let data: any; + try { + // Try to parse as JSON first + data = JSON.parse(dataStr); + } catch { + // If not JSON, treat as string + data = dataStr; + } + + markers.push({ + type, + data, + start, + end: end + PROTOCOL.MARKERS.END.length + }); + } + + searchFrom = end + 1; + } + + return markers; + } + + /** + * Remove protocol markers from a line + */ + static cleanLine(line: string): string { + const markers = this.extractMarkers(line); + + // Remove markers from end to start to preserve indices + let cleanedLine = line; + for (let i = markers.length - 1; i >= 0; i--) { + const marker = markers[i]; + cleanedLine = cleanedLine.substring(0, marker.start) + + cleanedLine.substring(marker.end); + } + + return cleanedLine.trim(); + } + + /** + * Parse a test line and extract metadata + */ + static parseTestLine(line: string): { + cleaned: string; + metadata: TestMetadata; + } { + const markers = this.extractMarkers(line); + const metadata: TestMetadata = {}; + + for (const marker of markers) { + switch (marker.type) { + case PROTOCOL.TYPES.META: + Object.assign(metadata, marker.data); + break; + case PROTOCOL.TYPES.SKIP: + metadata.skip = marker.data; + break; + case PROTOCOL.TYPES.TODO: + metadata.todo = marker.data; + break; + } + } + + return { + cleaned: this.cleanLine(line), + metadata + }; + } + + /** + * Check if a line starts a protocol block + */ + static isBlockStart(line: string): {isBlock: boolean, type?: string} { + const trimmed = line.trim(); + if (trimmed.startsWith(PROTOCOL.MARKERS.START) && trimmed.endsWith(PROTOCOL.MARKERS.END)) { + const content = trimmed.slice(PROTOCOL.MARKERS.START.length, -PROTOCOL.MARKERS.END.length); + if (!content.includes(':')) { + return { isBlock: true, type: content }; + } + } + return { isBlock: false }; + } + + /** + * Check if a line ends a protocol block + */ + static isBlockEnd(line: string, type: string): boolean { + return line.trim() === `${PROTOCOL.MARKERS.BLOCK_END}${type}${PROTOCOL.MARKERS.END}`; + } +} \ No newline at end of file