Compare commits
14 Commits
Author | SHA1 | Date | |
---|---|---|---|
91880f8d42 | |||
7b1732abcc | |||
7d09b39f2b | |||
96efba5903 | |||
3c535a8a77 | |||
0954265095 | |||
e1d90589bc | |||
33f705d961 | |||
13b11ab1bf | |||
63280e4a9a | |||
23addc2d2f | |||
3649114c8d | |||
2841aba8a4 | |||
31bf090410 |
50
changelog.md
50
changelog.md
@ -1,5 +1,55 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2025-05-25 - 2.0.0 - BREAKING CHANGE(protocol)
|
||||||
|
Introduce protocol v2 implementation and update build configuration with revised build order, new tspublish files, and enhanced documentation
|
||||||
|
|
||||||
|
- Added ts_tapbundle_protocol directory with isomorphic implementation for protocol v2
|
||||||
|
- Updated readme.hints.md and readme.plan.md to explain the complete replacement of the v1 protocol and new build process
|
||||||
|
- Revised build order in tspublish.json files across ts, ts_tapbundle, ts_tapbundle_node, and ts_tapbundle_protocol
|
||||||
|
- Introduced .claude/settings.local.json with updated permission settings for CLI and build tools
|
||||||
|
|
||||||
|
## 2025-05-24 - 1.11.5 - fix(tstest)
|
||||||
|
Fix timeout handling to correctly evaluate TAP results after killing the test process.
|
||||||
|
|
||||||
|
- Added call to evaluateFinalResult() after killing the process in runInNode to ensure final TAP output is processed.
|
||||||
|
|
||||||
|
## 2025-05-24 - 1.11.4 - fix(logging)
|
||||||
|
Improve warning logging and add permission settings file
|
||||||
|
|
||||||
|
- Replace multiple logger.error calls with logger.warning for tests running over 1 minute
|
||||||
|
- Add warning method in tstest logger to display warning messages consistently
|
||||||
|
- Introduce .claude/settings.local.json to configure allowed permissions
|
||||||
|
|
||||||
|
## 2025-05-24 - 1.11.3 - fix(tstest)
|
||||||
|
Add timeout warning for long-running tests and introduce local settings configuration
|
||||||
|
|
||||||
|
- Add .claude/settings.local.json with permission configuration for local development
|
||||||
|
- Implement a timeout warning timer that notifies when tests run longer than 1 minute without an explicit timeout
|
||||||
|
- Clear the timeout warning timer upon test completion
|
||||||
|
- Remove unused import of logPrefixes in tstest.classes.tstest.ts
|
||||||
|
|
||||||
|
## 2025-05-24 - 1.11.2 - fix(tstest)
|
||||||
|
Improve timeout and error handling in test execution along with TAP parser timeout logic improvements.
|
||||||
|
|
||||||
|
- In the TAP parser, ensure that expected tests are properly set when no tests are defined to avoid false negatives on timeout.
|
||||||
|
- Use smartshell's terminate method and fallback kill to properly stop the entire process tree on timeout.
|
||||||
|
- Clean up browser, server, and WebSocket instances reliably even when a timeout occurs.
|
||||||
|
- Minor improvements in log file filtering and error logging for better clarity.
|
||||||
|
|
||||||
|
## 2025-05-24 - 1.11.1 - fix(tstest)
|
||||||
|
Clear timeout identifiers after successful test execution and add local CLAUDE settings
|
||||||
|
|
||||||
|
- Ensure timeout IDs are cleared when tests complete to prevent lingering timeouts
|
||||||
|
- Add .claude/settings.local.json with updated permission settings for CLI commands
|
||||||
|
|
||||||
|
## 2025-05-24 - 1.11.0 - feat(cli)
|
||||||
|
Add new timeout and file range options with enhanced logfile diff logging
|
||||||
|
|
||||||
|
- Introduce --timeout <seconds> option to safeguard tests from running too long
|
||||||
|
- Add --startFrom and --stopAt options to control the range of test files executed
|
||||||
|
- Enhance logfile organization by automatically moving previous logs and generating diff reports for failed or changed test outputs
|
||||||
|
- Update CLI argument parsing and internal timeout handling for both Node.js and browser tests
|
||||||
|
|
||||||
## 2025-05-24 - 1.10.2 - fix(tstest-logging)
|
## 2025-05-24 - 1.10.2 - fix(tstest-logging)
|
||||||
Improve log file handling with log rotation and diff reporting
|
Improve log file handling with log rotation and diff reporting
|
||||||
|
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@git.zone/tstest",
|
"name": "@git.zone/tstest",
|
||||||
"version": "1.10.2",
|
"version": "2.0.0",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "a test utility to run tests that match test/**/*.ts",
|
"description": "a test utility to run tests that match test/**/*.ts",
|
||||||
"exports": {
|
"exports": {
|
||||||
|
@ -40,9 +40,17 @@ This project integrates tstest with tapbundle through a modular architecture:
|
|||||||
- Automatically detects browser environment and only enables in browser context
|
- Automatically detects browser environment and only enables in browser context
|
||||||
|
|
||||||
3. **Build System**
|
3. **Build System**
|
||||||
- Uses `tsbuild tsfolders` to compile TypeScript
|
- Uses `tsbuild tsfolders` to compile TypeScript (invoked by `pnpm build`)
|
||||||
- Maintains separate output directories: `/dist_ts/`, `/dist_ts_tapbundle/`, `/dist_ts_tapbundle_node/`
|
- Maintains separate output directories: `/dist_ts/`, `/dist_ts_tapbundle/`, `/dist_ts_tapbundle_node/`, `/dist_ts_tapbundle_protocol/`
|
||||||
- Compilation order is resolved automatically based on dependencies
|
- Compilation order is resolved automatically based on dependencies in tspublish.json files
|
||||||
|
- Protocol imports use compiled dist directories:
|
||||||
|
```typescript
|
||||||
|
// In ts/tstest.classes.tap.parser.ts
|
||||||
|
import { ProtocolParser } from '../dist_ts_tapbundle_protocol/index.js';
|
||||||
|
|
||||||
|
// In ts_tapbundle/tapbundle.classes.tap.ts
|
||||||
|
import { ProtocolEmitter } from '../dist_ts_tapbundle_protocol/index.js';
|
||||||
|
```
|
||||||
|
|
||||||
### Test Scripts
|
### Test Scripts
|
||||||
|
|
||||||
@ -102,6 +110,19 @@ A new internal protocol is being designed that will:
|
|||||||
- Use Unicode delimiters `⟦TSTEST:⟧` that won't conflict with test content
|
- Use Unicode delimiters `⟦TSTEST:⟧` that won't conflict with test content
|
||||||
- Support structured JSON metadata
|
- Support structured JSON metadata
|
||||||
- Allow rich error reporting with stack traces and diffs
|
- Allow rich error reporting with stack traces and diffs
|
||||||
- Maintain backwards compatibility during migration
|
- Completely replace v1 protocol (no backwards compatibility)
|
||||||
|
|
||||||
See `readme.protocol.md` for the full specification and `tapbundle.protocols.ts` for the implementation utilities.
|
### ts_tapbundle_protocol Directory
|
||||||
|
The protocol v2 implementation is contained in a separate `ts_tapbundle_protocol` directory:
|
||||||
|
- **Isomorphic Code**: All protocol code works in both browser and Node.js environments
|
||||||
|
- **No Platform Dependencies**: No Node.js-specific imports, ensuring true cross-platform compatibility
|
||||||
|
- **Clean Separation**: Protocol logic is isolated from platform-specific code in tstest and tapbundle
|
||||||
|
- **Shared Implementation**: Both tstest (parser) and tapbundle (emitter) use the same protocol classes
|
||||||
|
- **Build Process**:
|
||||||
|
- Compiled by `pnpm build` via tsbuild to `dist_ts_tapbundle_protocol/`
|
||||||
|
- Build order managed through tspublish.json files
|
||||||
|
- Other modules import from the compiled dist directory, not source
|
||||||
|
|
||||||
|
This architectural decision ensures the protocol can be used in any JavaScript environment without modification and maintains proper build dependencies.
|
||||||
|
|
||||||
|
See `readme.protocol.md` for the full specification and `ts_tapbundle_protocol/` for the implementation.
|
128
readme.md
128
readme.md
@ -68,8 +68,11 @@ tstest "test/unit/*.ts"
|
|||||||
| `--verbose`, `-v` | Show all console output from tests |
|
| `--verbose`, `-v` | Show all console output from tests |
|
||||||
| `--no-color` | Disable colored output |
|
| `--no-color` | Disable colored output |
|
||||||
| `--json` | Output results as JSON |
|
| `--json` | Output results as JSON |
|
||||||
| `--logfile` | Save detailed logs to `.nogit/testlogs/[testname].log` |
|
| `--logfile` | Save detailed logs with automatic error and diff tracking |
|
||||||
| `--tags <tags>` | Run only tests with specific tags (comma-separated) |
|
| `--tags <tags>` | Run only tests with specific tags (comma-separated) |
|
||||||
|
| `--timeout <seconds>` | Timeout test files after specified seconds |
|
||||||
|
| `--startFrom <n>` | Start running from test file number n |
|
||||||
|
| `--stopAt <n>` | Stop running at test file number n |
|
||||||
|
|
||||||
### Example Outputs
|
### Example Outputs
|
||||||
|
|
||||||
@ -571,14 +574,88 @@ tstest "test/**/*.spec.ts" "test/**/*.test.ts"
|
|||||||
|
|
||||||
**Important**: Always quote glob patterns to prevent shell expansion. Without quotes, the shell will expand the pattern and only pass the first matching file to tstest.
|
**Important**: Always quote glob patterns to prevent shell expansion. Without quotes, the shell will expand the pattern and only pass the first matching file to tstest.
|
||||||
|
|
||||||
### Automatic Logging
|
### Enhanced Test Logging
|
||||||
|
|
||||||
|
The `--logfile` option provides intelligent test logging with automatic organization:
|
||||||
|
|
||||||
Use `--logfile` to automatically save test output:
|
|
||||||
```bash
|
```bash
|
||||||
tstest test/ --logfile
|
tstest test/ --logfile
|
||||||
```
|
```
|
||||||
|
|
||||||
This creates detailed logs in `.nogit/testlogs/[testname].log` for each test file.
|
**Log Organization:**
|
||||||
|
- **Current Run**: `.nogit/testlogs/[testname].log`
|
||||||
|
- **Previous Run**: `.nogit/testlogs/previous/[testname].log`
|
||||||
|
- **Failed Tests**: `.nogit/testlogs/00err/[testname].log`
|
||||||
|
- **Changed Output**: `.nogit/testlogs/00diff/[testname].log`
|
||||||
|
|
||||||
|
**Features:**
|
||||||
|
- Previous logs are automatically moved to the `previous/` folder
|
||||||
|
- Failed tests create copies in `00err/` for quick identification
|
||||||
|
- Tests with changed output create diff reports in `00diff/`
|
||||||
|
- The `00err/` and `00diff/` folders are cleared on each run
|
||||||
|
|
||||||
|
**Example Diff Report:**
|
||||||
|
```
|
||||||
|
DIFF REPORT: test__api__integration.log
|
||||||
|
Generated: 2025-05-24T01:29:13.847Z
|
||||||
|
================================================================================
|
||||||
|
|
||||||
|
- [Line 8] ✅ api test passes (150ms)
|
||||||
|
+ [Line 8] ✅ api test passes (165ms)
|
||||||
|
|
||||||
|
================================================================================
|
||||||
|
Previous version had 40 lines
|
||||||
|
Current version has 40 lines
|
||||||
|
```
|
||||||
|
|
||||||
|
### Test Timeout Protection
|
||||||
|
|
||||||
|
Prevent runaway tests with the `--timeout` option:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Timeout any test file that runs longer than 60 seconds
|
||||||
|
tstest test/ --timeout 60
|
||||||
|
|
||||||
|
# Shorter timeout for unit tests
|
||||||
|
tstest test/unit/ --timeout 10
|
||||||
|
```
|
||||||
|
|
||||||
|
When a test exceeds the timeout:
|
||||||
|
- The test process is terminated (SIGTERM)
|
||||||
|
- The test is marked as failed
|
||||||
|
- An error log is created in `.nogit/testlogs/00err/`
|
||||||
|
- Clear error message shows the timeout duration
|
||||||
|
|
||||||
|
### Test File Range Control
|
||||||
|
|
||||||
|
Run specific ranges of test files using `--startFrom` and `--stopAt`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run tests starting from the 5th file
|
||||||
|
tstest test/ --startFrom 5
|
||||||
|
|
||||||
|
# Run only files 5 through 10
|
||||||
|
tstest test/ --startFrom 5 --stopAt 10
|
||||||
|
|
||||||
|
# Run only the first 3 test files
|
||||||
|
tstest test/ --stopAt 3
|
||||||
|
```
|
||||||
|
|
||||||
|
This is particularly useful for:
|
||||||
|
- Debugging specific test failures in large test suites
|
||||||
|
- Running tests in chunks on different CI runners
|
||||||
|
- Quickly testing changes to specific test files
|
||||||
|
|
||||||
|
The output shows which files are skipped:
|
||||||
|
```
|
||||||
|
⏭️ test/auth.test.ts (1/10)
|
||||||
|
Skipped: before start range (5)
|
||||||
|
⏭️ test/user.test.ts (2/10)
|
||||||
|
Skipped: before start range (5)
|
||||||
|
▶️ test/api.test.ts (5/10)
|
||||||
|
Runtime: node.js
|
||||||
|
✅ api endpoints work (145ms)
|
||||||
|
```
|
||||||
|
|
||||||
### Performance Analysis
|
### Performance Analysis
|
||||||
|
|
||||||
@ -620,8 +697,51 @@ tstest test/ --json > test-results.json
|
|||||||
tstest test/ --quiet
|
tstest test/ --quiet
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Advanced CI Example:**
|
||||||
|
```bash
|
||||||
|
# Run tests with comprehensive logging and safety features
|
||||||
|
tstest test/ \
|
||||||
|
--timeout 300 \
|
||||||
|
--logfile \
|
||||||
|
--json > test-results.json
|
||||||
|
|
||||||
|
# Run specific test chunks in parallel CI jobs
|
||||||
|
tstest test/ --startFrom 1 --stopAt 10 # Job 1
|
||||||
|
tstest test/ --startFrom 11 --stopAt 20 # Job 2
|
||||||
|
tstest test/ --startFrom 21 # Job 3
|
||||||
|
```
|
||||||
|
|
||||||
|
### Debugging Failed Tests
|
||||||
|
|
||||||
|
When tests fail, use the enhanced logging features:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run with logging to capture detailed output
|
||||||
|
tstest test/ --logfile --verbose
|
||||||
|
|
||||||
|
# Check error logs
|
||||||
|
ls .nogit/testlogs/00err/
|
||||||
|
|
||||||
|
# Review diffs for flaky tests
|
||||||
|
cat .nogit/testlogs/00diff/test__api__endpoints.log
|
||||||
|
|
||||||
|
# Re-run specific failed tests
|
||||||
|
tstest test/api/endpoints.test.ts --verbose --timeout 60
|
||||||
|
```
|
||||||
|
|
||||||
## Changelog
|
## Changelog
|
||||||
|
|
||||||
|
### Version 1.10.0
|
||||||
|
- ⏱️ Added `--timeout <seconds>` option for test file timeout protection
|
||||||
|
- 🎯 Added `--startFrom <n>` and `--stopAt <n>` options for test file range control
|
||||||
|
- 📁 Enhanced `--logfile` with intelligent log organization:
|
||||||
|
- Previous logs moved to `previous/` folder
|
||||||
|
- Failed tests copied to `00err/` folder
|
||||||
|
- Changed tests create diff reports in `00diff/` folder
|
||||||
|
- 🔍 Improved test discovery to show skipped files with clear reasons
|
||||||
|
- 🐛 Fixed TypeScript compilation warnings and unused variables
|
||||||
|
- 📊 Test summaries now include skipped file counts
|
||||||
|
|
||||||
### Version 1.9.2
|
### Version 1.9.2
|
||||||
- 🐛 Fixed test timing display issue (removed duplicate timing in output)
|
- 🐛 Fixed test timing display issue (removed duplicate timing in output)
|
||||||
- 📝 Improved internal protocol design documentation
|
- 📝 Improved internal protocol design documentation
|
||||||
|
@ -13,12 +13,27 @@
|
|||||||
- Use Unicode delimiters `⟦TSTEST:META:{}⟧` that won't appear in test names
|
- Use Unicode delimiters `⟦TSTEST:META:{}⟧` that won't appear in test names
|
||||||
- Structured JSON metadata format
|
- Structured JSON metadata format
|
||||||
- Separate protocol blocks for complex data (errors, snapshots)
|
- Separate protocol blocks for complex data (errors, snapshots)
|
||||||
- Backwards compatible with gradual migration
|
- Complete replacement of v1 (no backwards compatibility needed)
|
||||||
|
|
||||||
### Implementation
|
### Implementation
|
||||||
- Phase 1: Add protocol v2 parser alongside v1
|
- Phase 1: Create protocol v2 implementation in ts_tapbundle_protocol
|
||||||
- Phase 2: Generate v2 by default with --legacy flag for v1
|
- Phase 2: Replace all v1 code in both tstest and tapbundle with v2
|
||||||
- Phase 3: Full migration to v2 in next major version
|
- Phase 3: Delete all v1 parsing and generation code
|
||||||
|
|
||||||
|
#### ts_tapbundle_protocol Directory
|
||||||
|
The protocol v2 implementation will be contained in the `ts_tapbundle_protocol` directory as isomorphic TypeScript code:
|
||||||
|
- **Isomorphic Design**: All code must work in both browser and Node.js environments
|
||||||
|
- **No Node.js Imports**: No Node.js-specific modules allowed (no fs, path, child_process, etc.)
|
||||||
|
- **Protocol Classes**: Contains classes implementing all sides of the protocol:
|
||||||
|
- `ProtocolEmitter`: For generating protocol v2 messages (used by tapbundle)
|
||||||
|
- `ProtocolParser`: For parsing protocol v2 messages (used by tstest)
|
||||||
|
- `ProtocolMessage`: Base classes for different message types
|
||||||
|
- `ProtocolTypes`: TypeScript interfaces and types for protocol structures
|
||||||
|
- **Pure TypeScript**: Only browser-compatible APIs and pure TypeScript/JavaScript code
|
||||||
|
- **Build Integration**:
|
||||||
|
- Compiled by `pnpm build` (via tsbuild) to `dist_ts_tapbundle_protocol/`
|
||||||
|
- Build order defined in tspublish.json files
|
||||||
|
- Imported by ts and ts_tapbundle modules from the compiled dist directory
|
||||||
|
|
||||||
See `readme.protocol.md` for detailed specification.
|
See `readme.protocol.md` for detailed specification.
|
||||||
|
|
||||||
@ -183,10 +198,18 @@ tstest --changed
|
|||||||
## Implementation Phases
|
## Implementation Phases
|
||||||
|
|
||||||
### Phase 1: Improved Internal Protocol (Priority: Critical) (NEW)
|
### Phase 1: Improved Internal Protocol (Priority: Critical) (NEW)
|
||||||
1. Implement Protocol V2 parser in tstest
|
1. Create ts_tapbundle_protocol directory with isomorphic protocol v2 implementation
|
||||||
2. Add protocol version negotiation
|
- Implement ProtocolEmitter class for message generation
|
||||||
3. Update tapbundle to generate V2 format with feature flag
|
- Implement ProtocolParser class for message parsing
|
||||||
4. Test with real-world test suites containing special characters
|
- Define ProtocolMessage types and interfaces
|
||||||
|
- Ensure all code is browser and Node.js compatible
|
||||||
|
- Add tspublish.json to configure build order
|
||||||
|
2. Update build configuration to compile ts_tapbundle_protocol first
|
||||||
|
3. Replace TAP parser in tstest with Protocol V2 parser importing from dist_ts_tapbundle_protocol
|
||||||
|
4. Replace TAP generation in tapbundle with Protocol V2 emitter importing from dist_ts_tapbundle_protocol
|
||||||
|
5. Delete all v1 TAP parsing code from tstest
|
||||||
|
6. Delete all v1 TAP generation code from tapbundle
|
||||||
|
7. Test with real-world test suites containing special characters
|
||||||
|
|
||||||
### Phase 2: Test Configuration System (Priority: High)
|
### Phase 2: Test Configuration System (Priority: High)
|
||||||
1. Implement tap.settings() API with TypeScript interfaces
|
1. Implement tap.settings() API with TypeScript interfaces
|
||||||
@ -214,10 +237,10 @@ tstest --changed
|
|||||||
## Technical Considerations
|
## Technical Considerations
|
||||||
|
|
||||||
### API Design Principles
|
### API Design Principles
|
||||||
- Maintain backward compatibility
|
- Clean, modern API design without legacy constraints
|
||||||
- Progressive enhancement approach
|
- Progressive enhancement approach
|
||||||
- Opt-in features to avoid breaking changes
|
- Well-documented features and APIs
|
||||||
- Clear migration paths for new features
|
- Clear, simple interfaces
|
||||||
|
|
||||||
### Performance Goals
|
### Performance Goals
|
||||||
- Minimal overhead for test execution
|
- Minimal overhead for test execution
|
||||||
|
@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@git.zone/tstest',
|
name: '@git.zone/tstest',
|
||||||
version: '1.10.2',
|
version: '2.0.0',
|
||||||
description: 'a test utility to run tests that match test/**/*.ts'
|
description: 'a test utility to run tests that match test/**/*.ts'
|
||||||
}
|
}
|
||||||
|
17
ts/index.ts
17
ts/index.ts
@ -15,6 +15,7 @@ export const runCli = async () => {
|
|||||||
let tags: string[] = [];
|
let tags: string[] = [];
|
||||||
let startFromFile: number | null = null;
|
let startFromFile: number | null = null;
|
||||||
let stopAtFile: number | null = null;
|
let stopAtFile: number | null = null;
|
||||||
|
let timeoutSeconds: number | null = null;
|
||||||
|
|
||||||
// Parse options
|
// Parse options
|
||||||
for (let i = 0; i < args.length; i++) {
|
for (let i = 0; i < args.length; i++) {
|
||||||
@ -70,6 +71,19 @@ export const runCli = async () => {
|
|||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case '--timeout':
|
||||||
|
if (i + 1 < args.length) {
|
||||||
|
const value = parseInt(args[++i], 10);
|
||||||
|
if (isNaN(value) || value < 1) {
|
||||||
|
console.error('Error: --timeout must be a positive integer (seconds)');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
timeoutSeconds = value;
|
||||||
|
} else {
|
||||||
|
console.error('Error: --timeout requires a number argument (seconds)');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
if (!arg.startsWith('-')) {
|
if (!arg.startsWith('-')) {
|
||||||
testPath = arg;
|
testPath = arg;
|
||||||
@ -95,6 +109,7 @@ export const runCli = async () => {
|
|||||||
console.error(' --tags <tags> Run only tests with specified tags (comma-separated)');
|
console.error(' --tags <tags> Run only tests with specified tags (comma-separated)');
|
||||||
console.error(' --startFrom <n> Start running from test file number n');
|
console.error(' --startFrom <n> Start running from test file number n');
|
||||||
console.error(' --stopAt <n> Stop running at test file number n');
|
console.error(' --stopAt <n> Stop running at test file number n');
|
||||||
|
console.error(' --timeout <s> Timeout test files after s seconds');
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -109,7 +124,7 @@ export const runCli = async () => {
|
|||||||
executionMode = TestExecutionMode.DIRECTORY;
|
executionMode = TestExecutionMode.DIRECTORY;
|
||||||
}
|
}
|
||||||
|
|
||||||
const tsTestInstance = new TsTest(process.cwd(), testPath, executionMode, logOptions, tags, startFromFile, stopAtFile);
|
const tsTestInstance = new TsTest(process.cwd(), testPath, executionMode, logOptions, tags, startFromFile, stopAtFile, timeoutSeconds);
|
||||||
await tsTestInstance.run();
|
await tsTestInstance.run();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
{
|
{
|
||||||
"order": 2
|
"order": 4
|
||||||
}
|
}
|
@ -32,6 +32,36 @@ export class TapParser {
|
|||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle test file timeout
|
||||||
|
*/
|
||||||
|
public handleTimeout(timeoutSeconds: number) {
|
||||||
|
// If no tests have been defined yet, set expected to 1
|
||||||
|
if (this.expectedTests === 0) {
|
||||||
|
this.expectedTests = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a fake failing test result for timeout
|
||||||
|
this._getNewTapTestResult();
|
||||||
|
this.activeTapTestResult.testOk = false;
|
||||||
|
this.activeTapTestResult.testSettled = true;
|
||||||
|
this.testStore.push(this.activeTapTestResult);
|
||||||
|
|
||||||
|
// Log the timeout error
|
||||||
|
if (this.logger) {
|
||||||
|
// First log the test result
|
||||||
|
this.logger.testResult(
|
||||||
|
`Test file timeout`,
|
||||||
|
false,
|
||||||
|
timeoutSeconds * 1000,
|
||||||
|
`Error: Test file exceeded timeout of ${timeoutSeconds} seconds`
|
||||||
|
);
|
||||||
|
this.logger.testErrorDetails(`Test execution was terminated after ${timeoutSeconds} seconds`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Don't call evaluateFinalResult here, let the caller handle it
|
||||||
|
}
|
||||||
|
|
||||||
private _getNewTapTestResult() {
|
private _getNewTapTestResult() {
|
||||||
this.activeTapTestResult = new TapTestResult(this.testStore.length + 1);
|
this.activeTapTestResult = new TapTestResult(this.testStore.length + 1);
|
||||||
}
|
}
|
||||||
@ -69,7 +99,7 @@ export class TapParser {
|
|||||||
} else if (this.testStatusRegex.test(logLine)) {
|
} else if (this.testStatusRegex.test(logLine)) {
|
||||||
logLineIsTapProtocol = true;
|
logLineIsTapProtocol = true;
|
||||||
const regexResult = this.testStatusRegex.exec(logLine);
|
const regexResult = this.testStatusRegex.exec(logLine);
|
||||||
const testId = parseInt(regexResult[2]);
|
// const testId = parseInt(regexResult[2]); // Currently unused
|
||||||
const testOk = (() => {
|
const testOk = (() => {
|
||||||
if (regexResult[1] === 'ok') {
|
if (regexResult[1] === 'ok') {
|
||||||
return true;
|
return true;
|
||||||
@ -81,21 +111,16 @@ export class TapParser {
|
|||||||
const testMetadata = regexResult[5]; // This will be either "time=XXXms" or "SKIP reason" or "TODO reason"
|
const testMetadata = regexResult[5]; // This will be either "time=XXXms" or "SKIP reason" or "TODO reason"
|
||||||
|
|
||||||
let testDuration = 0;
|
let testDuration = 0;
|
||||||
let isSkipped = false;
|
|
||||||
let isTodo = false;
|
|
||||||
|
|
||||||
if (testMetadata) {
|
if (testMetadata) {
|
||||||
const timeMatch = testMetadata.match(/time=(\d+)ms/);
|
const timeMatch = testMetadata.match(/time=(\d+)ms/);
|
||||||
const skipMatch = testMetadata.match(/SKIP\s*(.*)/);
|
// const skipMatch = testMetadata.match(/SKIP\s*(.*)/); // Currently unused
|
||||||
const todoMatch = testMetadata.match(/TODO\s*(.*)/);
|
// const todoMatch = testMetadata.match(/TODO\s*(.*)/); // Currently unused
|
||||||
|
|
||||||
if (timeMatch) {
|
if (timeMatch) {
|
||||||
testDuration = parseInt(timeMatch[1]);
|
testDuration = parseInt(timeMatch[1]);
|
||||||
} else if (skipMatch) {
|
|
||||||
isSkipped = true;
|
|
||||||
} else if (todoMatch) {
|
|
||||||
isTodo = true;
|
|
||||||
}
|
}
|
||||||
|
// Skip/todo handling could be added here in the future
|
||||||
}
|
}
|
||||||
|
|
||||||
// test for protocol error - disabled as it's not critical
|
// test for protocol error - disabled as it's not critical
|
||||||
@ -305,13 +330,16 @@ export class TapParser {
|
|||||||
this.logger.error(`Only ${this.receivedTests} out of ${this.expectedTests} completed!`);
|
this.logger.error(`Only ${this.receivedTests} out of ${this.expectedTests} completed!`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!this.expectedTests) {
|
if (!this.expectedTests && this.receivedTests === 0) {
|
||||||
if (this.logger) {
|
if (this.logger) {
|
||||||
this.logger.error('No tests were defined. Therefore the testfile failed!');
|
this.logger.error('No tests were defined. Therefore the testfile failed!');
|
||||||
|
this.logger.testFileEnd(0, 1, 0); // Count as 1 failure
|
||||||
}
|
}
|
||||||
} else if (this.expectedTests !== this.receivedTests) {
|
} else if (this.expectedTests !== this.receivedTests) {
|
||||||
if (this.logger) {
|
if (this.logger) {
|
||||||
this.logger.error('The amount of received tests and expectedTests is unequal! Therefore the testfile failed');
|
this.logger.error('The amount of received tests and expectedTests is unequal! Therefore the testfile failed');
|
||||||
|
const errorCount = this.getErrorTests().length || 1; // At least 1 error
|
||||||
|
this.logger.testFileEnd(this.receivedTests - errorCount, errorCount, 0);
|
||||||
}
|
}
|
||||||
} else if (this.getErrorTests().length === 0) {
|
} else if (this.getErrorTests().length === 0) {
|
||||||
if (this.logger) {
|
if (this.logger) {
|
||||||
|
@ -1,6 +1,5 @@
|
|||||||
import * as plugins from './tstest.plugins.js';
|
import * as plugins from './tstest.plugins.js';
|
||||||
import * as paths from './tstest.paths.js';
|
import * as paths from './tstest.paths.js';
|
||||||
import * as logPrefixes from './tstest.logprefixes.js';
|
|
||||||
|
|
||||||
import { coloredString as cs } from '@push.rocks/consolecolor';
|
import { coloredString as cs } from '@push.rocks/consolecolor';
|
||||||
|
|
||||||
@ -18,6 +17,8 @@ export class TsTest {
|
|||||||
public filterTags: string[];
|
public filterTags: string[];
|
||||||
public startFromFile: number | null;
|
public startFromFile: number | null;
|
||||||
public stopAtFile: number | null;
|
public stopAtFile: number | null;
|
||||||
|
public timeoutSeconds: number | null;
|
||||||
|
private timeoutWarningTimer: NodeJS.Timeout | null = null;
|
||||||
|
|
||||||
public smartshellInstance = new plugins.smartshell.Smartshell({
|
public smartshellInstance = new plugins.smartshell.Smartshell({
|
||||||
executor: 'bash',
|
executor: 'bash',
|
||||||
@ -28,13 +29,14 @@ export class TsTest {
|
|||||||
|
|
||||||
public tsbundleInstance = new plugins.tsbundle.TsBundle();
|
public tsbundleInstance = new plugins.tsbundle.TsBundle();
|
||||||
|
|
||||||
constructor(cwdArg: string, testPathArg: string, executionModeArg: TestExecutionMode, logOptions: LogOptions = {}, tags: string[] = [], startFromFile: number | null = null, stopAtFile: number | null = null) {
|
constructor(cwdArg: string, testPathArg: string, executionModeArg: TestExecutionMode, logOptions: LogOptions = {}, tags: string[] = [], startFromFile: number | null = null, stopAtFile: number | null = null, timeoutSeconds: number | null = null) {
|
||||||
this.executionMode = executionModeArg;
|
this.executionMode = executionModeArg;
|
||||||
this.testDir = new TestDirectory(cwdArg, testPathArg, executionModeArg);
|
this.testDir = new TestDirectory(cwdArg, testPathArg, executionModeArg);
|
||||||
this.logger = new TsTestLogger(logOptions);
|
this.logger = new TsTestLogger(logOptions);
|
||||||
this.filterTags = tags;
|
this.filterTags = tags;
|
||||||
this.startFromFile = startFromFile;
|
this.startFromFile = startFromFile;
|
||||||
this.stopAtFile = stopAtFile;
|
this.stopAtFile = stopAtFile;
|
||||||
|
this.timeoutSeconds = timeoutSeconds;
|
||||||
}
|
}
|
||||||
|
|
||||||
async run() {
|
async run() {
|
||||||
@ -43,6 +45,15 @@ export class TsTest {
|
|||||||
await this.movePreviousLogFiles();
|
await this.movePreviousLogFiles();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Start timeout warning timer if no timeout was specified
|
||||||
|
if (this.timeoutSeconds === null) {
|
||||||
|
this.timeoutWarningTimer = setTimeout(() => {
|
||||||
|
this.logger.warning('Test is running for more than 1 minute.');
|
||||||
|
this.logger.warning('Consider using --timeout option to set a timeout for test files.');
|
||||||
|
this.logger.warning('Example: tstest test --timeout=300 (for 5 minutes)');
|
||||||
|
}, 60000); // 1 minute
|
||||||
|
}
|
||||||
|
|
||||||
const testGroups = await this.testDir.getTestFileGroups();
|
const testGroups = await this.testDir.getTestFileGroups();
|
||||||
const allFiles = [...testGroups.serial, ...Object.values(testGroups.parallelGroups).flat()];
|
const allFiles = [...testGroups.serial, ...Object.values(testGroups.parallelGroups).flat()];
|
||||||
|
|
||||||
@ -81,6 +92,12 @@ export class TsTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clear the timeout warning timer if it was set
|
||||||
|
if (this.timeoutWarningTimer) {
|
||||||
|
clearTimeout(this.timeoutWarningTimer);
|
||||||
|
this.timeoutWarningTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
tapCombinator.evaluate();
|
tapCombinator.evaluate();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -147,7 +164,42 @@ export class TsTest {
|
|||||||
const execResultStreaming = await this.smartshellInstance.execStreamingSilent(
|
const execResultStreaming = await this.smartshellInstance.execStreamingSilent(
|
||||||
`tsrun ${fileNameArg}${tsrunOptions}`
|
`tsrun ${fileNameArg}${tsrunOptions}`
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Handle timeout if specified
|
||||||
|
if (this.timeoutSeconds !== null) {
|
||||||
|
const timeoutMs = this.timeoutSeconds * 1000;
|
||||||
|
let timeoutId: NodeJS.Timeout;
|
||||||
|
|
||||||
|
const timeoutPromise = new Promise<void>((_resolve, reject) => {
|
||||||
|
timeoutId = setTimeout(async () => {
|
||||||
|
// Use smartshell's terminate() to kill entire process tree
|
||||||
|
await execResultStreaming.terminate();
|
||||||
|
reject(new Error(`Test file timed out after ${this.timeoutSeconds} seconds`));
|
||||||
|
}, timeoutMs);
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await Promise.race([
|
||||||
|
tapParser.handleTapProcess(execResultStreaming.childProcess),
|
||||||
|
timeoutPromise
|
||||||
|
]);
|
||||||
|
// Clear timeout if test completed successfully
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
} catch (error) {
|
||||||
|
// Handle timeout error
|
||||||
|
tapParser.handleTimeout(this.timeoutSeconds);
|
||||||
|
// Ensure entire process tree is killed if still running
|
||||||
|
try {
|
||||||
|
await execResultStreaming.kill(); // This kills the entire process tree with SIGKILL
|
||||||
|
} catch (killError) {
|
||||||
|
// Process tree might already be dead
|
||||||
|
}
|
||||||
|
await tapParser.evaluateFinalResult();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
await tapParser.handleTapProcess(execResultStreaming.childProcess);
|
await tapParser.handleTapProcess(execResultStreaming.childProcess);
|
||||||
|
}
|
||||||
|
|
||||||
return tapParser;
|
return tapParser;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -205,9 +257,10 @@ export class TsTest {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// lets do the browser bit
|
// lets do the browser bit with timeout handling
|
||||||
await this.smartbrowserInstance.start();
|
await this.smartbrowserInstance.start();
|
||||||
await this.smartbrowserInstance.evaluateOnPage(
|
|
||||||
|
const evaluatePromise = this.smartbrowserInstance.evaluateOnPage(
|
||||||
`http://localhost:3007/test?bundleName=${bundleFileName}`,
|
`http://localhost:3007/test?bundleName=${bundleFileName}`,
|
||||||
async () => {
|
async () => {
|
||||||
// lets enable real time comms
|
// lets enable real time comms
|
||||||
@ -264,13 +317,56 @@ export class TsTest {
|
|||||||
return logStore.join('\n');
|
return logStore.join('\n');
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Handle timeout if specified
|
||||||
|
if (this.timeoutSeconds !== null) {
|
||||||
|
const timeoutMs = this.timeoutSeconds * 1000;
|
||||||
|
let timeoutId: NodeJS.Timeout;
|
||||||
|
|
||||||
|
const timeoutPromise = new Promise<void>((_resolve, reject) => {
|
||||||
|
timeoutId = setTimeout(() => {
|
||||||
|
reject(new Error(`Test file timed out after ${this.timeoutSeconds} seconds`));
|
||||||
|
}, timeoutMs);
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await Promise.race([
|
||||||
|
evaluatePromise,
|
||||||
|
timeoutPromise
|
||||||
|
]);
|
||||||
|
// Clear timeout if test completed successfully
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
} catch (error) {
|
||||||
|
// Handle timeout error
|
||||||
|
tapParser.handleTimeout(this.timeoutSeconds);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
await evaluatePromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Always clean up resources, even on timeout
|
||||||
|
try {
|
||||||
await this.smartbrowserInstance.stop();
|
await this.smartbrowserInstance.stop();
|
||||||
|
} catch (error) {
|
||||||
|
// Browser might already be stopped
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
await server.stop();
|
await server.stop();
|
||||||
|
} catch (error) {
|
||||||
|
// Server might already be stopped
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
wss.close();
|
wss.close();
|
||||||
|
} catch (error) {
|
||||||
|
// WebSocket server might already be closed
|
||||||
|
}
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`${cs('=> ', 'blue')} Stopped ${cs(fileNameArg, 'orange')} chromium instance and server.`
|
`${cs('=> ', 'blue')} Stopped ${cs(fileNameArg, 'orange')} chromium instance and server.`
|
||||||
);
|
);
|
||||||
// lets create the tap parser
|
// Always evaluate final result (handleTimeout just sets up the test state)
|
||||||
await tapParser.evaluateFinalResult();
|
await tapParser.evaluateFinalResult();
|
||||||
return tapParser;
|
return tapParser;
|
||||||
}
|
}
|
||||||
@ -280,28 +376,38 @@ export class TsTest {
|
|||||||
private async movePreviousLogFiles() {
|
private async movePreviousLogFiles() {
|
||||||
const logDir = plugins.path.join('.nogit', 'testlogs');
|
const logDir = plugins.path.join('.nogit', 'testlogs');
|
||||||
const previousDir = plugins.path.join('.nogit', 'testlogs', 'previous');
|
const previousDir = plugins.path.join('.nogit', 'testlogs', 'previous');
|
||||||
|
const errDir = plugins.path.join('.nogit', 'testlogs', '00err');
|
||||||
|
const diffDir = plugins.path.join('.nogit', 'testlogs', '00diff');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Get all files in log directory
|
// Delete 00err and 00diff directories if they exist
|
||||||
|
if (await plugins.smartfile.fs.isDirectory(errDir)) {
|
||||||
|
await plugins.smartfile.fs.remove(errDir);
|
||||||
|
}
|
||||||
|
if (await plugins.smartfile.fs.isDirectory(diffDir)) {
|
||||||
|
await plugins.smartfile.fs.remove(diffDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get all .log files in log directory (not in subdirectories)
|
||||||
const files = await plugins.smartfile.fs.listFileTree(logDir, '*.log');
|
const files = await plugins.smartfile.fs.listFileTree(logDir, '*.log');
|
||||||
if (files.length === 0) {
|
const logFiles = files.filter((file: string) => !file.includes('/'));
|
||||||
|
|
||||||
|
if (logFiles.length === 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure previous directory exists
|
// Ensure previous directory exists
|
||||||
await plugins.smartfile.fs.ensureDir(previousDir);
|
await plugins.smartfile.fs.ensureDir(previousDir);
|
||||||
|
|
||||||
// Move each file to previous directory
|
// Move each log file to previous directory
|
||||||
for (const file of files) {
|
for (const file of logFiles) {
|
||||||
const filename = plugins.path.basename(file);
|
const filename = plugins.path.basename(file);
|
||||||
const sourcePath = plugins.path.join(logDir, filename);
|
const sourcePath = plugins.path.join(logDir, filename);
|
||||||
const destPath = plugins.path.join(previousDir, filename);
|
const destPath = plugins.path.join(previousDir, filename);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Read file content and write to new location
|
// Copy file to new location and remove original
|
||||||
const content = await plugins.smartfile.fs.toStringSync(sourcePath);
|
await plugins.smartfile.fs.copy(sourcePath, destPath);
|
||||||
await plugins.smartfile.fs.toFs(content, destPath);
|
|
||||||
// Remove original file
|
|
||||||
await plugins.smartfile.fs.remove(sourcePath);
|
await plugins.smartfile.fs.remove(sourcePath);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Silently continue if a file can't be moved
|
// Silently continue if a file can't be moved
|
||||||
|
@ -256,7 +256,11 @@ export class TsTestLogger {
|
|||||||
|
|
||||||
// Create error copy if there were failures
|
// Create error copy if there were failures
|
||||||
if (failed > 0) {
|
if (failed > 0) {
|
||||||
const errorLogPath = path.join(logDir, `00err_${logBasename}`);
|
const errorDir = path.join(logDir, '00err');
|
||||||
|
if (!fs.existsSync(errorDir)) {
|
||||||
|
fs.mkdirSync(errorDir, { recursive: true });
|
||||||
|
}
|
||||||
|
const errorLogPath = path.join(errorDir, logBasename);
|
||||||
fs.writeFileSync(errorLogPath, logContent);
|
fs.writeFileSync(errorLogPath, logContent);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -267,7 +271,11 @@ export class TsTestLogger {
|
|||||||
|
|
||||||
// Simple check if content differs
|
// Simple check if content differs
|
||||||
if (previousContent !== logContent) {
|
if (previousContent !== logContent) {
|
||||||
const diffLogPath = path.join(logDir, `00diff_${logBasename}`);
|
const diffDir = path.join(logDir, '00diff');
|
||||||
|
if (!fs.existsSync(diffDir)) {
|
||||||
|
fs.mkdirSync(diffDir, { recursive: true });
|
||||||
|
}
|
||||||
|
const diffLogPath = path.join(diffDir, logBasename);
|
||||||
const diffContent = this.createDiff(previousContent, logContent, logBasename);
|
const diffContent = this.createDiff(previousContent, logContent, logBasename);
|
||||||
fs.writeFileSync(diffLogPath, diffContent);
|
fs.writeFileSync(diffLogPath, diffContent);
|
||||||
}
|
}
|
||||||
@ -435,6 +443,20 @@ export class TsTestLogger {
|
|||||||
this.log(this.format(`\n${status}`, statusColor));
|
this.log(this.format(`\n${status}`, statusColor));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Warning display
|
||||||
|
warning(message: string) {
|
||||||
|
if (this.options.json) {
|
||||||
|
this.logJson({ event: 'warning', message });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.options.quiet) {
|
||||||
|
console.log(`WARNING: ${message}`);
|
||||||
|
} else {
|
||||||
|
this.log(this.format(` ⚠️ ${message}`, 'orange'));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Error display
|
// Error display
|
||||||
error(message: string, file?: string, stack?: string) {
|
error(message: string, file?: string, stack?: string) {
|
||||||
if (this.options.json) {
|
if (this.options.json) {
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
{
|
{
|
||||||
"order": 1
|
"order": 2
|
||||||
}
|
}
|
3
ts_tapbundle_node/tspublish.json
Normal file
3
ts_tapbundle_node/tspublish.json
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"order": 3
|
||||||
|
}
|
3
ts_tapbundle_protocol/tspublish.json
Normal file
3
ts_tapbundle_protocol/tspublish.json
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"order": 1
|
||||||
|
}
|
Reference in New Issue
Block a user