git.zone/tstest
2
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

feat(tstest): Enhance tstest with fluent API, suite grouping, tag filtering, fixture & snapshot testing, and parallel execution improvements

Browse Source
This commit is contained in:
Philipp Kunz
2025-05-16 00:21:32 +00:00
parent 1c5cf46ba9
commit 2b01d949f2
30 changed files with 1504 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@git.zone/tstest',
version: '1.6.0',
version: '1.7.0',
description: 'a test utility to run tests that match test/**/*.ts'
}

9
ts/index.ts
View File

@ -12,6 +12,7 @@ export const runCli = async () => {
const args = process.argv.slice(2);
const logOptions: LogOptions = {};
let testPath: string | null = null;
let tags: string[] = [];
// Parse options
for (let i = 0; i < args.length; i++) {
@ -36,6 +37,11 @@ export const runCli = async () => {
case '--logfile':
logOptions.logFile = true; // Set this as a flag, not a value
break;
case '--tags':
if (i + 1 < args.length) {
tags = args[++i].split(',');
}
break;
default:
if (!arg.startsWith('-')) {
testPath = arg;
@ -52,6 +58,7 @@ export const runCli = async () => {
console.error(' --no-color Disable colored output');
console.error(' --json Output results as JSON');
console.error(' --logfile Write logs to .nogit/testlogs/[testfile].log');
console.error(' --tags Run only tests with specified tags (comma-separated)');
process.exit(1);
}
@ -66,6 +73,6 @@ export const runCli = async () => {
executionMode = TestExecutionMode.DIRECTORY;
}
const tsTestInstance = new TsTest(process.cwd(), testPath, executionMode, logOptions);
const tsTestInstance = new TsTest(process.cwd(), testPath, executionMode, logOptions, tags);
await tsTestInstance.run();
};

136
ts/tstest.classes.tap.parser.ts
View File

@ -16,7 +16,7 @@ export class TapParser {
expectedTests: number;
receivedTests: number;
testStatusRegex = /(ok|not\sok)\s([0-9]+)\s-\s(.*)\s#\stime=(.*)ms$/;
testStatusRegex = /(ok|not\sok)\s([0-9]+)\s-\s(.*)(\s#\s(.*))?$/;
activeTapTestResult: TapTestResult;
collectingErrorDetails: boolean = false;
currentTestError: string[] = [];
@ -78,14 +78,33 @@ export class TapParser {
})();
const testSubject = regexResult[3];
const testDuration = parseInt(regexResult[4]);
// test for protocol error
if (testId !== this.activeTapTestResult.id) {
if (this.logger) {
this.logger.error('Something is strange! Test Ids are not equal!');
const testMetadata = regexResult[5]; // This will be either "time=XXXms" or "SKIP reason" or "TODO reason"
let testDuration = 0;
let isSkipped = false;
let isTodo = false;
if (testMetadata) {
const timeMatch = testMetadata.match(/time=(\d+)ms/);
const skipMatch = testMetadata.match(/SKIP\s*(.*)/);
const todoMatch = testMetadata.match(/TODO\s*(.*)/);
if (timeMatch) {
testDuration = parseInt(timeMatch[1]);
} else if (skipMatch) {
isSkipped = true;
} else if (todoMatch) {
isTodo = true;
}
}
// test for protocol error - disabled as it's not critical
// The test ID mismatch can occur when tests are filtered, skipped, or use todo
// if (testId !== this.activeTapTestResult.id) {
// if (this.logger) {
// this.logger.error('Something is strange! Test Ids are not equal!');
// }
// }
this.activeTapTestResult.setTestResult(testOk);
if (testOk) {
@ -107,27 +126,41 @@ export class TapParser {
this.activeTapTestResult.addLogLine(logLine);
}
// Check if we're collecting error details
if (this.collectingErrorDetails) {
// Check if this line is an error detail (starts with Error: or has stack trace characteristics)
if (logLine.trim().startsWith('Error:') || logLine.trim().match(/^\s*at\s/)) {
this.currentTestError.push(logLine);
} else if (this.currentTestError.length > 0) {
// End of error details, show the error
const errorMessage = this.currentTestError.join('\n');
// Check for snapshot communication
const snapshotMatch = logLine.match(/###SNAPSHOT###(.+)###SNAPSHOT###/);
if (snapshotMatch) {
const base64Data = snapshotMatch[1];
try {
const snapshotData = JSON.parse(Buffer.from(base64Data, 'base64').toString());
this.handleSnapshot(snapshotData);
} catch (error) {
if (this.logger) {
this.logger.testErrorDetails(errorMessage);
this.logger.testConsoleOutput(`Error parsing snapshot data: ${error.message}`);
}
this.collectingErrorDetails = false;
this.currentTestError = [];
}
}
// Don't output TAP error details as console output when we're collecting them
if (!this.collectingErrorDetails || (!logLine.trim().startsWith('Error:') && !logLine.trim().match(/^\s*at\s/))) {
if (this.logger) {
// This is console output from the test file, not TAP protocol
this.logger.testConsoleOutput(logLine);
} else {
// Check if we're collecting error details
if (this.collectingErrorDetails) {
// Check if this line is an error detail (starts with Error: or has stack trace characteristics)
if (logLine.trim().startsWith('Error:') || logLine.trim().match(/^\s*at\s/)) {
this.currentTestError.push(logLine);
} else if (this.currentTestError.length > 0) {
// End of error details, show the error
const errorMessage = this.currentTestError.join('\n');
if (this.logger) {
this.logger.testErrorDetails(errorMessage);
}
this.collectingErrorDetails = false;
this.currentTestError = [];
}
}
// Don't output TAP error details as console output when we're collecting them
if (!this.collectingErrorDetails || (!logLine.trim().startsWith('Error:') && !logLine.trim().match(/^\s*at\s/))) {
if (this.logger) {
// This is console output from the test file, not TAP protocol
this.logger.testConsoleOutput(logLine);
}
}
}
}
@ -205,6 +238,59 @@ export class TapParser {
public async handleTapLog(tapLog: string) {
this._processLog(tapLog);
}
/**
* Handle snapshot data from the test
*/
private async handleSnapshot(snapshotData: { path: string; content: string; action: string }) {
try {
const smartfile = await import('@push.rocks/smartfile');
if (snapshotData.action === 'compare') {
// Try to read existing snapshot
try {
const existingSnapshot = await smartfile.fs.toStringSync(snapshotData.path);
if (existingSnapshot !== snapshotData.content) {
// Snapshot mismatch
if (this.logger) {
this.logger.testConsoleOutput(`Snapshot mismatch: ${snapshotData.path}`);
this.logger.testConsoleOutput(`Expected:\n${existingSnapshot}`);
this.logger.testConsoleOutput(`Received:\n${snapshotData.content}`);
}
// TODO: Communicate failure back to the test
} else {
if (this.logger) {
this.logger.testConsoleOutput(`Snapshot matched: ${snapshotData.path}`);
}
}
} catch (error: any) {
if (error.code === 'ENOENT') {
// Snapshot doesn't exist, create it
const dirPath = snapshotData.path.substring(0, snapshotData.path.lastIndexOf('/'));
await smartfile.fs.ensureDir(dirPath);
await smartfile.memory.toFs(snapshotData.content, snapshotData.path);
if (this.logger) {
this.logger.testConsoleOutput(`Snapshot created: ${snapshotData.path}`);
}
} else {
throw error;
}
}
} else if (snapshotData.action === 'update') {
// Update snapshot
const dirPath = snapshotData.path.substring(0, snapshotData.path.lastIndexOf('/'));
await smartfile.fs.ensureDir(dirPath);
await smartfile.memory.toFs(snapshotData.content, snapshotData.path);
if (this.logger) {
this.logger.testConsoleOutput(`Snapshot updated: ${snapshotData.path}`);
}
}
} catch (error: any) {
if (this.logger) {
this.logger.testConsoleOutput(`Error handling snapshot: ${error.message}`);
}
}
}
public async evaluateFinalResult() {
this.receivedTests = this.testStore.length;

39
ts/tstest.classes.testdirectory.ts
View File

@ -99,4 +99,43 @@ export class TestDirectory {
}
return testFilePaths;
}
/**
* Get test files organized by parallel execution groups
* @returns An object with grouped tests
*/
async getTestFileGroups(): Promise<{
serial: string[];
parallelGroups: { [groupName: string]: string[] };
}> {
await this._init();
const result = {
serial: [] as string[],
parallelGroups: {} as { [groupName: string]: string[] }
};
for (const testFile of this.testfileArray) {
const filePath = testFile.path;
const fileName = plugins.path.basename(filePath);
// Check if file has parallel group pattern
const parallelMatch = fileName.match(/\.para__(\d+)\./);
if (parallelMatch) {
const groupNumber = parallelMatch[1];
const groupName = `para__${groupNumber}`;
if (!result.parallelGroups[groupName]) {
result.parallelGroups[groupName] = [];
}
result.parallelGroups[groupName].push(filePath);
} else {
// File runs serially
result.serial.push(filePath);
}
}
return result;
}
}

88
ts/tstest.classes.tstest.ts
View File

@ -15,6 +15,7 @@ export class TsTest {
public testDir: TestDirectory;
public executionMode: TestExecutionMode;
public logger: TsTestLogger;
public filterTags: string[];
public smartshellInstance = new plugins.smartshell.Smartshell({
executor: 'bash',
@ -25,53 +26,81 @@ export class TsTest {
public tsbundleInstance = new plugins.tsbundle.TsBundle();
constructor(cwdArg: string, testPathArg: string, executionModeArg: TestExecutionMode, logOptions: LogOptions = {}) {
constructor(cwdArg: string, testPathArg: string, executionModeArg: TestExecutionMode, logOptions: LogOptions = {}, tags: string[] = []) {
this.executionMode = executionModeArg;
this.testDir = new TestDirectory(cwdArg, testPathArg, executionModeArg);
this.logger = new TsTestLogger(logOptions);
this.filterTags = tags;
}
async run() {
const fileNamesToRun: string[] = await this.testDir.getTestFilePathArray();
const testGroups = await this.testDir.getTestFileGroups();
const allFiles = [...testGroups.serial, ...Object.values(testGroups.parallelGroups).flat()];
// Log test discovery
this.logger.testDiscovery(
fileNamesToRun.length,
allFiles.length,
this.testDir.testPath,
this.executionMode
);
const tapCombinator = new TapCombinator(this.logger); // lets create the TapCombinator
let fileIndex = 0;
for (const fileNameArg of fileNamesToRun) {
// Execute serial tests first
for (const fileNameArg of testGroups.serial) {
fileIndex++;
switch (true) {
case process.env.CI && fileNameArg.includes('.nonci.'):
this.logger.tapOutput(`Skipping ${fileNameArg} - marked as non-CI`);
break;
case fileNameArg.endsWith('.browser.ts') || fileNameArg.endsWith('.browser.nonci.ts'):
const tapParserBrowser = await this.runInChrome(fileNameArg, fileIndex, fileNamesToRun.length);
tapCombinator.addTapParser(tapParserBrowser);
break;
case fileNameArg.endsWith('.both.ts') || fileNameArg.endsWith('.both.nonci.ts'):
this.logger.sectionStart('Part 1: Chrome');
const tapParserBothBrowser = await this.runInChrome(fileNameArg, fileIndex, fileNamesToRun.length);
tapCombinator.addTapParser(tapParserBothBrowser);
this.logger.sectionEnd();
this.logger.sectionStart('Part 2: Node');
const tapParserBothNode = await this.runInNode(fileNameArg, fileIndex, fileNamesToRun.length);
tapCombinator.addTapParser(tapParserBothNode);
this.logger.sectionEnd();
break;
default:
const tapParserNode = await this.runInNode(fileNameArg, fileIndex, fileNamesToRun.length);
tapCombinator.addTapParser(tapParserNode);
break;
await this.runSingleTest(fileNameArg, fileIndex, allFiles.length, tapCombinator);
}
// Execute parallel groups sequentially
const groupNames = Object.keys(testGroups.parallelGroups).sort();
for (const groupName of groupNames) {
const groupFiles = testGroups.parallelGroups[groupName];
if (groupFiles.length > 0) {
this.logger.sectionStart(`Parallel Group: ${groupName}`);
// Run all tests in this group in parallel
const parallelPromises = groupFiles.map(async (fileNameArg) => {
fileIndex++;
return this.runSingleTest(fileNameArg, fileIndex, allFiles.length, tapCombinator);
});
await Promise.all(parallelPromises);
this.logger.sectionEnd();
}
}
tapCombinator.evaluate();
}
private async runSingleTest(fileNameArg: string, fileIndex: number, totalFiles: number, tapCombinator: TapCombinator) {
switch (true) {
case process.env.CI && fileNameArg.includes('.nonci.'):
this.logger.tapOutput(`Skipping ${fileNameArg} - marked as non-CI`);
break;
case fileNameArg.endsWith('.browser.ts') || fileNameArg.endsWith('.browser.nonci.ts'):
const tapParserBrowser = await this.runInChrome(fileNameArg, fileIndex, totalFiles);
tapCombinator.addTapParser(tapParserBrowser);
break;
case fileNameArg.endsWith('.both.ts') || fileNameArg.endsWith('.both.nonci.ts'):
this.logger.sectionStart('Part 1: Chrome');
const tapParserBothBrowser = await this.runInChrome(fileNameArg, fileIndex, totalFiles);
tapCombinator.addTapParser(tapParserBothBrowser);
this.logger.sectionEnd();
this.logger.sectionStart('Part 2: Node');
const tapParserBothNode = await this.runInNode(fileNameArg, fileIndex, totalFiles);
tapCombinator.addTapParser(tapParserBothNode);
this.logger.sectionEnd();
break;
default:
const tapParserNode = await this.runInNode(fileNameArg, fileIndex, totalFiles);
tapCombinator.addTapParser(tapParserNode);
break;
}
}
public async runInNode(fileNameArg: string, index: number, total: number): Promise<TapParser> {
this.logger.testFileStart(fileNameArg, 'node.js', index, total);
@ -82,6 +111,11 @@ export class TsTest {
if (process.argv.includes('--web')) {
tsrunOptions += ' --web';
}
// Set filter tags as environment variable
if (this.filterTags.length > 0) {
process.env.TSTEST_FILTER_TAGS = this.filterTags.join(',');
}
const execResultStreaming = await this.smartshellInstance.execStreamingSilent(
`tsrun ${fileNameArg}${tsrunOptions}`