feat(context): Introduce smart context system: analyzer, lazy loader, cache and README/docs improvements
This commit is contained in:
242
test/test.lazyfileloader.node.ts
Normal file
242
test/test.lazyfileloader.node.ts
Normal file
@@ -0,0 +1,242 @@
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as path from 'path';
|
||||
import { LazyFileLoader } from '../ts/context/lazy-file-loader.js';
|
||||
import type { IFileMetadata } from '../ts/context/types.js';
|
||||
|
||||
const testProjectRoot = process.cwd();
|
||||
|
||||
tap.test('LazyFileLoader should create instance with project root', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
expect(loader).toBeInstanceOf(LazyFileLoader);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.getMetadata should return file metadata without loading contents', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
const packageJsonPath = path.join(testProjectRoot, 'package.json');
|
||||
|
||||
const metadata = await loader.getMetadata(packageJsonPath);
|
||||
|
||||
expect(metadata.path).toEqual(packageJsonPath);
|
||||
expect(metadata.relativePath).toEqual('package.json');
|
||||
expect(metadata.size).toBeGreaterThan(0);
|
||||
expect(metadata.mtime).toBeGreaterThan(0);
|
||||
expect(metadata.estimatedTokens).toBeGreaterThan(0);
|
||||
// Rough estimate: size / 4
|
||||
expect(metadata.estimatedTokens).toBeCloseTo(metadata.size / 4, 10);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.getMetadata should cache metadata for same file', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
const packageJsonPath = path.join(testProjectRoot, 'package.json');
|
||||
|
||||
const metadata1 = await loader.getMetadata(packageJsonPath);
|
||||
const metadata2 = await loader.getMetadata(packageJsonPath);
|
||||
|
||||
// Should return identical metadata from cache
|
||||
expect(metadata1.mtime).toEqual(metadata2.mtime);
|
||||
expect(metadata1.size).toEqual(metadata2.size);
|
||||
expect(metadata1.estimatedTokens).toEqual(metadata2.estimatedTokens);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.scanFiles should scan TypeScript files', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
|
||||
const metadata = await loader.scanFiles(['ts/context/types.ts']);
|
||||
|
||||
expect(metadata.length).toBeGreaterThan(0);
|
||||
const typesFile = metadata.find(m => m.relativePath.includes('types.ts'));
|
||||
expect(typesFile).toBeDefined();
|
||||
expect(typesFile!.size).toBeGreaterThan(0);
|
||||
expect(typesFile!.estimatedTokens).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.scanFiles should handle multiple globs', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
|
||||
const metadata = await loader.scanFiles([
|
||||
'package.json',
|
||||
'readme.md'
|
||||
]);
|
||||
|
||||
expect(metadata.length).toBeGreaterThanOrEqual(2);
|
||||
const hasPackageJson = metadata.some(m => m.relativePath === 'package.json');
|
||||
const hasReadme = metadata.some(m => m.relativePath.toLowerCase() === 'readme.md');
|
||||
expect(hasPackageJson).toBe(true);
|
||||
expect(hasReadme).toBe(true);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.loadFile should load file with actual token count', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
const packageJsonPath = path.join(testProjectRoot, 'package.json');
|
||||
|
||||
const tokenizer = (content: string) => Math.ceil(content.length / 4);
|
||||
const fileInfo = await loader.loadFile(packageJsonPath, tokenizer);
|
||||
|
||||
expect(fileInfo.path).toEqual(packageJsonPath);
|
||||
expect(fileInfo.contents).toBeDefined();
|
||||
expect(fileInfo.contents.length).toBeGreaterThan(0);
|
||||
expect(fileInfo.tokenCount).toBeGreaterThan(0);
|
||||
expect(fileInfo.relativePath).toEqual('package.json');
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.loadFiles should load multiple files in parallel', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
|
||||
const metadata: IFileMetadata[] = [
|
||||
{
|
||||
path: path.join(testProjectRoot, 'package.json'),
|
||||
relativePath: 'package.json',
|
||||
size: 100,
|
||||
mtime: Date.now(),
|
||||
estimatedTokens: 25
|
||||
},
|
||||
{
|
||||
path: path.join(testProjectRoot, 'readme.md'),
|
||||
relativePath: 'readme.md',
|
||||
size: 200,
|
||||
mtime: Date.now(),
|
||||
estimatedTokens: 50
|
||||
}
|
||||
];
|
||||
|
||||
const tokenizer = (content: string) => Math.ceil(content.length / 4);
|
||||
const startTime = Date.now();
|
||||
const files = await loader.loadFiles(metadata, tokenizer);
|
||||
const endTime = Date.now();
|
||||
|
||||
expect(files.length).toEqual(2);
|
||||
expect(files[0].contents).toBeDefined();
|
||||
expect(files[1].contents).toBeDefined();
|
||||
|
||||
// Should be fast (parallel loading)
|
||||
expect(endTime - startTime).toBeLessThan(5000); // 5 seconds max
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.updateImportanceScores should update cached metadata', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
const packageJsonPath = path.join(testProjectRoot, 'package.json');
|
||||
|
||||
// Get initial metadata
|
||||
await loader.getMetadata(packageJsonPath);
|
||||
|
||||
// Update importance scores
|
||||
const scores = new Map<string, number>();
|
||||
scores.set(packageJsonPath, 0.95);
|
||||
loader.updateImportanceScores(scores);
|
||||
|
||||
// Check cached metadata has updated score
|
||||
const cached = loader.getCachedMetadata();
|
||||
const packageJsonMeta = cached.find(m => m.path === packageJsonPath);
|
||||
|
||||
expect(packageJsonMeta).toBeDefined();
|
||||
expect(packageJsonMeta!.importanceScore).toEqual(0.95);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.getTotalEstimatedTokens should sum all cached metadata tokens', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
|
||||
// Scan some files
|
||||
await loader.scanFiles(['package.json', 'readme.md']);
|
||||
|
||||
const totalTokens = loader.getTotalEstimatedTokens();
|
||||
|
||||
expect(totalTokens).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.clearCache should clear metadata cache', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
|
||||
// Scan files to populate cache
|
||||
await loader.scanFiles(['package.json']);
|
||||
expect(loader.getCachedMetadata().length).toBeGreaterThan(0);
|
||||
|
||||
// Clear cache
|
||||
loader.clearCache();
|
||||
|
||||
expect(loader.getCachedMetadata().length).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.getCachedMetadata should return all cached entries', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
|
||||
// Scan files
|
||||
await loader.scanFiles(['package.json', 'readme.md']);
|
||||
|
||||
const cached = loader.getCachedMetadata();
|
||||
|
||||
expect(cached.length).toBeGreaterThanOrEqual(2);
|
||||
expect(cached.every(m => m.path && m.size && m.estimatedTokens)).toBe(true);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader should handle non-existent files gracefully', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
const nonExistentPath = path.join(testProjectRoot, 'this-file-does-not-exist.ts');
|
||||
|
||||
try {
|
||||
await loader.getMetadata(nonExistentPath);
|
||||
expect(false).toBe(true); // Should not reach here
|
||||
} catch (error) {
|
||||
expect(error).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader.loadFiles should filter out failed file loads', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
|
||||
const metadata: IFileMetadata[] = [
|
||||
{
|
||||
path: path.join(testProjectRoot, 'package.json'),
|
||||
relativePath: 'package.json',
|
||||
size: 100,
|
||||
mtime: Date.now(),
|
||||
estimatedTokens: 25
|
||||
},
|
||||
{
|
||||
path: path.join(testProjectRoot, 'non-existent-file.txt'),
|
||||
relativePath: 'non-existent-file.txt',
|
||||
size: 100,
|
||||
mtime: Date.now(),
|
||||
estimatedTokens: 25
|
||||
}
|
||||
];
|
||||
|
||||
const tokenizer = (content: string) => Math.ceil(content.length / 4);
|
||||
const files = await loader.loadFiles(metadata, tokenizer);
|
||||
|
||||
// Should only include the successfully loaded file
|
||||
expect(files.length).toEqual(1);
|
||||
expect(files[0].relativePath).toEqual('package.json');
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader should handle glob patterns for TypeScript source files', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
|
||||
const metadata = await loader.scanFiles(['ts/context/*.ts']);
|
||||
|
||||
expect(metadata.length).toBeGreaterThan(0);
|
||||
|
||||
// Should find multiple context files
|
||||
const hasEnhancedContext = metadata.some(m => m.relativePath.includes('enhanced-context.ts'));
|
||||
const hasTypes = metadata.some(m => m.relativePath.includes('types.ts'));
|
||||
|
||||
expect(hasEnhancedContext).toBe(true);
|
||||
expect(hasTypes).toBe(true);
|
||||
});
|
||||
|
||||
tap.test('LazyFileLoader should estimate tokens reasonably accurately', async () => {
|
||||
const loader = new LazyFileLoader(testProjectRoot);
|
||||
const packageJsonPath = path.join(testProjectRoot, 'package.json');
|
||||
|
||||
const metadata = await loader.getMetadata(packageJsonPath);
|
||||
const tokenizer = (content: string) => Math.ceil(content.length / 4);
|
||||
const fileInfo = await loader.loadFile(packageJsonPath, tokenizer);
|
||||
|
||||
// Estimated tokens should be close to actual (within reasonable range)
|
||||
const difference = Math.abs(metadata.estimatedTokens - fileInfo.tokenCount);
|
||||
const percentDiff = (difference / fileInfo.tokenCount) * 100;
|
||||
|
||||
// Should be within 20% accuracy (since it's just an estimate)
|
||||
expect(percentDiff).toBeLessThan(20);
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
Reference in New Issue
Block a user