549 lines
18 KiB
TypeScript
549 lines
18 KiB
TypeScript
import * as plugins from './npmextra.plugins.js';
|
|
import { KeyValueStore } from './npmextra.classes.keyvaluestore.js';
|
|
|
|
// ============================================================================
|
|
// Singleton Qenv Provider
|
|
// ============================================================================
|
|
let sharedQenv: plugins.qenv.Qenv | undefined;
|
|
|
|
function getQenv(): plugins.qenv.Qenv {
|
|
if (!sharedQenv) {
|
|
sharedQenv = new plugins.qenv.Qenv(
|
|
process.cwd(),
|
|
plugins.path.join(process.cwd(), '.nogit')
|
|
);
|
|
}
|
|
return sharedQenv;
|
|
}
|
|
|
|
// ============================================================================
|
|
// Security - Redaction for sensitive data
|
|
// ============================================================================
|
|
/**
|
|
* Redacts sensitive values in logs to prevent exposure of secrets
|
|
*/
|
|
function redactSensitiveValue(key: string, value: unknown): string {
|
|
// List of patterns that indicate sensitive data
|
|
const sensitivePatterns = [
|
|
/secret/i, /token/i, /key/i, /password/i, /pass/i,
|
|
/api/i, /credential/i, /auth/i, /private/i, /jwt/i,
|
|
/cert/i, /signature/i, /bearer/i
|
|
];
|
|
|
|
// Check if key contains sensitive pattern
|
|
const isSensitive = sensitivePatterns.some(pattern => pattern.test(key));
|
|
|
|
if (isSensitive) {
|
|
if (typeof value === 'string') {
|
|
// Show first 3 chars and length for debugging
|
|
return value.length > 3
|
|
? `${value.substring(0, 3)}...[${value.length} chars]`
|
|
: '[redacted]';
|
|
}
|
|
return '[redacted]';
|
|
}
|
|
|
|
// Check if value looks like a JWT token or base64 secret
|
|
if (typeof value === 'string') {
|
|
// JWT tokens start with eyJ
|
|
if (value.startsWith('eyJ')) {
|
|
return `eyJ...[${value.length} chars]`;
|
|
}
|
|
// Very long strings might be encoded secrets
|
|
if (value.length > 100) {
|
|
return `${value.substring(0, 50)}...[${value.length} chars total]`;
|
|
}
|
|
}
|
|
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Type Converters - Centralized conversion logic
|
|
// ============================================================================
|
|
function toBoolean(value: unknown): boolean {
|
|
// If already boolean, return as-is
|
|
if (typeof value === 'boolean') {
|
|
console.log(` 🔹 toBoolean: value is already boolean: ${value}`);
|
|
return value;
|
|
}
|
|
|
|
// Handle null/undefined
|
|
if (value == null) {
|
|
console.log(` 🔹 toBoolean: value is null/undefined, returning false`);
|
|
return false;
|
|
}
|
|
|
|
// Handle string representations
|
|
const s = String(value).toLowerCase().trim();
|
|
|
|
// True values: "true", "1", "yes", "y", "on"
|
|
if (['true', '1', 'yes', 'y', 'on'].includes(s)) {
|
|
console.log(` 🔹 toBoolean: converting "${value}" to true`);
|
|
return true;
|
|
}
|
|
|
|
// False values: "false", "0", "no", "n", "off"
|
|
if (['false', '0', 'no', 'n', 'off'].includes(s)) {
|
|
console.log(` 🔹 toBoolean: converting "${value}" to false`);
|
|
return false;
|
|
}
|
|
|
|
// Default: non-empty string = true, empty = false
|
|
const result = s.length > 0;
|
|
console.log(` 🔹 toBoolean: defaulting "${value}" to ${result}`);
|
|
return result;
|
|
}
|
|
|
|
function toJson<T = any>(value: unknown): T | undefined {
|
|
if (value == null) return undefined;
|
|
if (typeof value === 'string') {
|
|
try {
|
|
return JSON.parse(value);
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
return value as T;
|
|
}
|
|
|
|
function fromBase64(value: unknown): string | undefined {
|
|
if (value == null) return undefined;
|
|
try {
|
|
return Buffer.from(String(value), 'base64').toString('utf8');
|
|
} catch {
|
|
return String(value);
|
|
}
|
|
}
|
|
|
|
function toNumber(value: unknown): number | undefined {
|
|
if (value == null) return undefined;
|
|
const num = Number(value);
|
|
return Number.isNaN(num) ? undefined : num;
|
|
}
|
|
|
|
function toString(value: unknown): string | undefined {
|
|
if (value == null) return undefined;
|
|
return String(value);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Declarative Pipeline Architecture
|
|
// ============================================================================
|
|
type Transform = 'boolean' | 'json' | 'base64' | 'number';
|
|
|
|
type MappingSpec = {
|
|
source:
|
|
| { type: 'env'; key: string }
|
|
| { type: 'hard'; value: string };
|
|
transforms: Transform[];
|
|
};
|
|
|
|
// Transform registry for extensibility
|
|
const transformRegistry: Record<string, (v: unknown) => unknown> = {
|
|
boolean: toBoolean,
|
|
json: toJson,
|
|
base64: fromBase64,
|
|
number: toNumber,
|
|
};
|
|
|
|
/**
|
|
* Parse a mapping string into a declarative spec
|
|
*/
|
|
function parseMappingSpec(input: string): MappingSpec {
|
|
const transforms: Transform[] = [];
|
|
let remaining = input;
|
|
|
|
// Check for hardcoded prefixes with type conversion
|
|
if (remaining.startsWith('hard_boolean:')) {
|
|
return {
|
|
source: { type: 'hard', value: remaining.slice(13) },
|
|
transforms: ['boolean']
|
|
};
|
|
}
|
|
|
|
if (remaining.startsWith('hard_json:')) {
|
|
return {
|
|
source: { type: 'hard', value: remaining.slice(10) },
|
|
transforms: ['json']
|
|
};
|
|
}
|
|
|
|
if (remaining.startsWith('hard_base64:')) {
|
|
return {
|
|
source: { type: 'hard', value: remaining.slice(12) },
|
|
transforms: ['base64']
|
|
};
|
|
}
|
|
|
|
// Check for generic hard: prefix
|
|
if (remaining.startsWith('hard:')) {
|
|
remaining = remaining.slice(5);
|
|
// Check for legacy suffixes on hardcoded values
|
|
if (remaining.endsWith('_JSON')) {
|
|
transforms.push('json');
|
|
remaining = remaining.slice(0, -5);
|
|
} else if (remaining.endsWith('_BASE64')) {
|
|
transforms.push('base64');
|
|
remaining = remaining.slice(0, -7);
|
|
}
|
|
return {
|
|
source: { type: 'hard', value: remaining },
|
|
transforms
|
|
};
|
|
}
|
|
|
|
// Check for env var prefixes
|
|
if (remaining.startsWith('boolean:')) {
|
|
transforms.push('boolean');
|
|
remaining = remaining.slice(8);
|
|
} else if (remaining.startsWith('json:')) {
|
|
transforms.push('json');
|
|
remaining = remaining.slice(5);
|
|
} else if (remaining.startsWith('base64:')) {
|
|
transforms.push('base64');
|
|
remaining = remaining.slice(7);
|
|
}
|
|
|
|
// Check for legacy suffixes on env vars
|
|
if (remaining.endsWith('_JSON')) {
|
|
transforms.push('json');
|
|
remaining = remaining.slice(0, -5);
|
|
} else if (remaining.endsWith('_BASE64')) {
|
|
transforms.push('base64');
|
|
remaining = remaining.slice(0, -7);
|
|
}
|
|
|
|
return {
|
|
source: { type: 'env', key: remaining },
|
|
transforms
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Resolve the source value (env var or hardcoded)
|
|
*/
|
|
async function resolveSource(source: MappingSpec['source']): Promise<unknown> {
|
|
if (source.type === 'hard') {
|
|
return source.value;
|
|
}
|
|
// source.type === 'env'
|
|
// Workaround for Qenv bug where empty strings are treated as undefined
|
|
// Check process.env directly first to preserve empty strings
|
|
if (Object.prototype.hasOwnProperty.call(process.env, source.key)) {
|
|
return process.env[source.key];
|
|
}
|
|
// Fall back to Qenv for other sources (env.json, docker secrets, etc.)
|
|
return await getQenv().getEnvVarOnDemand(source.key);
|
|
}
|
|
|
|
/**
|
|
* Apply transformations in sequence
|
|
*/
|
|
function applyTransforms(value: unknown, transforms: Transform[]): unknown {
|
|
return transforms.reduce((acc, transform) => {
|
|
const fn = transformRegistry[transform];
|
|
return fn ? fn(acc) : acc;
|
|
}, value);
|
|
}
|
|
|
|
/**
|
|
* Process a mapping value through the complete pipeline
|
|
*/
|
|
async function processMappingValue(mappingString: string): Promise<unknown> {
|
|
const spec = parseMappingSpec(mappingString);
|
|
const keyName = spec.source.type === 'env' ? spec.source.key : 'hardcoded';
|
|
|
|
console.log(` 🔍 Processing mapping: "${mappingString}"`);
|
|
console.log(` Source: ${spec.source.type === 'env' ? `env:${spec.source.key}` : `hard:${spec.source.value}`}`);
|
|
console.log(` Transforms: ${spec.transforms.length > 0 ? spec.transforms.join(', ') : 'none'}`);
|
|
|
|
const rawValue = await resolveSource(spec.source);
|
|
console.log(` Raw value: ${redactSensitiveValue(keyName, rawValue)} (type: ${typeof rawValue})`);
|
|
|
|
if (rawValue === undefined || rawValue === null) {
|
|
console.log(` ⚠️ Raw value is undefined/null, returning undefined`);
|
|
return undefined;
|
|
}
|
|
|
|
const result = applyTransforms(rawValue, spec.transforms);
|
|
console.log(` Final value: ${redactSensitiveValue(keyName, result)} (type: ${typeof result})`);
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Recursively evaluate mapping values (strings or nested objects)
|
|
*/
|
|
async function evaluateMappingValue(mappingValue: any): Promise<any> {
|
|
// Handle null explicitly - it should return null, not be treated as object
|
|
if (mappingValue === null) {
|
|
console.log(` 📌 Value is null, returning null`);
|
|
return null;
|
|
}
|
|
|
|
// Handle strings (mapping specs)
|
|
if (typeof mappingValue === 'string') {
|
|
return processMappingValue(mappingValue);
|
|
}
|
|
|
|
// Handle objects (but not arrays or null)
|
|
if (mappingValue && typeof mappingValue === 'object' && !Array.isArray(mappingValue)) {
|
|
console.log(` 📂 Processing nested object with ${Object.keys(mappingValue).length} keys`);
|
|
const result: any = {};
|
|
for (const [key, value] of Object.entries(mappingValue)) {
|
|
console.log(` → Processing nested key "${key}"`);
|
|
const evaluated = await evaluateMappingValue(value);
|
|
// Important: Don't filter out false or other falsy values!
|
|
// Only skip if explicitly undefined
|
|
if (evaluated !== undefined) {
|
|
result[key] = evaluated;
|
|
console.log(` ✓ Nested key "${key}" = ${redactSensitiveValue(key, evaluated)} (type: ${typeof evaluated})`);
|
|
} else {
|
|
console.log(` ⚠️ Nested key "${key}" evaluated to undefined, skipping`);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// For any other type (numbers, booleans, etc.), return as-is
|
|
// Note: We don't have key context here, so we'll just indicate the type
|
|
console.log(` 📎 Returning value as-is: [value] (type: ${typeof mappingValue})`);
|
|
return mappingValue;
|
|
}
|
|
|
|
// ============================================================================
|
|
// AppData Interface and Class
|
|
// ============================================================================
|
|
export interface IAppDataOptions<T = any> {
|
|
dirPath?: string;
|
|
requiredKeys?: Array<keyof T>;
|
|
|
|
/**
|
|
* Whether keys should be persisted on disk or not
|
|
*/
|
|
ephemeral?: boolean;
|
|
|
|
/**
|
|
* @deprecated Use 'ephemeral' instead
|
|
*/
|
|
ephermal?: boolean;
|
|
|
|
/**
|
|
* kvStoreKey: 'MY_ENV_VAR'
|
|
*/
|
|
envMapping?: plugins.tsclass.typeFest.PartialDeep<T>;
|
|
overwriteObject?: plugins.tsclass.typeFest.PartialDeep<T>;
|
|
}
|
|
|
|
export class AppData<T = any> {
|
|
/**
|
|
* creates appdata. If no pathArg is given, data will be stored here:
|
|
* ${PWD}/.nogit/appdata
|
|
* @param pathArg
|
|
* @returns
|
|
*/
|
|
public static async createAndInit<T = any>(
|
|
optionsArg: IAppDataOptions<T> = {},
|
|
): Promise<AppData<T>> {
|
|
const appData = new AppData<T>(optionsArg);
|
|
await appData.readyDeferred.promise;
|
|
return appData;
|
|
}
|
|
|
|
/**
|
|
* Static helper to get an environment variable as a boolean
|
|
* @param envVarName The name of the environment variable
|
|
* @returns boolean value (true if env var is "true", false otherwise)
|
|
*/
|
|
public static async valueAsBoolean(envVarName: string): Promise<boolean> {
|
|
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
|
return toBoolean(value);
|
|
}
|
|
|
|
/**
|
|
* Static helper to get an environment variable as parsed JSON
|
|
* @param envVarName The name of the environment variable
|
|
* @returns Parsed JSON object/array
|
|
*/
|
|
public static async valueAsJson<R = any>(envVarName: string): Promise<R | undefined> {
|
|
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
|
return toJson<R>(value);
|
|
}
|
|
|
|
/**
|
|
* Static helper to get an environment variable as base64 decoded string
|
|
* @param envVarName The name of the environment variable
|
|
* @returns Decoded string
|
|
*/
|
|
public static async valueAsBase64(envVarName: string): Promise<string | undefined> {
|
|
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
|
return fromBase64(value);
|
|
}
|
|
|
|
/**
|
|
* Static helper to get an environment variable as a string
|
|
* @param envVarName The name of the environment variable
|
|
* @returns String value
|
|
*/
|
|
public static async valueAsString(envVarName: string): Promise<string | undefined> {
|
|
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
|
return toString(value);
|
|
}
|
|
|
|
/**
|
|
* Static helper to get an environment variable as a number
|
|
* @param envVarName The name of the environment variable
|
|
* @returns Number value
|
|
*/
|
|
public static async valueAsNumber(envVarName: string): Promise<number | undefined> {
|
|
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
|
return toNumber(value);
|
|
}
|
|
|
|
// instance
|
|
public readyDeferred = plugins.smartpromise.defer<void>();
|
|
public options: IAppDataOptions<T>;
|
|
private kvStore: KeyValueStore<T>;
|
|
|
|
constructor(optionsArg: IAppDataOptions<T> = {}) {
|
|
this.options = optionsArg;
|
|
this.init();
|
|
}
|
|
|
|
/**
|
|
* inits app data
|
|
*/
|
|
private async init() {
|
|
console.log('🚀 Initializing AppData...');
|
|
|
|
// Handle backward compatibility for typo
|
|
const isEphemeral = this.options.ephemeral ?? this.options.ephermal ?? false;
|
|
if (this.options.ephermal && !this.options.ephemeral) {
|
|
console.warn('⚠️ Option "ephermal" is deprecated, use "ephemeral" instead.');
|
|
}
|
|
|
|
if (this.options.dirPath) {
|
|
console.log(` 📁 Using custom directory: ${this.options.dirPath}`);
|
|
} else if (isEphemeral) {
|
|
console.log(` 💨 Using ephemeral storage (in-memory only)`);
|
|
} else {
|
|
const appDataDir = '/app/data';
|
|
const dataDir = '/data';
|
|
const nogitAppData = '.nogit/appdata';
|
|
const appDataExists = plugins.smartfile.fs.isDirectory(appDataDir);
|
|
const dataExists = plugins.smartfile.fs.isDirectory(dataDir);
|
|
if (appDataExists) {
|
|
this.options.dirPath = appDataDir;
|
|
console.log(` 📁 Auto-selected container directory: ${appDataDir}`);
|
|
} else if (dataExists) {
|
|
this.options.dirPath = dataDir;
|
|
console.log(` 📁 Auto-selected data directory: ${dataDir}`);
|
|
} else {
|
|
await plugins.smartfile.fs.ensureDir(nogitAppData);
|
|
this.options.dirPath = nogitAppData;
|
|
console.log(` 📁 Auto-selected local directory: ${nogitAppData}`);
|
|
}
|
|
}
|
|
|
|
this.kvStore = new KeyValueStore<T>({
|
|
typeArg: isEphemeral ? 'ephemeral' : 'custom',
|
|
identityArg: 'appkv',
|
|
customPath: this.options.dirPath,
|
|
mandatoryKeys: this.options.requiredKeys as Array<keyof T>,
|
|
});
|
|
|
|
if (this.options.envMapping) {
|
|
console.log(`📦 Processing envMapping for AppData...`);
|
|
const totalKeys = Object.keys(this.options.envMapping).length;
|
|
let processedCount = 0;
|
|
|
|
// Process each top-level key in envMapping
|
|
for (const key in this.options.envMapping) {
|
|
try {
|
|
const mappingSpec = this.options.envMapping[key];
|
|
const specType = mappingSpec === null ? 'null' :
|
|
typeof mappingSpec === 'string' ? mappingSpec :
|
|
typeof mappingSpec === 'object' ? 'nested object' :
|
|
typeof mappingSpec;
|
|
console.log(` → Processing key "${key}" with spec: ${specType}`);
|
|
|
|
const evaluated = await evaluateMappingValue(mappingSpec);
|
|
// Important: Don't skip false, 0, empty string, or null values!
|
|
// Only skip if explicitly undefined
|
|
if (evaluated !== undefined) {
|
|
await this.kvStore.writeKey(key as keyof T, evaluated);
|
|
processedCount++;
|
|
const valueType = evaluated === null ? 'null' :
|
|
Array.isArray(evaluated) ? 'array' :
|
|
typeof evaluated;
|
|
const valuePreview = evaluated === null ? 'null' :
|
|
typeof evaluated === 'object' ?
|
|
(Array.isArray(evaluated) ? `[${evaluated.length} items]` : `{${Object.keys(evaluated).length} keys}`) :
|
|
redactSensitiveValue(key, evaluated);
|
|
console.log(` ✅ Successfully processed key "${key}" = ${valuePreview} (type: ${valueType})`);
|
|
} else {
|
|
console.log(` ⚠️ Key "${key}" evaluated to undefined, skipping`);
|
|
}
|
|
} catch (err) {
|
|
console.error(` ❌ Failed to evaluate envMapping for key "${key}":`, err);
|
|
}
|
|
}
|
|
|
|
console.log(`📊 EnvMapping complete: ${processedCount}/${totalKeys} keys successfully processed`);
|
|
}
|
|
|
|
// Apply overwrite object after env mapping
|
|
if (this.options.overwriteObject) {
|
|
const overwriteKeys = Object.keys(this.options.overwriteObject);
|
|
console.log(`🔄 Applying overwriteObject with ${overwriteKeys.length} key(s)...`);
|
|
|
|
for (const key of overwriteKeys) {
|
|
const value = this.options.overwriteObject[key];
|
|
const valueType = Array.isArray(value) ? 'array' : typeof value;
|
|
console.log(` 🔧 Overwriting key "${key}" with ${valueType} value`);
|
|
|
|
await this.kvStore.writeKey(
|
|
key as keyof T,
|
|
value,
|
|
);
|
|
}
|
|
|
|
console.log(`✅ OverwriteObject complete: ${overwriteKeys.length} key(s) overwritten`);
|
|
}
|
|
|
|
this.readyDeferred.resolve();
|
|
console.log('✨ AppData initialization complete!');
|
|
}
|
|
|
|
/**
|
|
* returns a kvstore that resides in appdata
|
|
*/
|
|
public async getKvStore(): Promise<KeyValueStore<T>> {
|
|
await this.readyDeferred.promise;
|
|
return this.kvStore;
|
|
}
|
|
|
|
public async logMissingKeys(): Promise<Array<keyof T>> {
|
|
const kvStore = await this.getKvStore();
|
|
const missingMandatoryKeys = await kvStore.getMissingMandatoryKeys();
|
|
if (missingMandatoryKeys.length > 0) {
|
|
console.log(
|
|
`The following mandatory keys are missing in the appdata:\n -> ${missingMandatoryKeys.join(
|
|
',\n -> ',
|
|
)}`,
|
|
);
|
|
} else {
|
|
console.log('All mandatory keys are present in the appdata');
|
|
}
|
|
return missingMandatoryKeys;
|
|
}
|
|
|
|
public async waitForAndGetKey<K extends keyof T>(
|
|
keyArg: K,
|
|
): Promise<T[K] | undefined> {
|
|
await this.readyDeferred.promise;
|
|
await this.kvStore.waitForKeysPresent([keyArg]);
|
|
return this.kvStore.readKey(keyArg);
|
|
}
|
|
}
|