rename package from @push.rocks/npmextra to @push.rocks/smartconfig
- Rename all source files from npmextra.* to simpler names (classes.appdata.ts, etc.) - Rename Npmextra class to Smartconfig - Config file changed from npmextra.json to smartconfig.json - KV store path changed from ~/.npmextra/kv to ~/.smartconfig/kv - Update all imports, tests, and metadata
This commit is contained in:
548
ts/classes.appdata.ts
Normal file
548
ts/classes.appdata.ts
Normal file
@@ -0,0 +1,548 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import { KeyValueStore } from './classes.keyvaluestore.js';
|
||||
|
||||
// ============================================================================
|
||||
// Singleton Qenv Provider
|
||||
// ============================================================================
|
||||
let sharedQenv: plugins.qenv.Qenv | undefined;
|
||||
|
||||
function getQenv(): plugins.qenv.Qenv {
|
||||
if (!sharedQenv) {
|
||||
sharedQenv = new plugins.qenv.Qenv(
|
||||
process.cwd(),
|
||||
plugins.path.join(process.cwd(), '.nogit')
|
||||
);
|
||||
}
|
||||
return sharedQenv;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Security - Redaction for sensitive data
|
||||
// ============================================================================
|
||||
/**
|
||||
* Redacts sensitive values in logs to prevent exposure of secrets
|
||||
*/
|
||||
function redactSensitiveValue(key: string, value: unknown): string {
|
||||
// List of patterns that indicate sensitive data
|
||||
const sensitivePatterns = [
|
||||
/secret/i, /token/i, /key/i, /password/i, /pass/i,
|
||||
/api/i, /credential/i, /auth/i, /private/i, /jwt/i,
|
||||
/cert/i, /signature/i, /bearer/i
|
||||
];
|
||||
|
||||
// Check if key contains sensitive pattern
|
||||
const isSensitive = sensitivePatterns.some(pattern => pattern.test(key));
|
||||
|
||||
if (isSensitive) {
|
||||
if (typeof value === 'string') {
|
||||
// Show first 3 chars and length for debugging
|
||||
return value.length > 3
|
||||
? `${value.substring(0, 3)}...[${value.length} chars]`
|
||||
: '[redacted]';
|
||||
}
|
||||
return '[redacted]';
|
||||
}
|
||||
|
||||
// Check if value looks like a JWT token or base64 secret
|
||||
if (typeof value === 'string') {
|
||||
// JWT tokens start with eyJ
|
||||
if (value.startsWith('eyJ')) {
|
||||
return `eyJ...[${value.length} chars]`;
|
||||
}
|
||||
// Very long strings might be encoded secrets
|
||||
if (value.length > 100) {
|
||||
return `${value.substring(0, 50)}...[${value.length} chars total]`;
|
||||
}
|
||||
}
|
||||
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Type Converters - Centralized conversion logic
|
||||
// ============================================================================
|
||||
function toBoolean(value: unknown): boolean {
|
||||
// If already boolean, return as-is
|
||||
if (typeof value === 'boolean') {
|
||||
console.log(` 🔹 toBoolean: value is already boolean: ${value}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
// Handle null/undefined
|
||||
if (value == null) {
|
||||
console.log(` 🔹 toBoolean: value is null/undefined, returning false`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Handle string representations
|
||||
const s = String(value).toLowerCase().trim();
|
||||
|
||||
// True values: "true", "1", "yes", "y", "on"
|
||||
if (['true', '1', 'yes', 'y', 'on'].includes(s)) {
|
||||
console.log(` 🔹 toBoolean: converting "${value}" to true`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// False values: "false", "0", "no", "n", "off"
|
||||
if (['false', '0', 'no', 'n', 'off'].includes(s)) {
|
||||
console.log(` 🔹 toBoolean: converting "${value}" to false`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Default: non-empty string = true, empty = false
|
||||
const result = s.length > 0;
|
||||
console.log(` 🔹 toBoolean: defaulting "${value}" to ${result}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function toJson<T = any>(value: unknown): T | undefined {
|
||||
if (value == null) return undefined;
|
||||
if (typeof value === 'string') {
|
||||
try {
|
||||
return JSON.parse(value);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
return value as T;
|
||||
}
|
||||
|
||||
function fromBase64(value: unknown): string | undefined {
|
||||
if (value == null) return undefined;
|
||||
try {
|
||||
return Buffer.from(String(value), 'base64').toString('utf8');
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
function toNumber(value: unknown): number | undefined {
|
||||
if (value == null) return undefined;
|
||||
const num = Number(value);
|
||||
return Number.isNaN(num) ? undefined : num;
|
||||
}
|
||||
|
||||
function toString(value: unknown): string | undefined {
|
||||
if (value == null) return undefined;
|
||||
return String(value);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Declarative Pipeline Architecture
|
||||
// ============================================================================
|
||||
type Transform = 'boolean' | 'json' | 'base64' | 'number';
|
||||
|
||||
type MappingSpec = {
|
||||
source:
|
||||
| { type: 'env'; key: string }
|
||||
| { type: 'hard'; value: string };
|
||||
transforms: Transform[];
|
||||
};
|
||||
|
||||
// Transform registry for extensibility
|
||||
const transformRegistry: Record<string, (v: unknown) => unknown> = {
|
||||
boolean: toBoolean,
|
||||
json: toJson,
|
||||
base64: fromBase64,
|
||||
number: toNumber,
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a mapping string into a declarative spec
|
||||
*/
|
||||
function parseMappingSpec(input: string): MappingSpec {
|
||||
const transforms: Transform[] = [];
|
||||
let remaining = input;
|
||||
|
||||
// Check for hardcoded prefixes with type conversion
|
||||
if (remaining.startsWith('hard_boolean:')) {
|
||||
return {
|
||||
source: { type: 'hard', value: remaining.slice(13) },
|
||||
transforms: ['boolean']
|
||||
};
|
||||
}
|
||||
|
||||
if (remaining.startsWith('hard_json:')) {
|
||||
return {
|
||||
source: { type: 'hard', value: remaining.slice(10) },
|
||||
transforms: ['json']
|
||||
};
|
||||
}
|
||||
|
||||
if (remaining.startsWith('hard_base64:')) {
|
||||
return {
|
||||
source: { type: 'hard', value: remaining.slice(12) },
|
||||
transforms: ['base64']
|
||||
};
|
||||
}
|
||||
|
||||
// Check for generic hard: prefix
|
||||
if (remaining.startsWith('hard:')) {
|
||||
remaining = remaining.slice(5);
|
||||
// Check for legacy suffixes on hardcoded values
|
||||
if (remaining.endsWith('_JSON')) {
|
||||
transforms.push('json');
|
||||
remaining = remaining.slice(0, -5);
|
||||
} else if (remaining.endsWith('_BASE64')) {
|
||||
transforms.push('base64');
|
||||
remaining = remaining.slice(0, -7);
|
||||
}
|
||||
return {
|
||||
source: { type: 'hard', value: remaining },
|
||||
transforms
|
||||
};
|
||||
}
|
||||
|
||||
// Check for env var prefixes
|
||||
if (remaining.startsWith('boolean:')) {
|
||||
transforms.push('boolean');
|
||||
remaining = remaining.slice(8);
|
||||
} else if (remaining.startsWith('json:')) {
|
||||
transforms.push('json');
|
||||
remaining = remaining.slice(5);
|
||||
} else if (remaining.startsWith('base64:')) {
|
||||
transforms.push('base64');
|
||||
remaining = remaining.slice(7);
|
||||
}
|
||||
|
||||
// Check for legacy suffixes on env vars
|
||||
if (remaining.endsWith('_JSON')) {
|
||||
transforms.push('json');
|
||||
remaining = remaining.slice(0, -5);
|
||||
} else if (remaining.endsWith('_BASE64')) {
|
||||
transforms.push('base64');
|
||||
remaining = remaining.slice(0, -7);
|
||||
}
|
||||
|
||||
return {
|
||||
source: { type: 'env', key: remaining },
|
||||
transforms
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the source value (env var or hardcoded)
|
||||
*/
|
||||
async function resolveSource(source: MappingSpec['source']): Promise<unknown> {
|
||||
if (source.type === 'hard') {
|
||||
return source.value;
|
||||
}
|
||||
// source.type === 'env'
|
||||
// Workaround for Qenv bug where empty strings are treated as undefined
|
||||
// Check process.env directly first to preserve empty strings
|
||||
if (Object.prototype.hasOwnProperty.call(process.env, source.key)) {
|
||||
return process.env[source.key];
|
||||
}
|
||||
// Fall back to Qenv for other sources (env.json, docker secrets, etc.)
|
||||
return await getQenv().getEnvVarOnDemand(source.key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply transformations in sequence
|
||||
*/
|
||||
function applyTransforms(value: unknown, transforms: Transform[]): unknown {
|
||||
return transforms.reduce((acc, transform) => {
|
||||
const fn = transformRegistry[transform];
|
||||
return fn ? fn(acc) : acc;
|
||||
}, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a mapping value through the complete pipeline
|
||||
*/
|
||||
async function processMappingValue(mappingString: string): Promise<unknown> {
|
||||
const spec = parseMappingSpec(mappingString);
|
||||
const keyName = spec.source.type === 'env' ? spec.source.key : 'hardcoded';
|
||||
|
||||
console.log(` 🔍 Processing mapping: "${mappingString}"`);
|
||||
console.log(` Source: ${spec.source.type === 'env' ? `env:${spec.source.key}` : `hard:${spec.source.value}`}`);
|
||||
console.log(` Transforms: ${spec.transforms.length > 0 ? spec.transforms.join(', ') : 'none'}`);
|
||||
|
||||
const rawValue = await resolveSource(spec.source);
|
||||
console.log(` Raw value: ${redactSensitiveValue(keyName, rawValue)} (type: ${typeof rawValue})`);
|
||||
|
||||
if (rawValue === undefined || rawValue === null) {
|
||||
console.log(` ⚠️ Raw value is undefined/null, returning undefined`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const result = applyTransforms(rawValue, spec.transforms);
|
||||
console.log(` Final value: ${redactSensitiveValue(keyName, result)} (type: ${typeof result})`);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively evaluate mapping values (strings or nested objects)
|
||||
*/
|
||||
async function evaluateMappingValue(mappingValue: any): Promise<any> {
|
||||
// Handle null explicitly - it should return null, not be treated as object
|
||||
if (mappingValue === null) {
|
||||
console.log(` 📌 Value is null, returning null`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Handle strings (mapping specs)
|
||||
if (typeof mappingValue === 'string') {
|
||||
return processMappingValue(mappingValue);
|
||||
}
|
||||
|
||||
// Handle objects (but not arrays or null)
|
||||
if (mappingValue && typeof mappingValue === 'object' && !Array.isArray(mappingValue)) {
|
||||
console.log(` 📂 Processing nested object with ${Object.keys(mappingValue).length} keys`);
|
||||
const result: any = {};
|
||||
for (const [key, value] of Object.entries(mappingValue)) {
|
||||
console.log(` → Processing nested key "${key}"`);
|
||||
const evaluated = await evaluateMappingValue(value);
|
||||
// Important: Don't filter out false or other falsy values!
|
||||
// Only skip if explicitly undefined
|
||||
if (evaluated !== undefined) {
|
||||
result[key] = evaluated;
|
||||
console.log(` ✓ Nested key "${key}" = ${redactSensitiveValue(key, evaluated)} (type: ${typeof evaluated})`);
|
||||
} else {
|
||||
console.log(` ⚠️ Nested key "${key}" evaluated to undefined, skipping`);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// For any other type (numbers, booleans, etc.), return as-is
|
||||
// Note: We don't have key context here, so we'll just indicate the type
|
||||
console.log(` 📎 Returning value as-is: [value] (type: ${typeof mappingValue})`);
|
||||
return mappingValue;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// AppData Interface and Class
|
||||
// ============================================================================
|
||||
export interface IAppDataOptions<T = any> {
|
||||
dirPath?: string;
|
||||
requiredKeys?: Array<keyof T>;
|
||||
|
||||
/**
|
||||
* Whether keys should be persisted on disk or not
|
||||
*/
|
||||
ephemeral?: boolean;
|
||||
|
||||
/**
|
||||
* @deprecated Use 'ephemeral' instead
|
||||
*/
|
||||
ephermal?: boolean;
|
||||
|
||||
/**
|
||||
* kvStoreKey: 'MY_ENV_VAR'
|
||||
*/
|
||||
envMapping?: plugins.tsclass.typeFest.PartialDeep<T>;
|
||||
overwriteObject?: plugins.tsclass.typeFest.PartialDeep<T>;
|
||||
}
|
||||
|
||||
export class AppData<T = any> {
|
||||
/**
|
||||
* creates appdata. If no pathArg is given, data will be stored here:
|
||||
* ${PWD}/.nogit/appdata
|
||||
* @param pathArg
|
||||
* @returns
|
||||
*/
|
||||
public static async createAndInit<T = any>(
|
||||
optionsArg: IAppDataOptions<T> = {},
|
||||
): Promise<AppData<T>> {
|
||||
const appData = new AppData<T>(optionsArg);
|
||||
await appData.readyDeferred.promise;
|
||||
return appData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper to get an environment variable as a boolean
|
||||
* @param envVarName The name of the environment variable
|
||||
* @returns boolean value (true if env var is "true", false otherwise)
|
||||
*/
|
||||
public static async valueAsBoolean(envVarName: string): Promise<boolean> {
|
||||
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
||||
return toBoolean(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper to get an environment variable as parsed JSON
|
||||
* @param envVarName The name of the environment variable
|
||||
* @returns Parsed JSON object/array
|
||||
*/
|
||||
public static async valueAsJson<R = any>(envVarName: string): Promise<R | undefined> {
|
||||
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
||||
return toJson<R>(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper to get an environment variable as base64 decoded string
|
||||
* @param envVarName The name of the environment variable
|
||||
* @returns Decoded string
|
||||
*/
|
||||
public static async valueAsBase64(envVarName: string): Promise<string | undefined> {
|
||||
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
||||
return fromBase64(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper to get an environment variable as a string
|
||||
* @param envVarName The name of the environment variable
|
||||
* @returns String value
|
||||
*/
|
||||
public static async valueAsString(envVarName: string): Promise<string | undefined> {
|
||||
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
||||
return toString(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Static helper to get an environment variable as a number
|
||||
* @param envVarName The name of the environment variable
|
||||
* @returns Number value
|
||||
*/
|
||||
public static async valueAsNumber(envVarName: string): Promise<number | undefined> {
|
||||
const value = await getQenv().getEnvVarOnDemand(envVarName);
|
||||
return toNumber(value);
|
||||
}
|
||||
|
||||
// instance
|
||||
public readyDeferred = plugins.smartpromise.defer<void>();
|
||||
public options: IAppDataOptions<T>;
|
||||
private kvStore: KeyValueStore<T>;
|
||||
|
||||
constructor(optionsArg: IAppDataOptions<T> = {}) {
|
||||
this.options = optionsArg;
|
||||
this.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* inits app data
|
||||
*/
|
||||
private async init() {
|
||||
console.log('🚀 Initializing AppData...');
|
||||
|
||||
// Handle backward compatibility for typo
|
||||
const isEphemeral = this.options.ephemeral ?? this.options.ephermal ?? false;
|
||||
if (this.options.ephermal && !this.options.ephemeral) {
|
||||
console.warn('⚠️ Option "ephermal" is deprecated, use "ephemeral" instead.');
|
||||
}
|
||||
|
||||
if (this.options.dirPath) {
|
||||
console.log(` 📁 Using custom directory: ${this.options.dirPath}`);
|
||||
} else if (isEphemeral) {
|
||||
console.log(` 💨 Using ephemeral storage (in-memory only)`);
|
||||
} else {
|
||||
const appDataDir = '/app/data';
|
||||
const dataDir = '/data';
|
||||
const nogitAppData = '.nogit/appdata';
|
||||
const appDataExists = plugins.smartfile.fs.isDirectory(appDataDir);
|
||||
const dataExists = plugins.smartfile.fs.isDirectory(dataDir);
|
||||
if (appDataExists) {
|
||||
this.options.dirPath = appDataDir;
|
||||
console.log(` 📁 Auto-selected container directory: ${appDataDir}`);
|
||||
} else if (dataExists) {
|
||||
this.options.dirPath = dataDir;
|
||||
console.log(` 📁 Auto-selected data directory: ${dataDir}`);
|
||||
} else {
|
||||
await plugins.smartfile.fs.ensureDir(nogitAppData);
|
||||
this.options.dirPath = nogitAppData;
|
||||
console.log(` 📁 Auto-selected local directory: ${nogitAppData}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.kvStore = new KeyValueStore<T>({
|
||||
typeArg: isEphemeral ? 'ephemeral' : 'custom',
|
||||
identityArg: 'appkv',
|
||||
customPath: this.options.dirPath,
|
||||
mandatoryKeys: this.options.requiredKeys as Array<keyof T>,
|
||||
});
|
||||
|
||||
if (this.options.envMapping) {
|
||||
console.log(`📦 Processing envMapping for AppData...`);
|
||||
const totalKeys = Object.keys(this.options.envMapping).length;
|
||||
let processedCount = 0;
|
||||
|
||||
// Process each top-level key in envMapping
|
||||
for (const key in this.options.envMapping) {
|
||||
try {
|
||||
const mappingSpec = this.options.envMapping[key];
|
||||
const specType = mappingSpec === null ? 'null' :
|
||||
typeof mappingSpec === 'string' ? mappingSpec :
|
||||
typeof mappingSpec === 'object' ? 'nested object' :
|
||||
typeof mappingSpec;
|
||||
console.log(` → Processing key "${key}" with spec: ${specType}`);
|
||||
|
||||
const evaluated = await evaluateMappingValue(mappingSpec);
|
||||
// Important: Don't skip false, 0, empty string, or null values!
|
||||
// Only skip if explicitly undefined
|
||||
if (evaluated !== undefined) {
|
||||
await this.kvStore.writeKey(key as keyof T, evaluated);
|
||||
processedCount++;
|
||||
const valueType = evaluated === null ? 'null' :
|
||||
Array.isArray(evaluated) ? 'array' :
|
||||
typeof evaluated;
|
||||
const valuePreview = evaluated === null ? 'null' :
|
||||
typeof evaluated === 'object' ?
|
||||
(Array.isArray(evaluated) ? `[${evaluated.length} items]` : `{${Object.keys(evaluated).length} keys}`) :
|
||||
redactSensitiveValue(key, evaluated);
|
||||
console.log(` ✅ Successfully processed key "${key}" = ${valuePreview} (type: ${valueType})`);
|
||||
} else {
|
||||
console.log(` ⚠️ Key "${key}" evaluated to undefined, skipping`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(` ❌ Failed to evaluate envMapping for key "${key}":`, err);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`📊 EnvMapping complete: ${processedCount}/${totalKeys} keys successfully processed`);
|
||||
}
|
||||
|
||||
// Apply overwrite object after env mapping
|
||||
if (this.options.overwriteObject) {
|
||||
const overwriteKeys = Object.keys(this.options.overwriteObject);
|
||||
console.log(`🔄 Applying overwriteObject with ${overwriteKeys.length} key(s)...`);
|
||||
|
||||
for (const key of overwriteKeys) {
|
||||
const value = this.options.overwriteObject[key];
|
||||
const valueType = Array.isArray(value) ? 'array' : typeof value;
|
||||
console.log(` 🔧 Overwriting key "${key}" with ${valueType} value`);
|
||||
|
||||
await this.kvStore.writeKey(
|
||||
key as keyof T,
|
||||
value,
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`✅ OverwriteObject complete: ${overwriteKeys.length} key(s) overwritten`);
|
||||
}
|
||||
|
||||
this.readyDeferred.resolve();
|
||||
console.log('✨ AppData initialization complete!');
|
||||
}
|
||||
|
||||
/**
|
||||
* returns a kvstore that resides in appdata
|
||||
*/
|
||||
public async getKvStore(): Promise<KeyValueStore<T>> {
|
||||
await this.readyDeferred.promise;
|
||||
return this.kvStore;
|
||||
}
|
||||
|
||||
public async logMissingKeys(): Promise<Array<keyof T>> {
|
||||
const kvStore = await this.getKvStore();
|
||||
const missingMandatoryKeys = await kvStore.getMissingMandatoryKeys();
|
||||
if (missingMandatoryKeys.length > 0) {
|
||||
console.log(
|
||||
`The following mandatory keys are missing in the appdata:\n -> ${missingMandatoryKeys.join(
|
||||
',\n -> ',
|
||||
)}`,
|
||||
);
|
||||
} else {
|
||||
console.log('All mandatory keys are present in the appdata');
|
||||
}
|
||||
return missingMandatoryKeys;
|
||||
}
|
||||
|
||||
public async waitForAndGetKey<K extends keyof T>(
|
||||
keyArg: K,
|
||||
): Promise<T[K] | undefined> {
|
||||
await this.readyDeferred.promise;
|
||||
await this.kvStore.waitForKeysPresent([keyArg]);
|
||||
return this.kvStore.readKey(keyArg);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user