feat(mixer): enhance mixer functionality with interaction and tool legs
- Updated mixer to handle participant and isolated leg roles, allowing for IVR and consent interactions. - Introduced commands for starting and canceling interactions, managing tool legs for recording and transcription. - Implemented per-source audio handling for tool legs, enabling separate audio processing. - Enhanced DTMF handling to forward events between participant legs only. - Added support for PCM recording directly from tool legs, with WAV file generation. - Updated TypeScript definitions and functions to support new interaction and tool leg features.
This commit is contained in:
@@ -41,6 +41,32 @@ type TProxyCommands = {
|
||||
params: { call_id: string };
|
||||
result: { file_path: string; duration_ms: number };
|
||||
};
|
||||
start_interaction: {
|
||||
params: {
|
||||
call_id: string;
|
||||
leg_id: string;
|
||||
prompt_wav: string;
|
||||
expected_digits: string;
|
||||
timeout_ms: number;
|
||||
};
|
||||
result: { result: 'digit' | 'timeout' | 'cancelled'; digit?: string };
|
||||
};
|
||||
add_tool_leg: {
|
||||
params: {
|
||||
call_id: string;
|
||||
tool_type: 'recording' | 'transcription';
|
||||
config?: Record<string, unknown>;
|
||||
};
|
||||
result: { tool_leg_id: string };
|
||||
};
|
||||
remove_tool_leg: {
|
||||
params: { call_id: string; tool_leg_id: string };
|
||||
result: Record<string, never>;
|
||||
};
|
||||
set_leg_metadata: {
|
||||
params: { call_id: string; leg_id: string; key: string; value: unknown };
|
||||
result: Record<string, never>;
|
||||
};
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -280,11 +306,107 @@ export async function webrtcClose(sessionId: string): Promise<void> {
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Leg interaction & tool leg commands
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Start an interaction on a specific leg — isolate it, play a prompt, collect DTMF.
|
||||
* Blocks until the interaction completes (digit pressed, timeout, or cancelled).
|
||||
*/
|
||||
export async function startInteraction(
|
||||
callId: string,
|
||||
legId: string,
|
||||
promptWav: string,
|
||||
expectedDigits: string,
|
||||
timeoutMs: number,
|
||||
): Promise<{ result: 'digit' | 'timeout' | 'cancelled'; digit?: string } | null> {
|
||||
if (!bridge || !initialized) return null;
|
||||
try {
|
||||
const result = await bridge.sendCommand('start_interaction', {
|
||||
call_id: callId,
|
||||
leg_id: legId,
|
||||
prompt_wav: promptWav,
|
||||
expected_digits: expectedDigits,
|
||||
timeout_ms: timeoutMs,
|
||||
} as any);
|
||||
return result as any;
|
||||
} catch (e: any) {
|
||||
logFn?.(`[proxy-engine] start_interaction error: ${e?.message || e}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a tool leg (recording or transcription) to a call.
|
||||
* Tool legs receive per-source unmerged audio from all participants.
|
||||
*/
|
||||
export async function addToolLeg(
|
||||
callId: string,
|
||||
toolType: 'recording' | 'transcription',
|
||||
config?: Record<string, unknown>,
|
||||
): Promise<string | null> {
|
||||
if (!bridge || !initialized) return null;
|
||||
try {
|
||||
const result = await bridge.sendCommand('add_tool_leg', {
|
||||
call_id: callId,
|
||||
tool_type: toolType,
|
||||
config,
|
||||
} as any);
|
||||
return (result as any)?.tool_leg_id || null;
|
||||
} catch (e: any) {
|
||||
logFn?.(`[proxy-engine] add_tool_leg error: ${e?.message || e}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a tool leg from a call. Triggers finalization (WAV files, metadata).
|
||||
*/
|
||||
export async function removeToolLeg(callId: string, toolLegId: string): Promise<boolean> {
|
||||
if (!bridge || !initialized) return false;
|
||||
try {
|
||||
await bridge.sendCommand('remove_tool_leg', {
|
||||
call_id: callId,
|
||||
tool_leg_id: toolLegId,
|
||||
} as any);
|
||||
return true;
|
||||
} catch (e: any) {
|
||||
logFn?.(`[proxy-engine] remove_tool_leg error: ${e?.message || e}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a metadata key-value pair on a leg.
|
||||
*/
|
||||
export async function setLegMetadata(
|
||||
callId: string,
|
||||
legId: string,
|
||||
key: string,
|
||||
value: unknown,
|
||||
): Promise<boolean> {
|
||||
if (!bridge || !initialized) return false;
|
||||
try {
|
||||
await bridge.sendCommand('set_leg_metadata', {
|
||||
call_id: callId,
|
||||
leg_id: legId,
|
||||
key,
|
||||
value,
|
||||
} as any);
|
||||
return true;
|
||||
} catch (e: any) {
|
||||
logFn?.(`[proxy-engine] set_leg_metadata error: ${e?.message || e}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to an event from the proxy engine.
|
||||
* Event names: incoming_call, outbound_device_call, call_ringing,
|
||||
* call_answered, call_ended, provider_registered, device_registered,
|
||||
* dtmf_digit, recording_done, sip_unhandled
|
||||
* dtmf_digit, recording_done, tool_recording_done, tool_transcription_done,
|
||||
* leg_added, leg_removed, sip_unhandled
|
||||
*/
|
||||
export function onProxyEvent(event: string, handler: (data: any) => void): void {
|
||||
if (!bridge) throw new Error('proxy engine not initialized');
|
||||
|
||||
@@ -96,6 +96,16 @@ interface IDeviceStatus {
|
||||
isBrowser: boolean;
|
||||
}
|
||||
|
||||
interface IActiveLeg {
|
||||
id: string;
|
||||
type: 'sip-device' | 'sip-provider' | 'webrtc' | 'tool';
|
||||
state: string;
|
||||
codec: string | null;
|
||||
rtpPort: number | null;
|
||||
remoteMedia: string | null;
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface IActiveCall {
|
||||
id: string;
|
||||
direction: string;
|
||||
@@ -104,6 +114,13 @@ interface IActiveCall {
|
||||
providerUsed: string | null;
|
||||
state: string;
|
||||
startedAt: number;
|
||||
legs: Map<string, IActiveLeg>;
|
||||
}
|
||||
|
||||
interface IHistoryLeg {
|
||||
id: string;
|
||||
type: string;
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface ICallHistoryEntry {
|
||||
@@ -113,6 +130,7 @@ interface ICallHistoryEntry {
|
||||
calleeNumber: string | null;
|
||||
startedAt: number;
|
||||
duration: number;
|
||||
legs: IHistoryLeg[];
|
||||
}
|
||||
|
||||
const providerStatuses = new Map<string, IProviderStatus>();
|
||||
@@ -187,7 +205,18 @@ function getStatus() {
|
||||
calls: [...activeCalls.values()].map((c) => ({
|
||||
...c,
|
||||
duration: Math.floor((Date.now() - c.startedAt) / 1000),
|
||||
legs: [],
|
||||
legs: [...c.legs.values()].map((l) => ({
|
||||
id: l.id,
|
||||
type: l.type,
|
||||
state: l.state,
|
||||
codec: l.codec,
|
||||
rtpPort: l.rtpPort,
|
||||
remoteMedia: l.remoteMedia,
|
||||
metadata: l.metadata || {},
|
||||
pktSent: 0,
|
||||
pktReceived: 0,
|
||||
transcoding: false,
|
||||
})),
|
||||
})),
|
||||
callHistory,
|
||||
contacts: appConfig.contacts || [],
|
||||
@@ -242,6 +271,7 @@ async function startProxyEngine(): Promise<void> {
|
||||
providerUsed: data.provider_id,
|
||||
state: 'ringing',
|
||||
startedAt: Date.now(),
|
||||
legs: new Map(),
|
||||
});
|
||||
|
||||
// Notify browsers of incoming call.
|
||||
@@ -266,6 +296,7 @@ async function startProxyEngine(): Promise<void> {
|
||||
providerUsed: null,
|
||||
state: 'setting-up',
|
||||
startedAt: Date.now(),
|
||||
legs: new Map(),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -279,6 +310,7 @@ async function startProxyEngine(): Promise<void> {
|
||||
providerUsed: data.provider_id,
|
||||
state: 'setting-up',
|
||||
startedAt: Date.now(),
|
||||
legs: new Map(),
|
||||
});
|
||||
|
||||
// Notify all browser devices — they can connect via WebRTC to listen/talk.
|
||||
@@ -303,6 +335,20 @@ async function startProxyEngine(): Promise<void> {
|
||||
if (call) {
|
||||
call.state = 'connected';
|
||||
log(`[call] ${data.call_id} connected`);
|
||||
|
||||
// Enrich provider leg with media info from the answered event.
|
||||
if (data.provider_media_addr && data.provider_media_port) {
|
||||
for (const leg of call.legs.values()) {
|
||||
if (leg.type === 'sip-provider') {
|
||||
leg.remoteMedia = `${data.provider_media_addr}:${data.provider_media_port}`;
|
||||
if (data.sip_pt !== undefined) {
|
||||
const codecNames: Record<number, string> = { 0: 'PCMU', 8: 'PCMA', 9: 'G.722', 111: 'Opus' };
|
||||
leg.codec = codecNames[data.sip_pt] || `PT${data.sip_pt}`;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to link WebRTC session to this call for audio bridging.
|
||||
@@ -331,6 +377,15 @@ async function startProxyEngine(): Promise<void> {
|
||||
const call = activeCalls.get(data.call_id);
|
||||
if (call) {
|
||||
log(`[call] ${data.call_id} ended: ${data.reason} (${data.duration}s)`);
|
||||
// Snapshot legs with metadata for history.
|
||||
const historyLegs: IHistoryLeg[] = [];
|
||||
for (const [, leg] of call.legs) {
|
||||
historyLegs.push({
|
||||
id: leg.id,
|
||||
type: leg.type,
|
||||
metadata: leg.metadata || {},
|
||||
});
|
||||
}
|
||||
// Move to history.
|
||||
callHistory.unshift({
|
||||
id: call.id,
|
||||
@@ -339,6 +394,7 @@ async function startProxyEngine(): Promise<void> {
|
||||
calleeNumber: call.calleeNumber,
|
||||
startedAt: call.startedAt,
|
||||
duration: data.duration,
|
||||
legs: historyLegs,
|
||||
});
|
||||
if (callHistory.length > MAX_HISTORY) callHistory.pop();
|
||||
activeCalls.delete(data.call_id);
|
||||
@@ -361,17 +417,50 @@ async function startProxyEngine(): Promise<void> {
|
||||
log(`[sip] unhandled ${data.method_or_status} Call-ID=${data.call_id?.slice(0, 20)} from=${data.from_addr}:${data.from_port}`);
|
||||
});
|
||||
|
||||
// Leg events (multiparty).
|
||||
// Leg events (multiparty) — update shadow state so the dashboard shows legs.
|
||||
onProxyEvent('leg_added', (data: any) => {
|
||||
log(`[leg] added: call=${data.call_id} leg=${data.leg_id} kind=${data.kind} state=${data.state}`);
|
||||
const call = activeCalls.get(data.call_id);
|
||||
if (call) {
|
||||
call.legs.set(data.leg_id, {
|
||||
id: data.leg_id,
|
||||
type: data.kind,
|
||||
state: data.state,
|
||||
codec: null,
|
||||
rtpPort: null,
|
||||
remoteMedia: null,
|
||||
metadata: data.metadata || {},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
onProxyEvent('leg_removed', (data: any) => {
|
||||
log(`[leg] removed: call=${data.call_id} leg=${data.leg_id}`);
|
||||
activeCalls.get(data.call_id)?.legs.delete(data.leg_id);
|
||||
});
|
||||
|
||||
onProxyEvent('leg_state_changed', (data: any) => {
|
||||
log(`[leg] state: call=${data.call_id} leg=${data.leg_id} → ${data.state}`);
|
||||
const call = activeCalls.get(data.call_id);
|
||||
if (!call) return;
|
||||
const leg = call.legs.get(data.leg_id);
|
||||
if (leg) {
|
||||
leg.state = data.state;
|
||||
if (data.metadata) leg.metadata = data.metadata;
|
||||
} else {
|
||||
// Initial legs (provider/device) don't emit leg_added — create on first state change.
|
||||
const legId: string = data.leg_id;
|
||||
const type = legId.includes('-prov') ? 'sip-provider' : legId.includes('-dev') ? 'sip-device' : 'webrtc';
|
||||
call.legs.set(data.leg_id, {
|
||||
id: data.leg_id,
|
||||
type,
|
||||
state: data.state,
|
||||
codec: null,
|
||||
rtpPort: null,
|
||||
remoteMedia: null,
|
||||
metadata: data.metadata || {},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// WebRTC events from Rust — forward ICE candidates to browser via WebSocket.
|
||||
@@ -484,6 +573,7 @@ initWebUi(
|
||||
providerUsed: providerId || null,
|
||||
state: 'setting-up',
|
||||
startedAt: Date.now(),
|
||||
legs: new Map(),
|
||||
});
|
||||
} else {
|
||||
log(`[dashboard] call failed for ${number}`);
|
||||
|
||||
Reference in New Issue
Block a user