Agentic AI Telephony Solution

// Plivo Client Setup
const plivo = require('plivo');
const client = new plivo.Client(PLIVO_AUTH_ID, PLIVO_AUTH_TOKEN);

// Initiate Call
async initiateCall(to, answerUrl, hangupUrl, from) {
  const response = await this.client.calls.create(
    from,
    to,
    answerUrl,
    { answer_method: 'POST', hangup_url: hangupUrl }
  );
  return response.requestUuid;
}

// WebSocket Connection for Media Streaming
app.ws('/answer', (ws, req) => {
  const handler = new PlivoStreamHandler(ws, req);
  // Handles bidirectional audio streaming
});

// Handle Media Stream Events
handleMessage(data) {
  const msg = JSON.parse(data);
  if (msg.event === 'media') {
    // Send audio chunks to Deepgram
    this.transcriptionService.send(
      Buffer.from(msg.media.payload, 'base64')
    );
  }
}

// Deepgram Live Transcription Setup
const { createClient, LiveTranscriptionEvents } = require("@deepgram/sdk");

this.connection = this.deepgram.listen.live({
  model: "nova-2",
  encoding: "mulaw",        // μ-law encoding
  sample_rate: 8000,        // 8kHz sample rate
  interim_results: true,     // Real-time partial results
  language: "hi",           // Hindi language support
  punctuate: true,
  endpointing: 300,         // Endpoint detection
  utterance_end_ms: 1500,   // Utterance end timeout
  filler_words: true        // Detect filler words
});

// Handle Transcription Events
this.connection.on(LiveTranscriptionEvents.Transcript, (event) => {
  const text = event.channel?.alternatives[0]?.transcript;
  
  // Interim results for interruption detection
  if (!event.is_final) {
    this.emit('interim-transcription', text, event.start);
  }
  
  // Final results when speech is complete
  if (event.is_final && event.speech_final) {
    this.emit('transcription', this.finalResult, timestamp);
  }
});

// Send Audio to Deepgram
send(audioData) {
  this.connection.send(audioData);
}

// GPT Realtime WebSocket Connection
initWebSocket() {
  const url = `${this.endpoint}/openai/realtime?api-version=${this.apiVersion}&deployment=${this.deploymentId}`;
  
  this.websocket = new WebSocket(url, {
    headers: { "Authorization": `Bearer ${this.apiKey}` }
  });
  
  this.websocket.on('message', (data) => {
    const event = JSON.parse(data);
    
    // Handle streaming text responses
    if (event.type === 'response.text.delta') {
      this.sentenceBuffer += event.delta;
      this.emit('gptreply', this.sentenceBuffer, event.item_id);
    }
    
    // Handle function calls
    if (event.type === 'response.function_call') {
      if (event.name === 'call_hangup') {
        this.emit('hangup-call');
      }
      if (event.name === 'machine_detection') {
        this.emit('machine-detected');
      }
    }
  });
}

// Send User Message
sendMessage(text, interactionCount) {
  const message = {
    type: 'conversation.item.create',
    item: {
      type: 'message',
      role: 'user',
      content: [{ type: 'input_text', text }]
    }
  };
  this.websocket.send(JSON.stringify(message));
}

// ElevenLabs WebSocket TTS Setup
initWebSocket() {
  const uri = `wss://api.elevenlabs.io/v1/text-to-speech/${this.voiceId}/stream-input?model_id=${this.model}&output_format=ulaw_8000&inactivity_timeout=180&auto_mode=true&enable_ssml_parsing=true`;
  
  this.websocket = new WebSocket(uri, {
    headers: { "xi-api-key": this.apiKey }
  });
  
  this.websocket.on('message', (event) => {
    const data = JSON.parse(event.toString());
    
    if (data.audio) {
      const base64Chunk = Buffer.from(data.audio).toString("base64");
      
      // Calculate audio duration from alignment data
      const startTimes = data.alignment?.charStartTimesMs ?? [];
      const durations = data.alignment?.charDurationsMs ?? [];
      const audioDuration = startTimes[startTimes.length - 1] + 
                           durations[durations.length - 1];
      
      // Emit audio chunk with duration
      this.emit("speech", data.audio, audioDuration);
    }
  });
}

// Generate Speech
generate(text) {
  const message = {
    text: text,
    flush: false  // Stream mode
  };
  this.websocket.send(JSON.stringify(message));
}

// Intent Severity Levels
setIntentSeverity() {
  if (this.interactionCount <= 2) {
    this.intentSeverity = "MEDIUM";  // Early in conversation
  } else {
    this.intentSeverity = "LOW";     // Later in conversation
  }
}

// Initial state: HIGH severity (welcome message)
this.intentSeverity = "HIGH";

// Interruption Rules by Severity:
// HIGH:   Never interrupt - critical messages (welcome, important info)
// MEDIUM: Allow interrupts for ≤2 words only
// LOW:    Allow all valid interrupts

// Usage in Interruption Detection
if (intentSeverity === "HIGH" && isTTSPlaying) {
  return true; // Never interrupt
}

if (words.length <= 2 && isTTSPlaying && intentSeverity === "MEDIUM") {
  return true; // Ignore short interruptions
}