reactjs - Send Deepgram generated STT transcription from live Twilio call to client in Node.js and React.js - Stack Overflow

admin管理员组

文章数量:1323029

I want to make a twilio call to my phone number and transcribe it using Deepgram, then want to display the transcription on the browser.

The call part is successfully done.

Now, I want to send data to the client and display it on the browser. But I am not able to do it. The console.log('Transcription to send ----- ', dataToSend) in the sendToClient function is printing the data, but not able to send to the client.

I am using Web Socket to achieve this.

Any help would be really appriciated.

This is the code in the app.js file

const { TranscriptionService } = require('./services/transcription-service');

app.post('/make-call', async (req, res) => {
  try {
    const { toNumber } = req.body;
    const accountSid = process.env.TWILIO_ACCOUNT_SID;
    const authToken = process.env.TWILIO_AUTH_TOKEN;
    
    const client = require('twilio')(accountSid, authToken);
    await client.calls.create({
      url: `https://${process.env.SERVER}/incoming`,
      to: toNumber,
      from: process.env.FROM_NUMBER
    })
    .then(call => {
      res.json({ success: true, message: `Call initiated to ${toNumber}`, data: { callSid: call.sid } });
    });
  }
  catch (error) {
    console.log(error.message)
    res.json({ success: false, message: `${error.message}` });
  }
});

app.post('/incoming', (req, res) => {
  try {
    const response = new VoiceResponse();
    const connect = response.connect();
    connect.stream({ url: `wss://${process.env.SERVER}/connection` });
  
    res.type('text/xml');
    res.end(response.toString());
  } catch (err) {
    console.log(err);
  }
});

const clients = new Map(); // Store connected clients by their ids

app.ws('/connection', (ws, req) => {
  try {
    ws.on('error', (error) => {
      console.error('WebSocket Error:', error); // Log the entire error object
    });
    // Filled in from start message
    let streamSid;
    let callSid;
    
    // Unique ID for the connection (e.g., based on remote address or timestamp)
    const clientId = Date.now().toString();
    clients.set(clientId, ws);

    const transcriptionService = new TranscriptionService();
  
    let marks = [];
    let interactionCount = 0;
  
    // Incoming from MediaStream
    ws.on('message', function message(data) {
      const msg = JSON.parse(data);
      console.log(msg.payload)
      if (msg.event === 'start') {
        streamSid = msg.start.streamSid;
        callSid = msg.start.callSid;
      }
      else if (msg.event === 'media') {
        transcriptionService.send(msg?.media?.payload);
      }
      else if (msg.event === 'mark') {
        marks = marks.filter(m => m !== msg.mark.name);
      }
      else if (msg.event === 'stop') {
        console.log(`Twilio -> Media stream ${streamSid} ended.`.underline.red);
      }
      
    });
  
    transcriptionService.on('transcription', async (text) => {
      if (!text) { return; }
      sendToClient(text) // Send Data to the client
    });

    async function sendToClient(dataToSend) {
      const client = clients.get(clientId);
      if (client && client.readyState === WebSocket.OPEN) {
        console.log('Transcription to send ----- ', dataToSend)
        await client.send(JSON.stringify({
          streamSid,
          event: 'clear',
          transcription: dataToSend,
        }));
      }
    }

  } catch (err) {
    console.log(err);
  }
});

This is the code block from the TranscriptionService file

this.dgConnection = deepgram.listen.live({
  encoding: 'mulaw',
  sample_rate: '8000',
  model: 'nova-2',
  punctuate: true,
  interim_results: true,
  utterance_end_ms: 1000,
});

this.finalResult = '';

this.dgConnection.on(LiveTranscriptionEvents.Open, () => {
  this.dgConnection.on(LiveTranscriptionEvents.Transcript, (transcriptionEvent) => {
    
    if (transcriptionEvent.is_final === true && text.trim().length > 0) {
      this.finalResult += ` ${text}`;
      if (transcriptionEvent.speech_final === true) {
        this.emit('transcription', this.finalResult);
        this.finalResult = '';
      }
    }
  });
});

本文标签：