Skip to content

Commit

Permalink
[Live Avatar] update chat sample to make some refinements (#2182)
Browse files Browse the repository at this point in the history 
* [TalkingAvatar] Add sample code for TTS talking avatar real-time API

* sample codes for batch avatar synthesis

* Address repository check failure

* update

* [Avatar] Update real time avatar sample code to support multi-lingual

* [avatar] update real time avatar chat sample to receive GPT response streamingly

* [Live Avatar] update chat sample to make some refinements

---------

Co-authored-by: Yulin Li <[email protected]>
  • Loading branch information
@yinhew
yinhew and Yulin Li authored Dec 19, 2023
1 parent 9ea2566 commit 114559e
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 43 deletions.
2 changes: 1 addition & 1 deletion samples/js/browser/avatar/chat.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ <h2 style="background-color: white; width: 300px;">Avatar Configuration</h2>

<h2 style="background-color: white; width: 400px;">Avatar Conversation Control Panel</h2>
<label for="prompt">System Prompt</label><br/>
<textarea id="prompt">Please provide concise answers, not exceeding 25 words.</textarea>
<textarea id="prompt">You are an AI assistant that helps people find information.</textarea>
<br/>

<button id="startSession" onclick="window.startSession()">Open Video Connection</button>
Expand Down
99 changes: 57 additions & 42 deletions samples/js/browser/avatar/js/chat.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,14 +200,23 @@ function speakNext(text, endingSilenceMs = 0) {
}).catch(
(error) => {
console.log(`Error occurred while speaking the SSML: [ ${error} ]`)

if (spokenTextQueue.length > 0) {
speakNext(spokenTextQueue.shift())
} else {
isSpeaking = false
}
}
)
}

function stopSpeaking() {
spokenTextQueue = []
avatarSynthesizer.stopSpeakingAsync().then(
log("[" + (new Date()).toISOString() + "] Stop speaking request sent.")
() => {
isSpeaking = false
log("[" + (new Date()).toISOString() + "] Stop speaking request sent.")
}
).catch(
(error) => {
console.log("Error occurred while stopping speaking: " + error)
Expand Down Expand Up @@ -318,6 +327,10 @@ window.startMicrophone = () => {

messages.push(chatMessage)
let chatHistoryTextArea = document.getElementById('chatHistory')
if (chatHistoryTextArea.innerHTML !== '' && !chatHistoryTextArea.innerHTML.endsWith('\n\n')) {
chatHistoryTextArea.innerHTML += '\n\n'
}

chatHistoryTextArea.innerHTML += "User: " + userQuery + '\n\n'
chatHistoryTextArea.scrollTop = chatHistoryTextArea.scrollHeight

Expand Down Expand Up @@ -390,60 +403,65 @@ window.startMicrophone = () => {
chunkString = previousChunkString + chunkString
}

if (!chunkString.includes('\n\n')) {
if (!chunkString.endsWith('}}]}\n\n') && !chunkString.endsWith('[DONE]\n\n')) {
// This is a incomplete chunk, read the next chunk
return read(chunkString)
}

chunkString.split('\n\n').forEach((line) => {
if (line.startsWith('data:') && !line.endsWith('[DONE]')) {
const responseJson = JSON.parse(line.substring(5).trim())
let responseToken = undefined
if (dataSources.length === 0) {
responseToken = responseJson.choices[0].delta.content
} else {
let role = responseJson.choices[0].messages[0].delta.role
if (role === 'tool') {
toolContent = responseJson.choices[0].messages[0].delta.content
try {
if (line.startsWith('data:') && !line.endsWith('[DONE]')) {
const responseJson = JSON.parse(line.substring(5).trim())
let responseToken = undefined
if (dataSources.length === 0) {
responseToken = responseJson.choices[0].delta.content
} else {
responseToken = responseJson.choices[0].messages[0].delta.content
if (responseToken !== undefined) {
if (byodDocRegex.test(responseToken)) {
responseToken = responseToken.replace(byodDocRegex, '').trim()
}
let role = responseJson.choices[0].messages[0].delta.role
if (role === 'tool') {
toolContent = responseJson.choices[0].messages[0].delta.content
} else {
responseToken = responseJson.choices[0].messages[0].delta.content
if (responseToken !== undefined) {
if (byodDocRegex.test(responseToken)) {
responseToken = responseToken.replace(byodDocRegex, '').trim()
}

if (responseToken === '[DONE]') {
responseToken = undefined
if (responseToken === '[DONE]') {
responseToken = undefined
}
}
}
}
}

if (responseToken !== undefined) {
assistantReply += responseToken // build up the assistant message
displaySentence += responseToken // build up the display sentence

// console.log(`Current token: ${responseToken}`)

if (responseToken === '\n' || responseToken === '\n\n') {
speak(spokenSentence.trim())
spokenSentence = ''
} else {
responseToken = responseToken.replace(/\n/g, '')
spokenSentence += responseToken // build up the spoken sentence

if (responseToken.length === 1 || responseToken.length === 2) {
for (let i = 0; i < sentenceLevelPunctuations.length; ++i) {
let sentenceLevelPunctuation = sentenceLevelPunctuations[i]
if (responseToken.startsWith(sentenceLevelPunctuation)) {
speak(spokenSentence.trim())
spokenSentence = ''
break
if (responseToken !== undefined) {
assistantReply += responseToken // build up the assistant message
displaySentence += responseToken // build up the display sentence

// console.log(`Current token: ${responseToken}`)

if (responseToken === '\n' || responseToken === '\n\n') {
speak(spokenSentence.trim())
spokenSentence = ''
} else {
responseToken = responseToken.replace(/\n/g, '')
spokenSentence += responseToken // build up the spoken sentence

if (responseToken.length === 1 || responseToken.length === 2) {
for (let i = 0; i < sentenceLevelPunctuations.length; ++i) {
let sentenceLevelPunctuation = sentenceLevelPunctuations[i]
if (responseToken.startsWith(sentenceLevelPunctuation)) {
speak(spokenSentence.trim())
spokenSentence = ''
break
}
}
}
}
}
}
} catch (error) {
console.log(`Error occurred while parsing the response: ${error}`)
console.log(chunkString)
}
})

Expand All @@ -460,9 +478,6 @@ window.startMicrophone = () => {
return read()
})
.then(() => {
let chatHistoryTextArea = document.getElementById('chatHistory')
chatHistoryTextArea.innerHTML += '\n\n'

if (spokenSentence !== '') {
speak(spokenSentence.trim())
spokenSentence = ''
Expand Down

0 comments on commit 114559e

Please sign in to comment.