Client side integration
This example includes server and client applications.
Server application is built using express and ws.
Client application allows capturing sounds from the browser and playing character responses. It is build using React.
Find example source code here.
Installation
Set up variables in the .env file for the server application
Name | Description | Details |
---|---|---|
INWORLD_KEY | Inworld application key | Get key from integrations page |
INWORLD_SECRET | Inworld application secret | Get secret from integrations page |
Setup environment variables for client application
Please specify the values for both REACT_APP_INWORLD_CHARACTER and REACT_APP_INWORLD_SCENE environment variables, either by setting them directly in your system or by filling in the corresponding fields on the web form provided after launching the application.
Install dependencies for both applications
yarn install
Start applications
yarn start
How it works
Client application
1. Open connection and attach handlers to WebSocket
// Use a unique key here.
// We need a key to differentiate connections and conversation context between messages.
const key = v4();
const { character, player, scene } = formMethods.getValues();
// 1. Load character info: name, assets.
const response = await fetch(`${config.LOAD_URL}?key=${key}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
scene: scene?.name,
player: player?.name,
character: character?.name,
}),
});
const data = await response.json();
if (!response.ok) {
return console.log(response.statusText, ': ', data.errors);
};
if (data.character) {
setCharacter(data.character as Character);
}
// 2. Open WebSocket connection.
const ws = new WebSocket(`${config.SESSION_URL}?key=${key}`);
// 3. Attach handlers.
ws.addEventListener('open', () => console.log('Open!'));
ws.addEventListener('message', (message: MessageEvent) => console.log(message));
ws.addEventListener('disconnect', () => console.log('Disconnect!'));
2. Close connection and remove handlers
// Stop audio playing.
player.stop();
// Clear history.
setChatHistory([]);
// Close connection.
ws.close();
ws.removeEventListener('open', onOpen);
ws.removeEventListener('message', onMessage);
ws.removeEventListener('disconnect', onDisconnect);
3. onMessage
handler
const onOpen = () => {
console.log('Open!');
}
const onDisconnect = () => {
console.log('Disconnect!');
};
const onMessage = (message: MessageEvent) => {
const packet = JSON.parse(message.data);
let chatItem: ChatHistoryItem | undefined = undefined;
if (packet?.type === 'AUDIO') {
player.addToQueue(packet.audio?.chunk);
} else if (packet?.type === 'TEXT') {
const { character, playerName } = stateRef.current || {};
chatItem = {
id: packet.packetId?.utteranceId,
type: CHAT_HISTORY_TYPE.TEXT,
date: new Date(packet.date!),
source: packet.routing?.source,
text: packet.text.text,
interactionId: packet.packetId?.interactionId,
isRecognizing: !packet.text.final,
author: packet.routing!.source!.isCharacter
? character?.displayName
: playerName,
};
} else if (packet?.control?.type === 'INTERACTION_END') {
chatItem = {
id: packet.packetId?.utteranceId,
type: CHAT_HISTORY_TYPE.INTERACTION_END,
date: new Date(packet.date!),
interactionId: packet.packetId?.interactionId
};
}
if (chatItem) {
setChatHistory((currentState) => {
let newState = undefined;
let currentHistoryIndex = currentState.findIndex((item) => {
return item.id === chatItem?.id;
});
if (currentHistoryIndex >= 0 && chatItem) {
newState = [...currentState];
newState[currentHistoryIndex] = chatItem;
} else {
newState = [...currentState, chatItem!];
}
return newState;
});
}
};
4. How to capture audio
let interval: NodeJS.Timeout;
const arrayBufferToBase64 = (buffer: ArrayBuffer) => {
let binary = '';
const bytes = new Uint8Array(buffer);
const length = bytes.byteLength;
for (let i = 0; i < length; i++) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary);
}
const mergeBuffers = (channelBuffer: Float32Array[], recordingLength: number) => {
const result = new Float32Array(recordingLength);
let offset = 0;
for (let i = 0; i < channelBuffer.length; i++) {
result.set(channelBuffer[i], offset);
offset += channelBuffer[i].length;
}
return Array.prototype.slice.call(result);
}
const startRecording = async () => {
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
sampleRate: 16000,
echoCancellation: { ideal: true },
suppressLocalAudioPlayback: { ideal: true },
},
video: false,
});
const audioCtx = new AudioContext({
sampleRate: 16000,
});
const source = audioCtx.createMediaStreamSource(stream);
const scriptNode = audioCtx.createScriptProcessor(2048, 1, 1);
let leftChannel: Float32Array[] = [];
let recordingLength = 0;
scriptNode.onaudioprocess = (audioProcessingEvent) => {
const samples = audioProcessingEvent.inputBuffer.getChannelData(0);
leftChannel.push(new Float32Array(samples));
recordingLength += 2048;
};
source.connect(scriptNode);
scriptNode.connect(audioCtx.destination);
interval = setInterval(() => {
const PCM16iSamples = Int16Array.from(
mergeBuffers(
leftChannel,
recordingLength,
),
(k) => 32767 * Math.min(1, k),
);
ws.send(JSON.stringify({
type: 'audio',
audio: arrayBufferToBase64(PCM16iSamples.buffer)
}));
//clear buffer
leftChannel = [];
recordingLength = 0;
}, 200);
};
const stopRecording = () => {
clearInterval(interval);
ws.send(JSON.stringify({ type: 'audioSessionEnd' }));
};
5. How to play audio
interface PlayerProps {
audio: HTMLAudioElement;
}
export class Player {
private audioPacketQueue: string[] = [];
private isPlaying = false;
private audioElement!: HTMLAudioElement;
preparePlayer(props: PlayerProps): void {
this.audioElement = props.audio;
this.audioElement.onended = () => {
this.playQueue();
};
}
getIsPlaying(): boolean {
return this.isPlaying;
}
stop() {
this.audioElement.pause();
this.audioElement.currentTime = 0;
}
addToQueue(packet: string): void {
this.audioPacketQueue.push(packet);
if (!this.isPlaying) {
this.playQueue();
}
}
clearQueue() {
this.isPlaying = false;
this.audioPacketQueue = [];
}
private playQueue = (): void => {
if (!this.audioPacketQueue.length) {
this.isPlaying = false;
this.audioElement.src = '';
return;
}
const currentPacket = this.audioPacketQueue.shift();
this.isPlaying = true;
this.audioElement.src = 'data:audio/wav;base64,' + currentPacket;
this.audioElement.play().catch((e) => console.error(e));
};
}
Server application
1. Load character info
import cors from 'cors';
import express from 'express';
import {
Character,
InworldClient,
} from '@inworld/nodejs-sdk';
import { createClient as createRedisClient } from 'redis';
const app = express();
app.use(cors());
app.use(express.json());
app.post(
'/load',
async (req, res) => {
res.setHeader('Content-Type', 'application/json');
await redisClient.set(req.query.key?.toString()!, JSON.stringify(req.body));
const connection = new InworldClient()
.setApiKey({
key: process.env.INWORLD_KEY!,
secret: process.env.INWORLD_SECRET!,
})
.setScene(req.body.scene)
.build();
const characters = await connection.getCharacters();
const character = characters.find(
(c: Character) => c.getResourceName() === req.body.character,
);
res.end(JSON.stringify({ character }));
},
);
2. Receive packets from UI and send them to Inworld server
server.on('upgrade', async (request, socket, head) => {
const { pathname } = parse(request.url!);
if (pathname === '/session') {
webSocket.handleUpgrade(request, socket, head, (ws) => {
webSocket.emit('connection', ws, request);
});
} else {
socket.destroy();
}
});
const connections: Connections = {};
const sent: SentState = {};
webSocket.on('connection', (ws, request) => {
const { query } = parse(request.url!, true);
const key = query.key?.toString();
if (!key) throw Error('Key is not provided!');
ws.on('error', console.error);
ws.on('message', async (data: RawData) => {
const {
player,
scene,
session: savedSession,
} = await storage.get(key);
const message = JSON.parse(data.toString());
if (!connections[key]) {
const client = new InworldClient()
.setConfiguration({
connection: { disconnectTimeout: DISCONNECT_TIMEOUT },
})
.setApiKey({
key: process.env.INWORLD_KEY!,
secret: process.env.INWORLD_SECRET!,
})
.setOnSession({
get: () => ({
sessionToken: savedSession?.sessionToken,
scene: savedSession?.scene,
}),
set: (session: Session) =>
storage.set(key, { player, scene, session }),
})
.setOnMessage((packet: InworldPacket) => {
ws.send(JSON.stringify(packet));
if (packet.isInteractionEnd()) {
connections[key].close();
}
})
.setOnError(handleError(key))
.setOnDisconnect(() => {
delete connections[key];
delete sent[key];
});
if (settings.player) {
client.setUser({ fullName: settings.player });
}
if (settings.scene) {
client.setScene(settings.scene);
}
connections[key] = client.build();
}
switch (message.type) {
case EVENT_TYPE.TEXT:
ws.send(JSON.stringify(await connections[key].sendText(message.text)));
break;
case EVENT_TYPE.AUDIO:
// Start audio session before send audio.
// It will be called after each disconnected initiated from client or server.
if (sent[key] === AUDIO_SESSION_STATE.ACTIVE) {
connections[key].sendAudio(message.audio);
} else {
sent[key] = AUDIO_SESSION_STATE.PROCESSING;
await connections[key].sendAudioSessionStart();
sent[key] = AUDIO_SESSION_STATE.ACTIVE;
}
break;
case EVENT_TYPE.AUDIO_SESSION_END:
delete sent[key];
connections[key].sendAudioSessionEnd();
break;
}
});
});
3. Define Redis storage class
import { Session } from '@inworld/nodejs-sdk';
import { createClient } from 'redis';
interface StorageValue {
session: string;
player?: string;
scene?: string;
}
interface StorageRecord {
session?: Session;
player?: string;
scene?: string;
}
export class Storage {
private redisClient = createClient();
async connect({ onError }: { onError?: (err: Error) => void }) {
await this.redisClient.connect();
if (onError) {
this.redisClient.on('error', onError);
}
}
disconnect() {
this.redisClient.disconnect();
}
async get(key: string) {
const json = await this.redisClient.get(key);
if (!json) return {};
let parsed: StorageValue;
let session: Session;
try {
parsed = JSON.parse(json) as StorageValue;
if (parsed.session) {
session = Session.deserialize(parsed.session);
}
} catch (e) {}
return {
session,
player: parsed.player,
scene: parsed.scene,
} as StorageRecord;
}
set(key: string, entity: StorageRecord) {
return this.redisClient.set(
key,
JSON.stringify({
session: Session.serialize(entity.session),
player: entity.player,
scene: entity.scene,
}),
);
}
delete(key: string) {
this.redisClient.del(key);
}
}