Client side integration

This example includes server and client applications.

Server application is built using express and ws.

Client application allows capturing sounds from the browser and playing character responses. It is build using React.

Find example source code here.

Installation

Set up variables in the .env file for the server application

Name	Description	Details
INWORLD_KEY	Inworld application key	Get key from integrations page
INWORLD_SECRET	Inworld application secret	Get secret from integrations page

Setup environment variables for client application

Please specify the values for both REACT_APP_INWORLD_CHARACTER and REACT_APP_INWORLD_SCENE environment variables, either by setting them directly in your system or by filling in the corresponding fields on the web form provided after launching the application.

Install dependencies for both applications

yarn install

Start applications

yarn start

How it works

Client application

1. Open connection and attach handlers to WebSocket

// Use a unique key here.
// We need a key to differentiate connections and conversation context between messages.
const key = v4();
const { character, player, scene } = formMethods.getValues();

// 1. Load character info: name, assets.
const response = await fetch(`${config.LOAD_URL}?key=${key}`, {
  method: 'POST',
  headers: { 'Content-Type': 'application/json' },
  body: JSON.stringify({
    scene: scene?.name,
    player: player?.name,
    character: character?.name,
  }),
});
const data = await response.json();

if (!response.ok) {
  return console.log(response.statusText, ': ', data.errors);
};

if (data.character) {
  setCharacter(data.character as Character);
}

// 2. Open WebSocket connection.
const ws = new WebSocket(`${config.SESSION_URL}?key=${key}`);

// 3. Attach handlers.
ws.addEventListener('open', () => console.log('Open!'));
ws.addEventListener('message', (message: MessageEvent) => console.log(message));
ws.addEventListener('disconnect', () => console.log('Disconnect!'));

2. Close connection and remove handlers

// Stop audio playing.
player.stop();

// Clear history.
setChatHistory([]);

// Close connection.
ws.close();
ws.removeEventListener('open', onOpen);
ws.removeEventListener('message', onMessage);
ws.removeEventListener('disconnect', onDisconnect);

3. onMessage handler

const onOpen = () => {
  console.log('Open!');
}

const onDisconnect = () => {
  console.log('Disconnect!');
};

const onMessage = (message: MessageEvent) => {
  const packet = JSON.parse(message.data);

  let chatItem: ChatHistoryItem | undefined = undefined;

  if (packet?.type === 'AUDIO') {
    player.addToQueue(packet.audio?.chunk);
  } else if (packet?.type === 'TEXT') {
    const { character, playerName } = stateRef.current || {};

    chatItem = {
      id: packet.packetId?.utteranceId,
      type: CHAT_HISTORY_TYPE.TEXT,
      date: new Date(packet.date!),
      source: packet.routing?.source,
      text: packet.text.text,
      interactionId: packet.packetId?.interactionId,
      isRecognizing: !packet.text.final,
      author: packet.routing!.source!.isCharacter
        ? character?.displayName
        : playerName,
    };
  } else if (packet?.control?.type === 'INTERACTION_END') {
    chatItem = {
      id: packet.packetId?.utteranceId,
      type: CHAT_HISTORY_TYPE.INTERACTION_END,
      date: new Date(packet.date!),
      interactionId: packet.packetId?.interactionId
    };
  }

  if (chatItem) {
    setChatHistory((currentState) => {
      let newState = undefined;
      let currentHistoryIndex = currentState.findIndex((item) => {
        return item.id === chatItem?.id;
      });

      if (currentHistoryIndex >= 0 && chatItem) {
        newState = [...currentState];
        newState[currentHistoryIndex] = chatItem;
      } else {
        newState = [...currentState, chatItem!];
      }
      return newState;
    });
  }
};

4. How to capture audio

let interval: NodeJS.Timeout;

const arrayBufferToBase64 = (buffer: ArrayBuffer) => {
  let binary = '';
  const bytes = new Uint8Array(buffer);
  const length = bytes.byteLength;
  for (let i = 0; i < length; i++) {
    binary += String.fromCharCode(bytes[i]);
  }
  return window.btoa(binary);
}
  
const mergeBuffers = (channelBuffer: Float32Array[], recordingLength: number) => {
  const result = new Float32Array(recordingLength);
  let offset = 0;

  for (let i = 0; i < channelBuffer.length; i++) {
    result.set(channelBuffer[i], offset);
    offset += channelBuffer[i].length;
  }

  return Array.prototype.slice.call(result);
}

const startRecording = async () => {
  const stream = await navigator.mediaDevices.getUserMedia({
    audio: {
      sampleRate: 16000,
      echoCancellation: { ideal: true },
      suppressLocalAudioPlayback: { ideal: true },
    },
    video: false,
  });
  const audioCtx = new AudioContext({
    sampleRate: 16000,
  });
  const source = audioCtx.createMediaStreamSource(stream);
  const scriptNode = audioCtx.createScriptProcessor(2048, 1, 1);
  let leftChannel: Float32Array[] = [];
  let recordingLength = 0;

  scriptNode.onaudioprocess = (audioProcessingEvent) => {
    const samples = audioProcessingEvent.inputBuffer.getChannelData(0);
    leftChannel.push(new Float32Array(samples));
    recordingLength += 2048;
  };

  source.connect(scriptNode);
  scriptNode.connect(audioCtx.destination);
  
  interval = setInterval(() => {
    const PCM16iSamples = Int16Array.from(
      mergeBuffers(
        leftChannel,
        recordingLength,
      ),
      (k) => 32767 * Math.min(1, k),
    );

    ws.send(JSON.stringify({
      type: 'audio',
      audio: arrayBufferToBase64(PCM16iSamples.buffer)
    }));
    //clear buffer
    leftChannel = [];
    recordingLength = 0;
  }, 200);
};

const stopRecording = () => {
  clearInterval(interval);
  ws.send(JSON.stringify({ type: 'audioSessionEnd' }));
};

5. How to play audio

interface PlayerProps {
  audio: HTMLAudioElement;
}

export class Player {
  private audioPacketQueue: string[] = [];
  private isPlaying = false;
  private audioElement!: HTMLAudioElement;

  preparePlayer(props: PlayerProps): void {
    this.audioElement = props.audio;
    this.audioElement.onended = () => {
      this.playQueue();
    };
  }

  getIsPlaying(): boolean {
    return this.isPlaying;
  }

  stop() {
    this.audioElement.pause();
    this.audioElement.currentTime = 0;
  }

  addToQueue(packet: string): void {
    this.audioPacketQueue.push(packet);
    if (!this.isPlaying) {
      this.playQueue();
    }
  }

  clearQueue() {
    this.isPlaying = false;
    this.audioPacketQueue = [];
  }

  private playQueue = (): void => {
    if (!this.audioPacketQueue.length) {
      this.isPlaying = false;
      this.audioElement.src = '';
      return;
    }

    const currentPacket = this.audioPacketQueue.shift();

    this.isPlaying = true;
    this.audioElement.src = 'data:audio/wav;base64,' + currentPacket;
    this.audioElement.play().catch((e) => console.error(e));
  };
}

Server application

1. Load character info

import cors from 'cors';
import express from 'express';

import {
  Character,
  InworldClient,
} from '@inworld/nodejs-sdk';
import { createClient as createRedisClient } from 'redis';

const app = express();

app.use(cors());
app.use(express.json());

app.post(
  '/load',
  async (req, res) => {
    res.setHeader('Content-Type', 'application/json');

    await redisClient.set(req.query.key?.toString()!, JSON.stringify(req.body));

    const connection = new InworldClient()
      .setApiKey({
        key: process.env.INWORLD_KEY!,
        secret: process.env.INWORLD_SECRET!,
      })
      .setScene(req.body.scene)
      .build();

    const characters = await connection.getCharacters();
    const character = characters.find(
      (c: Character) => c.getResourceName() === req.body.character,
    );

    res.end(JSON.stringify({ character }));
  },
);

2. Receive packets from UI and send them to Inworld server

server.on('upgrade', async (request, socket, head) => {
  const { pathname } = parse(request.url!);

  if (pathname === '/session') {
    webSocket.handleUpgrade(request, socket, head, (ws) => {
      webSocket.emit('connection', ws, request);
    });
  } else {
    socket.destroy();
  }
});

const connections: Connections = {};
const sent: SentState = {};

webSocket.on('connection', (ws, request) => {
  const { query } = parse(request.url!, true);
  const key = query.key?.toString();

  if (!key) throw Error('Key is not provided!');

  ws.on('error', console.error);

  ws.on('message', async (data: RawData) => {
    const {
      player,
      scene,
      session: savedSession,
    } = await storage.get(key);

    const message = JSON.parse(data.toString());

    if (!connections[key]) {
      const client = new InworldClient()
        .setConfiguration({
          connection: { disconnectTimeout: DISCONNECT_TIMEOUT },
        })
        .setApiKey({
          key: process.env.INWORLD_KEY!,
          secret: process.env.INWORLD_SECRET!,
        })
        .setOnSession({
          get: () => ({
            sessionToken: savedSession?.sessionToken,
            scene: savedSession?.scene,
          }),
          set: (session: Session) =>
            storage.set(key, { player, scene, session }),
        })
        .setOnMessage((packet: InworldPacket) => {
          ws.send(JSON.stringify(packet));

          if (packet.isInteractionEnd()) {
            connections[key].close();
          }
        })
        .setOnError(handleError(key))
        .setOnDisconnect(() => {
          delete connections[key];
          delete sent[key];
        });

      if (settings.player) {
        client.setUser({ fullName: settings.player });
      }

      if (settings.scene) {
        client.setScene(settings.scene);
      }

      connections[key] = client.build();
    }

    switch (message.type) {
      case EVENT_TYPE.TEXT:
        ws.send(JSON.stringify(await connections[key].sendText(message.text)));
        break;
      case EVENT_TYPE.AUDIO:
        // Start audio session before send audio.
        // It will be called after each disconnected initiated from client or server.
        if (sent[key] === AUDIO_SESSION_STATE.ACTIVE) {
          connections[key].sendAudio(message.audio);
        } else {
          sent[key] = AUDIO_SESSION_STATE.PROCESSING;
          await connections[key].sendAudioSessionStart();
          sent[key] = AUDIO_SESSION_STATE.ACTIVE;
        }
        break;
      case EVENT_TYPE.AUDIO_SESSION_END:
        delete sent[key];
        connections[key].sendAudioSessionEnd();
        break;
    }
  });
});

3. Define Redis storage class

import { Session } from '@inworld/nodejs-sdk';
import { createClient } from 'redis';

interface StorageValue {
  session: string;
  player?: string;
  scene?: string;
}

interface StorageRecord {
  session?: Session;
  player?: string;
  scene?: string;
}

export class Storage {
  private redisClient = createClient();

  async connect({ onError }: { onError?: (err: Error) => void }) {
    await this.redisClient.connect();

    if (onError) {
      this.redisClient.on('error', onError);
    }
  }

  disconnect() {
    this.redisClient.disconnect();
  }

  async get(key: string) {
    const json = await this.redisClient.get(key);

    if (!json) return {};

    let parsed: StorageValue;
    let session: Session;

    try {
      parsed = JSON.parse(json) as StorageValue;

      if (parsed.session) {
        session = Session.deserialize(parsed.session);
      }
    } catch (e) {}

    return {
      session,
      player: parsed.player,
      scene: parsed.scene,
    } as StorageRecord;
  }

  set(key: string, entity: StorageRecord) {
    return this.redisClient.set(
      key,
      JSON.stringify({
        session: Session.serialize(entity.session),
        player: entity.player,
        scene: entity.scene,
      }),
    );
  }

  delete(key: string) {
    this.redisClient.del(key);
  }
}