custom layout for OpenAI Realtime API

component/src/deepChat.css

@@ -1,4 +1,5 @@
 @import url('./views/validateKeyProperty/validateKeyPropertyView.css');
+@import url('./services/openAI/openAIRealtimeIO.css');
 @import url('./views/insertKey/insertKeyView.css');
 @import url('./views/error/errorView.css');
 @import url('./utils/loading/loading.css');

component/src/services/openAI/openAIRealtimeIO.css

@@ -0,0 +1,17 @@
+#deep-chat-openai-realtime-container {
+  height: 100%;
+  width: 100%;
+}
+
+#deep-chat-openai-realtime-avatar-container {
+  height: 70%;
+  width: 100%;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}
+
+#deep-chat-openai-realtime-avatar {
+  border-radius: 50%;
+  height: 120px;
+}

component/src/services/openAI/openAIRealtimeIO.ts

@@ -1,26 +1,66 @@
+import {ChatFunctionHandler, OpenAIRealTime} from '../../types/openAI';
 import {DirectConnection} from '../../types/directConnection';
+import avatarUrl from '../../../assets/person-avatar.png';
 import {DirectServiceIO} from '../utils/directServiceIO';
-import {ChatFunctionHandler} from '../../types/openAI';
 import {OpenAIUtils} from './utils/openAIUtils';
+import {APIKey} from '../../types/APIKey';
 import {DeepChat} from '../../deepChat';
 
+// https://platform.openai.com/docs/guides/realtime-webrtc
+// https://platform.openai.com/docs/api-reference/realtime-server-events/conversation
 export class OpenAIRealtimeIO extends DirectServiceIO {
   override insertKeyPlaceholderText = 'OpenAI API Key';
   override keyHelpUrl = 'https://platform.openai.com/account/api-keys';
   url = 'https://api.openai.com/v1/chat/completions';
   permittedErrorPrefixes = ['Incorrect'];
   _functionHandler?: ChatFunctionHandler;
   asyncCallInProgress = false; // used when streaming tools
+  private readonly _avatarConfig: OpenAIRealTime['avatar'];
 
   constructor(deepChat: DeepChat) {
     const directConnectionCopy = JSON.parse(JSON.stringify(deepChat.directConnection)) as DirectConnection;
-    const apiKey = directConnectionCopy.openAI;
-    super(deepChat, OpenAIUtils.buildKeyVerificationDetails(), OpenAIUtils.buildHeaders, apiKey);
-    this.maxMessages ??= -1;
+    const {key} = directConnectionCopy.openAI as APIKey;
+    super(deepChat, OpenAIUtils.buildKeyVerificationDetails(), OpenAIUtils.buildHeaders, {key: key || 'asdsd'});
+    const config = directConnectionCopy.openAI?.realtime as OpenAIRealTime;
+    if (typeof config === 'object') {
+      if (config.avatar) this._avatarConfig = config.avatar;
+    }
     this.rawBody.model ??= 'gpt-4o';
     this.init();
   }
 
+  public setUpView(containerElement: HTMLElement, parentElement: HTMLElement) {
+    containerElement.style.display = 'none';
+    parentElement.appendChild(this.createContainer());
+  }
+
+  private createContainer() {
+    const container = document.createElement('div');
+    container.id = 'deep-chat-openai-realtime-container';
+    container.appendChild(this.avatarContainer());
+    return container;
+  }
+
+  private avatarContainer() {
+    const avatarContainer = document.createElement('div');
+    avatarContainer.id = 'deep-chat-openai-realtime-avatar-container';
+    Object.assign(avatarContainer.style, this._avatarConfig?.styles?.container);
+    avatarContainer.appendChild(this.createAvatar());
+    return avatarContainer;
+  }
+
+  private createAvatar() {
+    const avatar = document.createElement('img');
+    avatar.id = 'deep-chat-openai-realtime-avatar';
+    Object.assign(avatar.style, this._avatarConfig?.styles?.avatar);
+    avatar.src = this._avatarConfig?.src || avatarUrl;
+    return avatar;
+  }
+
+  override isCustomView() {
+    return true;
+  }
+
   private async init() {
     // Get an ephemeral key from your server - see server code below
     // const tokenResponse = await fetch('/session');

component/src/services/serviceIO.ts

@@ -108,6 +108,8 @@ export interface ServiceIO {
 
   isWebModel(): boolean;
 
+  isCustomView(): boolean;
+
   isSubmitProgrammaticallyDisabled?: boolean;
 
   sessionId?: string;

component/src/services/utils/baseServiceIO.ts

@@ -156,4 +156,8 @@ export class BaseServiceIO implements ServiceIO {
   public isWebModel() {
     return false;
   }
+
+  public isCustomView() {
+    return false;
+  }
 }

component/src/types/openAI.ts

@@ -1,3 +1,5 @@
+import {CustomStyle} from './styles';
+
 // https://platform.openai.com/docs/api-reference/audio/createSpeech
 export type OpenAITextToSpeech = {
   model?: string;
@@ -34,6 +36,17 @@ export interface OpenAIImagesDalle3 {
   user?: string;
 }
 
+// https://platform.openai.com/docs/api-reference/realtime
+export type OpenAIRealTime = {
+  avatar?: {
+    src?: string;
+    styles?: {
+      avatar?: CustomStyle;
+      container?: CustomStyle;
+    };
+  };
+};
+
 export type FunctionsDetails = {name: string; arguments: string}[];
 
 export type AssistantFunctionHandlerResponse =
@@ -104,7 +117,7 @@ export type OpenAIChat = {
 export interface OpenAI {
   chat?: true | OpenAIChat;
   assistant?: true | OpenAIAssistant;
-  realtime?: true;
+  realtime?: true | OpenAIRealTime;
   images?: true | OpenAIImagesDalle2 | OpenAIImagesDalle3;
   textToSpeech?: true | OpenAITextToSpeech;
   speechToText?: true | OpenAISpeechToText;

component/src/views/chat/chatView.ts

@@ -1,3 +1,4 @@
+import {OpenAIRealtimeIO} from '../../services/openAI/openAIRealtimeIO';
 import {ElementUtils} from '../../utils/element/elementUtils';
 import {Websocket} from '../../utils/HTTP/websocket';
 import {ServiceIO} from '../../services/serviceIO';
@@ -19,5 +20,6 @@ export class ChatView {
   public static render(deepChat: DeepChat, containerRef: HTMLElement, serviceIO: ServiceIO, panel?: HTMLElement) {
     const containerElement = ChatView.createElements(deepChat, serviceIO, panel);
     containerRef.replaceChildren(containerElement);
+    if (serviceIO.isCustomView()) (serviceIO as OpenAIRealtimeIO).setUpView(containerElement, containerRef);
   }
 }