From a858e267367b0a86400818c70fc398f01d27f778 Mon Sep 17 00:00:00 2001 From: Ovidijus Parsiunas Date: Mon, 13 Jan 2025 23:26:16 +0900 Subject: [PATCH] custom layout for OpenAI Realtime API --- component/src/deepChat.css | 1 + .../src/services/openAI/openAIRealtimeIO.css | 17 +++++++ .../src/services/openAI/openAIRealtimeIO.ts | 48 +++++++++++++++++-- component/src/services/serviceIO.ts | 2 + component/src/services/utils/baseServiceIO.ts | 4 ++ component/src/types/openAI.ts | 15 +++++- component/src/views/chat/chatView.ts | 2 + 7 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 component/src/services/openAI/openAIRealtimeIO.css diff --git a/component/src/deepChat.css b/component/src/deepChat.css index 33b59d204..f1202ab2f 100644 --- a/component/src/deepChat.css +++ b/component/src/deepChat.css @@ -1,4 +1,5 @@ @import url('./views/validateKeyProperty/validateKeyPropertyView.css'); +@import url('./services/openAI/openAIRealtimeIO.css'); @import url('./views/insertKey/insertKeyView.css'); @import url('./views/error/errorView.css'); @import url('./utils/loading/loading.css'); diff --git a/component/src/services/openAI/openAIRealtimeIO.css b/component/src/services/openAI/openAIRealtimeIO.css new file mode 100644 index 000000000..1abb14d3c --- /dev/null +++ b/component/src/services/openAI/openAIRealtimeIO.css @@ -0,0 +1,17 @@ +#deep-chat-openai-realtime-container { + height: 100%; + width: 100%; +} + +#deep-chat-openai-realtime-avatar-container { + height: 70%; + width: 100%; + display: flex; + justify-content: center; + align-items: center; +} + +#deep-chat-openai-realtime-avatar { + border-radius: 50%; + height: 120px; +} diff --git a/component/src/services/openAI/openAIRealtimeIO.ts b/component/src/services/openAI/openAIRealtimeIO.ts index edcca50e3..359997922 100644 --- a/component/src/services/openAI/openAIRealtimeIO.ts +++ b/component/src/services/openAI/openAIRealtimeIO.ts @@ -1,9 +1,13 @@ +import {ChatFunctionHandler, OpenAIRealTime} from '../../types/openAI'; import {DirectConnection} from '../../types/directConnection'; +import avatarUrl from '../../../assets/person-avatar.png'; import {DirectServiceIO} from '../utils/directServiceIO'; -import {ChatFunctionHandler} from '../../types/openAI'; import {OpenAIUtils} from './utils/openAIUtils'; +import {APIKey} from '../../types/APIKey'; import {DeepChat} from '../../deepChat'; +// https://platform.openai.com/docs/guides/realtime-webrtc +// https://platform.openai.com/docs/api-reference/realtime-server-events/conversation export class OpenAIRealtimeIO extends DirectServiceIO { override insertKeyPlaceholderText = 'OpenAI API Key'; override keyHelpUrl = 'https://platform.openai.com/account/api-keys'; @@ -11,16 +15,52 @@ export class OpenAIRealtimeIO extends DirectServiceIO { permittedErrorPrefixes = ['Incorrect']; _functionHandler?: ChatFunctionHandler; asyncCallInProgress = false; // used when streaming tools + private readonly _avatarConfig: OpenAIRealTime['avatar']; constructor(deepChat: DeepChat) { const directConnectionCopy = JSON.parse(JSON.stringify(deepChat.directConnection)) as DirectConnection; - const apiKey = directConnectionCopy.openAI; - super(deepChat, OpenAIUtils.buildKeyVerificationDetails(), OpenAIUtils.buildHeaders, apiKey); - this.maxMessages ??= -1; + const {key} = directConnectionCopy.openAI as APIKey; + super(deepChat, OpenAIUtils.buildKeyVerificationDetails(), OpenAIUtils.buildHeaders, {key: key || 'asdsd'}); + const config = directConnectionCopy.openAI?.realtime as OpenAIRealTime; + if (typeof config === 'object') { + if (config.avatar) this._avatarConfig = config.avatar; + } this.rawBody.model ??= 'gpt-4o'; this.init(); } + public setUpView(containerElement: HTMLElement, parentElement: HTMLElement) { + containerElement.style.display = 'none'; + parentElement.appendChild(this.createContainer()); + } + + private createContainer() { + const container = document.createElement('div'); + container.id = 'deep-chat-openai-realtime-container'; + container.appendChild(this.avatarContainer()); + return container; + } + + private avatarContainer() { + const avatarContainer = document.createElement('div'); + avatarContainer.id = 'deep-chat-openai-realtime-avatar-container'; + Object.assign(avatarContainer.style, this._avatarConfig?.styles?.container); + avatarContainer.appendChild(this.createAvatar()); + return avatarContainer; + } + + private createAvatar() { + const avatar = document.createElement('img'); + avatar.id = 'deep-chat-openai-realtime-avatar'; + Object.assign(avatar.style, this._avatarConfig?.styles?.avatar); + avatar.src = this._avatarConfig?.src || avatarUrl; + return avatar; + } + + override isCustomView() { + return true; + } + private async init() { // Get an ephemeral key from your server - see server code below // const tokenResponse = await fetch('/session'); diff --git a/component/src/services/serviceIO.ts b/component/src/services/serviceIO.ts index 13456d4ee..b009d5305 100644 --- a/component/src/services/serviceIO.ts +++ b/component/src/services/serviceIO.ts @@ -108,6 +108,8 @@ export interface ServiceIO { isWebModel(): boolean; + isCustomView(): boolean; + isSubmitProgrammaticallyDisabled?: boolean; sessionId?: string; diff --git a/component/src/services/utils/baseServiceIO.ts b/component/src/services/utils/baseServiceIO.ts index 5b4435bed..02bb2536d 100644 --- a/component/src/services/utils/baseServiceIO.ts +++ b/component/src/services/utils/baseServiceIO.ts @@ -156,4 +156,8 @@ export class BaseServiceIO implements ServiceIO { public isWebModel() { return false; } + + public isCustomView() { + return false; + } } diff --git a/component/src/types/openAI.ts b/component/src/types/openAI.ts index ae1e44bb3..b1b5bc21c 100644 --- a/component/src/types/openAI.ts +++ b/component/src/types/openAI.ts @@ -1,3 +1,5 @@ +import {CustomStyle} from './styles'; + // https://platform.openai.com/docs/api-reference/audio/createSpeech export type OpenAITextToSpeech = { model?: string; @@ -34,6 +36,17 @@ export interface OpenAIImagesDalle3 { user?: string; } +// https://platform.openai.com/docs/api-reference/realtime +export type OpenAIRealTime = { + avatar?: { + src?: string; + styles?: { + avatar?: CustomStyle; + container?: CustomStyle; + }; + }; +}; + export type FunctionsDetails = {name: string; arguments: string}[]; export type AssistantFunctionHandlerResponse = @@ -104,7 +117,7 @@ export type OpenAIChat = { export interface OpenAI { chat?: true | OpenAIChat; assistant?: true | OpenAIAssistant; - realtime?: true; + realtime?: true | OpenAIRealTime; images?: true | OpenAIImagesDalle2 | OpenAIImagesDalle3; textToSpeech?: true | OpenAITextToSpeech; speechToText?: true | OpenAISpeechToText; diff --git a/component/src/views/chat/chatView.ts b/component/src/views/chat/chatView.ts index 804a59023..e5958553b 100644 --- a/component/src/views/chat/chatView.ts +++ b/component/src/views/chat/chatView.ts @@ -1,3 +1,4 @@ +import {OpenAIRealtimeIO} from '../../services/openAI/openAIRealtimeIO'; import {ElementUtils} from '../../utils/element/elementUtils'; import {Websocket} from '../../utils/HTTP/websocket'; import {ServiceIO} from '../../services/serviceIO'; @@ -19,5 +20,6 @@ export class ChatView { public static render(deepChat: DeepChat, containerRef: HTMLElement, serviceIO: ServiceIO, panel?: HTMLElement) { const containerElement = ChatView.createElements(deepChat, serviceIO, panel); containerRef.replaceChildren(containerElement); + if (serviceIO.isCustomView()) (serviceIO as OpenAIRealtimeIO).setUpView(containerElement, containerRef); } }