-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #366 from harmony-one/luma-video
Luma video
- Loading branch information
Showing
8 changed files
with
302 additions
and
3 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| import axios, { AxiosError } from 'axios' | ||
| import { pino } from 'pino' | ||
| import { LumaAI } from 'lumaai' | ||
| import config from '../../../config' | ||
| import { headers } from './helper' | ||
|
|
||
| const logger = pino({ | ||
| name: 'luma - LumaBot', | ||
| transport: { | ||
| target: 'pino-pretty', | ||
| options: { colorize: true } | ||
| } | ||
| }) | ||
|
|
||
| const lumaClient = new LumaAI({ authToken: config.luma.apiKey }) | ||
|
|
||
| const API_ENDPOINT = config.llms.apiEndpoint | ||
|
|
||
| export interface LumaGenerationResponse { | ||
| gnerationId: string | ||
| generationInProgress: string | ||
| queueTime: string | ||
| } | ||
|
|
||
| export const lumaGeneration = async ( | ||
| chatId: number, | ||
| prompt: string, | ||
| loop = true | ||
| ): Promise<LumaGenerationResponse> => { | ||
| logger.info(`Handling luma generation for this prompt: "${prompt}"`) | ||
| const data = { | ||
| chat_id: chatId, | ||
| prompt, | ||
| loop | ||
| } | ||
| const url = `${API_ENDPOINT}/luma/generations` | ||
| const response = await axios.post(url, data, headers) | ||
| const respJson = response.data | ||
| return { | ||
| gnerationId: respJson.generation_id, | ||
| generationInProgress: respJson.in_progress, | ||
| queueTime: respJson.queue_time | ||
| } | ||
| } | ||
|
|
||
| export const getGeneration = async (generationId: string): Promise<LumaAI.Generations.Generation> => { | ||
| const generation = await lumaClient.generations.get(generationId) | ||
| return generation | ||
| } | ||
|
|
||
| export const deleteGeneration = async (generationId: string): Promise<boolean> => { | ||
| try { | ||
| logger.info(`Deleting luma generation ${generationId}`) | ||
| const url = `${API_ENDPOINT}/luma/generations/${generationId}` | ||
| const response = await axios.delete(url, headers) | ||
| if (response.status === 204) { | ||
| logger.info(`Successfully deleted luma generation ${generationId}`) | ||
| return true | ||
| } | ||
| logger.warn(`Unexpected response status ${response.status} when deleting generation ${generationId}`) | ||
| return false | ||
| } catch (e) { | ||
| if (e instanceof AxiosError) { | ||
| const status = e.response?.status | ||
| if (status === 404) { | ||
| logger.warn(`Generation ${generationId} not found`) | ||
| } else if (status === 403) { | ||
| logger.error(`Unauthorized to delete generation ${generationId}`) | ||
| } else { | ||
| logger.error(`Error deleting generation ${generationId}: ${e.message}`) | ||
| } | ||
| } else { | ||
| logger.error(`Unexpected error deleting generation ${generationId}: ${e}`) | ||
| } | ||
| return false | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,167 @@ | ||
| import { type BotPayments } from '../payment' | ||
| import { | ||
| type OnMessageContext, | ||
| type OnCallBackQueryData, | ||
| type ChatConversation, | ||
| RequestState | ||
| } from '../types' | ||
| import { | ||
| MAX_TRIES, | ||
| PRICE_ADJUSTMENT, | ||
| sendMessage | ||
| } from './utils/helpers' | ||
| import * as Sentry from '@sentry/node' | ||
| import { LlmsBase } from './llmsBase' | ||
| import config from '../../config' | ||
| import { now } from '../../utils/perf' | ||
| import { type ModelVersion } from './utils/llmModelsManager' | ||
| import { Callbacks } from './utils/types' | ||
| import { type LlmCompletion } from './api/llmApi' | ||
| import { deleteGeneration, getGeneration, lumaGeneration } from './api/luma' | ||
|
|
||
| interface VideoGeneration { | ||
| msgId: number | ||
| generationId: string | ||
| prompt: string | ||
| } | ||
|
|
||
| export class LumaBot extends LlmsBase { | ||
| private generationList: VideoGeneration[] | ||
| protected supportedCommands: string[] | ||
| protected supportedPrefixes: string[] | ||
|
|
||
| constructor (payments: BotPayments) { | ||
| super(payments, 'LumaBot', 'luma') | ||
| this.generationList = [] | ||
|
|
||
| if (!config.luma.isEnabled) { | ||
| this.logger.warn('Luma AI is disabled in config') | ||
| } | ||
| } | ||
|
|
||
| public getEstimatedPrice (ctx: any): number { | ||
| try { | ||
| // $0.0032 per frame or about $0.4 for 5s 24fps video at 1280×720p | ||
| // price in cents | ||
| return PRICE_ADJUSTMENT ? 40 * PRICE_ADJUSTMENT : 40 * 2 | ||
| } catch (e) { | ||
| Sentry.captureException(e) | ||
| this.logger.error(`getEstimatedPrice error ${e}`) | ||
| throw e | ||
| } | ||
| } | ||
|
|
||
| public isSupportedEvent ( | ||
| ctx: OnMessageContext | OnCallBackQueryData | ||
| ): boolean { | ||
| const hasCommand = ctx.hasCommand(this.supportedCommands) | ||
| const chatPrefix = this.hasPrefix(ctx.message?.text ?? '') | ||
| if (chatPrefix !== '') { | ||
| return true | ||
| } | ||
| return hasCommand || this.isSupportedCallbackQuery(ctx) | ||
| } | ||
|
|
||
| public isSupportedCallbackQuery ( | ||
| ctx: OnMessageContext | OnCallBackQueryData | ||
| ): boolean { | ||
| if (!ctx.callbackQuery?.data) { | ||
| return false | ||
| } | ||
| return ctx.callbackQuery?.data.startsWith(Callbacks.LumaDownloadVideo) | ||
| } | ||
|
|
||
| async chatStreamCompletion ( | ||
| conversation: ChatConversation[], | ||
| model: ModelVersion, | ||
| ctx: OnMessageContext | OnCallBackQueryData, | ||
| msgId: number, | ||
| limitTokens: boolean | ||
| ): Promise<LlmCompletion> { | ||
| throw new Error('chatStreamCompletion is not implemented for LumaAiBot') | ||
| } | ||
|
|
||
| async chatCompletion ( | ||
| conversation: ChatConversation[], | ||
| model: ModelVersion | ||
| ): Promise<LlmCompletion> { | ||
| throw new Error('chatCompletion is not implemented for LumaAiBot') | ||
| } | ||
|
|
||
| public async onEvent ( | ||
| ctx: OnMessageContext | OnCallBackQueryData, | ||
| refundCallback: (reason?: string) => void | ||
| ): Promise<void> { | ||
| ctx.transient.analytics.module = this.module | ||
|
|
||
| const isSupportedEvent = this.isSupportedEvent(ctx) | ||
| if (!isSupportedEvent && ctx.chat?.type !== 'private') { | ||
| this.logger.warn(`### unsupported command ${ctx.message?.text}`) | ||
| return | ||
| } | ||
|
|
||
| if (this.isSupportedCallbackQuery(ctx)) { | ||
| if (ctx.callbackQuery?.data) { | ||
| const data = ctx.callbackQuery.data.split(':') | ||
| await this.onHandleVideoDownload(ctx, data[1]) | ||
| return | ||
| } | ||
| } | ||
|
|
||
| const model = this.getModelFromContext(ctx) | ||
| if (model) { | ||
| await this.onGeneration(ctx) | ||
| return | ||
| } | ||
|
|
||
| ctx.transient.analytics.sessionState = RequestState.Error | ||
| await sendMessage(ctx, '### unsupported command').catch(async (e) => { | ||
| await this.onError(ctx, e, MAX_TRIES, '### unsupported command') | ||
| }) | ||
| ctx.transient.analytics.actualResponseTime = now() | ||
| } | ||
|
|
||
| private async onHandleVideoDownload (ctx: OnMessageContext | OnCallBackQueryData, generationId: string): Promise<void> { | ||
| try { | ||
| const generation = await getGeneration(generationId) | ||
| const videoUrl = generation.assets?.video | ||
| if (videoUrl && ctx.chatId) { | ||
| const videoGeneration = this.generationList.find(gen => gen.generationId === generationId) | ||
| if (videoGeneration) { | ||
| await ctx.api.deleteMessages(ctx.chatId, [ctx.msgId, videoGeneration.msgId]) | ||
| await ctx.replyWithVideo(videoUrl, { caption: videoGeneration.prompt }) | ||
| this.generationList = this.generationList.filter(gen => gen.generationId !== generationId) | ||
| await deleteGeneration(videoGeneration.generationId) | ||
| } | ||
| } | ||
| await ctx.answerCallbackQuery('Video sent successfully') | ||
| } catch (error) { | ||
| console.error('Error in video download:', error) | ||
| await ctx.answerCallbackQuery('Error processing video. Please try again.') | ||
| } | ||
| } | ||
|
|
||
| async onGeneration (ctx: OnMessageContext | OnCallBackQueryData): Promise<void> { | ||
| try { | ||
| const chatId = ctx.chat?.id | ||
| if (chatId) { | ||
| const prompt = ctx.match | ||
| const response = await lumaGeneration(chatId, prompt as string) | ||
| const msgId = ( | ||
| await ctx.reply(`You are #${response.generationInProgress} in line for the video generation. The wait time is about ${response.queueTime} seconds.`, { | ||
| message_thread_id: | ||
| ctx.message?.message_thread_id ?? | ||
| ctx.message?.reply_to_message?.message_thread_id | ||
| }) | ||
| ).message_id | ||
| this.generationList.push({ | ||
| generationId: response.gnerationId, | ||
| msgId, | ||
| prompt: prompt as string | ||
| }) | ||
| } | ||
| } catch (e: any) { | ||
| await this.onError(ctx, e) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters