work on LumaLabs integration

package.json

@@ -41,5 +41,8 @@
     "packages/broadway",
     "packages/clapper-services",
     "packages/app"
-  ]
+  ],
+  "dependencies": {
+    "lumaai": "^1.0.2"
+  }
 }

packages/app/src/app/api/resolve/providers/lumalabs/index.ts

@@ -0,0 +1,116 @@
+import LumaAI from 'lumaai'
+
+import { ClapImageRatio, ClapSegmentCategory } from '@aitube/clap'
+import { ResolveRequest } from '@aitube/clapper-services'
+import { TimelineSegment } from '@aitube/timeline'
+import { getWorkflowInputValues } from '../getWorkflowInputValues'
+
+import {
+  builtinProviderCredentialsLumalabs,
+  clapperApiKeyToUseBuiltinCredentials,
+} from '@/app/api/globalSettings'
+
+export async function resolveSegment(
+  request: ResolveRequest
+): Promise<TimelineSegment> {
+  let apiKey = request.settings.lumaLabsApiKey
+
+  if (!apiKey) {
+    if (clapperApiKeyToUseBuiltinCredentials) {
+      if (
+        request.settings.clapperApiKey !== clapperApiKeyToUseBuiltinCredentials
+      ) {
+        throw new Error(`Missing API key for "LumaLabs"`)
+      } else {
+        // user has a valid Clapper API key, so they are allowed to use the built-in credentials
+        apiKey = builtinProviderCredentialsLumalabs
+      }
+    } else {
+      // no Clapper API key is defined, so we give free access to the built-in credentials
+      apiKey = builtinProviderCredentialsLumalabs
+    }
+  }
+
+  const luma = new LumaAI({
+    authToken: apiKey,
+  })
+
+  const segment = request.segment
+
+  if (request.segment.category === ClapSegmentCategory.VIDEO) {
+    const { workflowValues } = getWorkflowInputValues(
+      request.settings.videoGenerationWorkflow
+    )
+
+    // Luma accepts:
+    //  "1:1" | "9:16" | "16:9" | "4:3" | "3:4" | "21:9" | "9:21" | undefined
+    const aspectRatio =
+      request.meta.orientation === ClapImageRatio.SQUARE
+        ? '1:1'
+        : request.meta.orientation === ClapImageRatio.PORTRAIT
+          ? '9:16'
+          : '16:9'
+
+    let params: LumaAI.GenerationCreateParams = {
+      // apply the default values from the workflow
+      ...workflowValues,
+
+      aspect_ratio: aspectRatio,
+      prompt: request.prompts.image.positive || '',
+    }
+
+    if (request.prompts.video.image) {
+      // If an image prompt is provided, add it to the parameters
+      params.keyframes = {
+        frame0: {
+          type: 'image',
+          url: request.prompts.video.image,
+        },
+      }
+    }
+
+    // if we have neither a text prompt or a frame,
+    // then there is no point in calling Luma Labs
+    if (!request.prompts.image.positive && !request.prompts.video.image) {
+      throw new Error('Cannot generate a video without a text or image')
+    }
+
+    try {
+      // Create the generation
+      const generation = await luma.generations.create(params)
+
+      // Poll for completion
+      let completedGeneration: LumaAI.Generation | null = null
+      while (!completedGeneration) {
+        if (!generation.id) {
+          throw new Error(`Generation failed: missing generation id`)
+        }
+        const status = await luma.generations.get(generation.id)
+        if (status.state === 'completed') {
+          completedGeneration = status
+          break
+        } else if (status.state === 'failed') {
+          throw new Error(`Generation failed: ${status.failure_reason}`)
+        }
+        // Wait for 5 seconds before polling again
+        await new Promise((resolve) => setTimeout(resolve, 5000))
+      }
+
+      // Store the URL of the final video
+      if (completedGeneration.assets?.video) {
+        segment.assetUrl = completedGeneration.assets.video
+      } else {
+        throw new Error('Generated video URL not found in the response')
+      }
+    } catch (error) {
+      console.error('Error generating video with LumaAI:', error)
+      throw error
+    }
+  } else {
+    throw new Error(
+      `Clapper doesn't support ${request.segment.category} generation for provider "LumaLabs". Please open a pull request with (working code) to solve this!`
+    )
+  }
+
+  return segment
+}

packages/app/src/components/toolbars/bottom-bar/MobileBottomBar.tsx

@@ -33,7 +33,7 @@ export function MobileBottomBar() {
       setShowExplorer(false)
       setShowVideoPlayer(true)
     }
-  }, [isMd])
+  }, [isMd, setShowExplorer, setShowVideoPlayer, setShowAssistant])
 
   return (
     <div

packages/app/src/services/editors/workflow-editor/workflows/common/defaultValues.ts

@@ -375,3 +375,22 @@ export const genericHeight2048: ClapInputField = {
   maxValue: 2048,
   defaultValue: 576,
 }
+
+export const genericAspectRatio: ClapInputField = {
+  id: 'aspect_ratio',
+  label: 'Aspect ratio',
+  description: 'Aspect ratio',
+  category: ClapInputCategory.ASPECT_RATIO,
+  type: 'string',
+  allowedValues: ['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21'],
+  defaultValue: '16:9',
+}
+
+export const genericKeyframes: ClapInputField = {
+  id: 'keyframes',
+  label: 'Keyframes',
+  description: 'Keyframes',
+  category: ClapInputCategory.KEYFRAMES,
+  type: 'object[]',
+  defaultValue: [],
+}

packages/app/src/services/editors/workflow-editor/workflows/index.ts

@@ -18,6 +18,7 @@ import { groqWorkflows } from './groq'
 import { hotshotWorkflows } from './hotshot'
 import { huggingfaceWorkflows } from './huggingface'
 import { letzAiWorkflows } from './letzai'
+import { lumalabsWorkflows } from './lumalabs'
 import { mistralaiWorkflows } from './mistralai'
 import { openaiWorkflows } from './openai'
 import { piApiWorkflows } from './piapi'
@@ -47,6 +48,7 @@ export const staticWorkflows: ClapWorkflow[] = [
   ...hotshotWorkflows,
   ...huggingfaceWorkflows,
   ...letzAiWorkflows,
+  ...lumalabsWorkflows,
   ...mistralaiWorkflows,
   ...openaiWorkflows,
   ...piApiWorkflows,

packages/app/src/services/editors/workflow-editor/workflows/lumalabs/index.ts

@@ -0,0 +1,69 @@
+import {
+  ClapWorkflow,
+  ClapWorkflowEngine,
+  ClapWorkflowCategory,
+  ClapWorkflowProvider,
+  ClapInputCategory,
+} from '@aitube/clap'
+
+import {
+  genericAspectRatio,
+  genericAudio,
+  genericBaseImageUrl,
+  genericDrivingVideo,
+  genericFaceImage,
+  genericGuidanceScale,
+  genericHeight1024,
+  genericHeight2048,
+  genericIdWeight,
+  genericImage,
+  genericImageUrl,
+  genericInferenceSteps,
+  genericKeyframes,
+  genericLora,
+  genericNegativePrompt,
+  genericPrompt,
+  genericReferenceImages,
+  genericSeed,
+  genericStartStep,
+  genericSwapImage,
+  genericSwapImageUrl,
+  genericTargetImage,
+  genericTrueCFG,
+  genericVideo,
+  genericWidth1024,
+  genericWidth2048,
+} from '../common/defaultValues'
+import { sampleDrivingVideo } from '@/lib/core/constants'
+
+// ------------------------------------------------------------------------------
+// if a user is already using one of those workflows and you change its settings,
+// they will have to reselect it in the UI for changes to be taken into account.
+//
+// -> we can create a ticket to fix this
+// ------------------------------------------------------------------------------
+export const lumalabsWorkflows: ClapWorkflow[] = [
+  {
+    id: 'lumalabs://dream-machine/v1',
+    label: 'Dream Machine',
+    description: '',
+    tags: ['LumaLabs', 'Dream Machine'],
+    author: 'Luma Labs (https://lumalabs.ai)',
+    thumbnailUrl: '',
+    nonCommercial: false,
+    engine: ClapWorkflowEngine.REST_API,
+    provider: ClapWorkflowProvider.LUMALABS,
+    category: ClapWorkflowCategory.VIDEO_GENERATION,
+    data: '',
+    schema: '',
+    /**
+     * Inputs of the workflow (this is used to build an UI for the workflow automatically)
+     */
+    inputFields: [genericPrompt, genericAspectRatio, genericKeyframes],
+    inputValues: {
+      [genericPrompt.id]: genericPrompt.defaultValue,
+      [genericAspectRatio.id]: genericAspectRatio.defaultValue,
+      [genericKeyframes.id]: genericKeyframes.defaultValue,
+    },
+  },
+]

packages/app/src/services/settings/getDefaultSettingsState.ts

@@ -33,6 +33,7 @@ export function getDefaultSettingsState(): SettingsState {
     piApiApiKey: '',
     civitaiApiKey: '',
     hotshotApiKey: '',
+    lumaLabsApiKey: '',
 
     broadcastObsServerHost: '192.168.1.22',
     broadcastObsServerPort: 4455,

packages/app/src/services/settings/useSettings.ts

@@ -259,7 +259,14 @@ export const useSettings = create<SettingsStore>()(
           ),
         })
       },
-
+      setLumaLabsApiKey: (lumaLabsApiKey?: string) => {
+        set({
+          lumaLabsApiKey: getValidString(
+            lumaLabsApiKey,
+            getDefaultSettingsState().lumaLabsApiKey
+          ),
+        })
+      },
       setBroadcastObsServerHost: (broadcastObsServerHost: string) => {
         set({ broadcastObsServerHost })
       },
@@ -960,6 +967,8 @@ export const useSettings = create<SettingsStore>()(
           piApiApiKey: state.piApiApiKey || defaultSettings.piApiApiKey,
           civitaiApiKey: state.civitaiApiKey || defaultSettings.civitaiApiKey,
           hotshotApiKey: state.hotshotApiKey || defaultSettings.hotshotApiKey,
+          lumaLabsApiKey:
+            state.lumaLabsApiKey || defaultSettings.lumaLabsApiKey,
 
           broadcastObsServerHost:
             state.broadcastObsServerHost ||

packages/clap/src/types.ts

@@ -520,6 +520,8 @@ export enum ClapInputCategory {
 
   WIDTH = "WIDTH",
   HEIGHT = "HEIGHT",
+  ASPECT_RATIO = "ASPECT_RATIO",
+  KEYFRAMES = "KEYFRAMES",
   SEED = "SEED",
   LORA = "LORA",
   //LORA_HF_MODEL = "LORA_HF_MODEL",
@@ -559,6 +561,16 @@ export type ClapInputFieldStrings = {
   defaultValue: string[]
 }
 
+export type ClapInputFieldObject = {
+  type: 'object'
+  defaultValue: {}
+}
+
+export type ClapInputFieldObjects = {
+  type: 'object[]'
+  defaultValue: []
+}
+
 export type ClapInputFieldBoolean = {
   type: 'boolean'
   defaultValue: boolean
@@ -616,6 +628,8 @@ export type ClapInputField<T = Record<string, any>> = {
   | ClapInputFieldInteger
   | ClapInputFieldString
   | ClapInputFieldStrings
+  | ClapInputFieldObject
+  | ClapInputFieldObjects
   | ClapInputFieldBoolean
   | ClapInputFieldAny
 )

packages/clapper-services/src/settings.ts

@@ -31,6 +31,7 @@ export type BaseSettings = {
   piApiApiKey: string
   civitaiApiKey: string
   hotshotApiKey: string
+  lumaLabsApiKey: string
 
   broadcastObsServerHost: string
   broadcastObsServerPort: number
@@ -155,11 +156,12 @@ export type SettingsControls = {
   setMistralAiApiKey: (mistralAiApiKey?: string) => void
   setKitsAiApiKey: (kitsAiApiKey?: string) => void
   setStabilityAiApiKey: (stabilityAiApiKey?: string) => void
-  setLetzAiApiKey: (letzAiApiKey?: string) =>  void
-  setBigModelApiKey: (bigModelApiKey?: string) =>  void
-  setPiApiApiKey: (piApiApiKey?: string) =>  void
-  setCivitaiApiKey: (civitaiApiKey?: string) =>  void
-  setHotshotApiKey: (hotshotApiKey?: string) =>  void
+  setLetzAiApiKey: (letzAiApiKey?: string) => void
+  setBigModelApiKey: (bigModelApiKey?: string) => void
+  setPiApiApiKey: (piApiApiKey?: string) => void
+  setCivitaiApiKey: (civitaiApiKey?: string) => void
+  setHotshotApiKey: (hotshotApiKey?: string) => void
+  setLumaLabsApiKey: (lumaLabsApiKey?: string) => void
 
   setCensorNotForAllAudiencesContent: (censorNotForAllAudiencesContent?: boolean) => void
   setImagePromptPrefix: (imagePromptPrefix?: string) => void