implement #117 to help cooling small GPUs down

packages/timeline/src/ClapTimeline.tsx

@@ -58,8 +58,10 @@ export function ClapTimeline({
     // frameloop: DEFAULT_FRAMELOOP
   }) {
   const theme = useTimeline(s => s.theme)
+  const invalidate = useTimeline(s => s.invalidate)
   const canvas = useTimeline(s => s.canvas)
   const setCanvas = useTimeline(s => s.setCanvas)
+  const handleMouseWheel = useTimeline(s => s.handleMouseWheel)
 
   const handleIsCreated = () => {
     useTimeline.setState({ isReady: true })
@@ -73,11 +75,39 @@ export function ClapTimeline({
     // for instance if the edited segment is being grabbed,
     // we are going to want to display the segments that are around it
     // console.log(`TODO @julian: implement edit here`)
+
+    // since we are un frameloop="demand" mode, we need to manual invalidate the scene
+    invalidate()
 
     event.stopPropagation()
     return false
   }
 
+  const handleWheel = (event: React.WheelEvent<HTMLDivElement>) => {
+    const rect = canvas?.getBoundingClientRect()
+    if (!rect) { return }
+
+    const clientY = event.clientY
+    const containerY = rect.y
+    const posY = clientY - containerY
+
+    // apparently we cannot stop the propagation from the scroll wheel event
+    // we attach to our to bar from the scroll wheel event set on the canvas
+    // (that makes sense, one is in DOM space, the other in WebGL space)
+    //
+    // there are probably better ways to do this, but for now here is a very
+    // crude fix to ignore global X-Y scroll events when we are over the timeline
+    if (posY <= topBarTimeScaleHeight) { return }
+
+    // since we are un frameloop="demand" mode, we need to manual invalidate the scene
+    invalidate()
+
+    handleMouseWheel({
+      deltaX: event.deltaX,
+      deltaY: event.deltaY
+    })
+  }
+
   return (
     <div
       className={cn(`w-full h-full`, className)}
@@ -99,7 +129,13 @@ export function ClapTimeline({
             id="clap-timeline"
 
             // must be active when playing back a video
-            frameloop="always"
+            // UPDATE on 20240912: right now we have disabled video preview from the timeline
+            // we don't need it anymore since we are displaying split frames for a video segment
+            // so we can disable the always on frame loop -> this should help cooling down the GPU
+            // frameloop="always"
+            frameloop="demand"
+            // note: if you disable frameloop="demand" then you need to
+            // search in the code for places that reference if (eg. manual instanciations)
 
             // those must stay ON otherwise colors will be washed out
             flat
@@ -115,29 +151,8 @@ export function ClapTimeline({
             }}
 
             onCreated={handleIsCreated}
-
-            onWheel={(wheelEvent) => {
-              const rect = canvas?.getBoundingClientRect()
-              if (!rect) { return }
-
-              const clientY = wheelEvent.clientY
-              const containerY = rect.y
-              const posY = clientY - containerY
-
-              // apparently we cannot stop the propagation from the scroll wheel event
-              // we attach to our to bar from the scroll wheel event set on the canvas
-              // (that makes sense, one is in DOM space, the other in WebGL space)
-              //
-              // there are probably better ways to do this, but for now here is a very
-              // crude fix to ignore global X-Y scroll events when we are over the timeline
-              if (posY <= topBarTimeScaleHeight) { return }
-
-              useTimeline.getState().handleMouseWheel({
-                deltaX: wheelEvent.deltaX,
-                deltaY: wheelEvent.deltaY
-              })
-            }}
 
+            onWheel={handleWheel}
             onMouseMove={handleMouseMove}
             onTouchMove={handleMouseMove}
             >

packages/timeline/src/components/controls/TimelineControls.tsx

@@ -22,11 +22,16 @@ export function TimelineControls({
   zoomSpeed: number
   zoomDampingFactor: number
 }) {
-  const { size, camera } = useThree()
+  const { size, camera, invalidate } = useThree()
+  const setInvalidate = useTimeline(s => s.setInvalidate)
   const timelineControls = useTimeline(s => s.timelineControls)
   const setTimelineControls = useTimeline(s => s.setTimelineControls)
   const initialPinchDistanceRef = useRef<number | null>(null)
 
+  useEffect(() => {
+    setInvalidate(invalidate)
+  }, [invalidate, setInvalidate])
+
   useEffect(() => {
     if (!timelineControls || !camera) return
 
@@ -206,7 +211,7 @@ export function TimelineControls({
       makeDefault
 
       // I don't remember why we put enabled false here
-      enabled={false}
+      enabled
 
       // minDistance={10}
       // maxDistance={10}

packages/timeline/src/hooks/useBreakpoints.ts

@@ -0,0 +1,43 @@
+import { useState, useEffect } from 'react'
+
+export type Breakpoints = {
+  isSm: boolean
+  isMd: boolean
+  isLg: boolean
+  isXl: boolean
+  is2xl: boolean
+}
+
+export function useBreakpoints(): Breakpoints {
+  const [breakpoints, setBreakpoints] = useState<Breakpoints>({
+    isSm: false,
+    isMd: false,
+    isLg: false,
+    isXl: false,
+    is2xl: false,
+  })
+
+  useEffect(() => {
+    const updateBreakpoints = () => {
+      const width = window.innerWidth
+      setBreakpoints({
+        isSm: width >= 640,
+        isMd: width >= 768,
+        isLg: width >= 1024,
+        isXl: width >= 1280,
+        is2xl: width >= 1536,
+      })
+    }
+
+    // Initial check
+    updateBreakpoints()
+
+    // Add event listener
+    window.addEventListener('resize', updateBreakpoints)
+
+    // Cleanup
+    return () => window.removeEventListener('resize', updateBreakpoints)
+  }, [])
+
+  return breakpoints
+}

packages/timeline/src/hooks/useTimeline.ts

@@ -3,16 +3,15 @@ import * as THREE from "three"
 import type { ThreeEvent } from "@react-three/fiber"
 import { ClapProject, ClapSegment, ClapSegmentCategory, isValidNumber, newClap, serializeClap, ClapTracks, ClapEntity, ClapMeta } from "@aitube/clap"
 
-import { TimelineSegment, SegmentEditionStatus, SegmentVisibility, TimelineStore, SegmentArea, SegmentPointerEvent, SegmentEventCallbackHandler } from "@/types/timeline"
+import { TimelineSegment, SegmentEditionStatus, SegmentVisibility, TimelineStore, SegmentArea, SegmentPointerEvent, SegmentEventCallbackHandler, Invalidate } from "@/types/timeline"
 import { getDefaultProjectState, getDefaultState } from "@/utils/getDefaultState"
 import { DEFAULT_NB_TRACKS, leftBarTrackScaleWidth } from "@/constants"
-import { hslToHex, findFreeTrack, removeFinalVideosAndConvertToTimelineSegments, clapSegmentToTimelineSegment, timelineSegmentToClapSegment } from "@/utils"
-import { ClapSegmentCategoryColors, ClapSegmentColorScheme, ClapTimelineTheme, SegmentResolver } from "@/types"
+import { findFreeTrack, removeFinalVideosAndConvertToTimelineSegments, clapSegmentToTimelineSegment, timelineSegmentToClapSegment } from "@/utils"
+import { ClapTimelineTheme, SegmentResolver } from "@/types"
 import { TimelineControlsImpl } from "@/components/controls/types"
 import { TimelineCameraImpl } from "@/components/camera/types"
 import { IsPlaying, JumpAt, TimelineCursorImpl, TogglePlayback } from "@/components/timeline/types"
 import { computeContentSizeMetrics } from "@/compute/computeContentSizeMetrics"
-import { useThree } from "@react-three/fiber"
 import { topBarTimeScaleHeight } from "@/constants/themes"
 
 export const useTimeline = create<TimelineStore>((set, get) => ({
@@ -347,11 +346,15 @@ export const useTimeline = create<TimelineStore>((set, get) => ({
     area?: SegmentArea 
   } = {}) => {
     const {
+      invalidate,
       hoveredSegment: previousHoveredSegment,
       atLeastOneSegmentChanged: previousAtLeastOneSegmentChanged,
       allSegmentsChanged: previousAllSegmentsChanged,
     } = get()
 
+    // since we are un frameloop="demand" mode, we need to manual invalidate the scene
+    invalidate()
+
     // note: we do all of this in order to avoid useless state updates
     if (segment && area) {
       if (previousHoveredSegment) {
@@ -401,11 +404,15 @@ export const useTimeline = create<TimelineStore>((set, get) => ({
     status: SegmentEditionStatus.EDITING
   }) => {
     const {
+      invalidate,
       editedSegment: previousEditedSegment,
       allSegmentsChanged: previousAllSegmentsChanged,
       atLeastOneSegmentChanged: previousAtLeastOneSegmentChanged
     } = get()
 
+    // since we are un frameloop="demand" mode, we need to manual invalidate the scene
+    invalidate()
+
     // note: we do all of this in order to avoid useless state updates
     if (segment) {
       if (previousEditedSegment) {
@@ -447,11 +454,16 @@ export const useTimeline = create<TimelineStore>((set, get) => ({
   } = {
   }) => {
     const {
+      invalidate,
       segments,
       selectedSegments: previousSelectedSegments,
       atLeastOneSegmentChanged: previousAtLeastOneSegmentChanged,
       allSegmentsChanged: previousAllSegmentsChanged
     } = get()
+
+    // since we are un frameloop="demand" mode, we need to manual invalidate the scene
+    invalidate()
+
     /*
     console.log(`setSelectedSegment() called with:`, {
       segment,
@@ -1293,6 +1305,10 @@ export const useTimeline = create<TimelineStore>((set, get) => ({
         entitiesChanged: previousEntitiesChanged + 1,
       })
     }
+  },
+
+  setInvalidate: (invalidate?: Invalidate) => {
+    set({ invalidate: invalidate || (() => {}) })
   }
 }
 ))

packages/timeline/src/types/timeline.ts

@@ -10,6 +10,8 @@ import { SegmentResolver } from "./rendering"
 
 export type SegmentEventCallbackHandler = (e: ThreeEvent<PointerEvent> | ThreeEvent<MouseEvent>) => boolean
 
+export type Invalidate = (frames?: number) => void
+
 export enum SegmentVisibility {
   // the segment is visible, and the user explicitly requested to render it before the others
   DEMANDED = "DEMANDED",
@@ -269,6 +271,8 @@ export type TimelineStorePreferencesState = {
   jumpAt: JumpAt
   isPlaying: IsPlaying
   togglePlayback: TogglePlayback
+
+  invalidate: Invalidate
 }
 
 export type TimelineStoreState = TimelineStoreProjectState & TimelineStorePreferencesState
@@ -396,6 +400,8 @@ export type TimelineStoreModifiers = {
   addEntities: (entities: ClapEntity[]) => Promise<void>
   updateEntities: (entities: ClapEntity[]) => Promise<void>
   deleteEntities: (entities: (ClapEntity|string)[]) => Promise<void>
+
+  setInvalidate: (invalidate?: Invalidate) => void
 }
 
 export type TimelineStore = TimelineStoreState & TimelineStoreModifiers

packages/timeline/src/utils/getDefaultState.ts

@@ -91,7 +91,9 @@ export function getDefaultPreferencesState(): TimelineStorePreferencesState {
     segmentResolver: async (segment) => segment,
     jumpAt: () => {},
     isPlaying: () => false,
-    togglePlayback: () => ({ wasPlaying: false, isPlaying: false })
+    togglePlayback: () => ({ wasPlaying: false, isPlaying: false }),
+
+    invalidate: () => {},
   }
 }