Skip to content

Commit

Permalink
✨(frontend) introduce blurring effects for Chromium-based users
Browse files Browse the repository at this point in the history
Implemented using MediaPipe, which is not supported on Firefox due to
limitations outlined in issue #38 of the track-processors-js repo.

We decided to release this first version exclusively for Chrome to quickly
offer a solution to users. Future iterations will address broader compatibility.

An informational message will be added to notify users that the feature is
experimental. For now, a text label will be used in place of an icon.
  • Loading branch information
lebaudantoine committed Sep 21, 2024
1 parent 8594245 commit af5b0d1
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 13 deletions.
104 changes: 94 additions & 10 deletions src/frontend/src/features/rooms/livekit/components/Effects.tsx
Original file line number Diff line number Diff line change
@@ -1,24 +1,72 @@
import { useEffect, useRef } from 'react'
import { useEffect, useRef, useState } from 'react'
import { useLocalParticipant } from '@livekit/components-react'
import { LocalVideoTrack } from 'livekit-client'
import { Div, P } from '@/primitives'
import { Text, P, ToggleButton } from '@/primitives'
import { useTranslation } from 'react-i18next'
import { HStack, VStack } from '@/styled-system/jsx'
import {
BackgroundBlur,
BackgroundOptions,
ProcessorWrapper,
BackgroundTransformer,
} from '@livekit/track-processors'

enum BlurRadius {
NONE = 0,
LIGHT = 5,
NORMAL = 10,
}

export const Effects = () => {
const { t } = useTranslation('rooms', { keyPrefix: 'effects' })
const { isCameraEnabled, cameraTrack } = useLocalParticipant()
const { isCameraEnabled, cameraTrack, localParticipant } =
useLocalParticipant()
const videoRef = useRef<HTMLVideoElement>(null)
const [processorPending, setProcessorPending] = useState(false)

const localCameraTrack = cameraTrack?.track as LocalVideoTrack

useEffect(() => {
const videoElement = videoRef.current
const getProcessor = () => {
return localCameraTrack?.getProcessor() as ProcessorWrapper<BackgroundOptions>
}

const attachVideoTrack = async () => {
if (!videoElement) return
localCameraTrack?.attach(videoElement)
const getBlurRadius = (): BlurRadius => {
const processor = getProcessor()
return (
(processor?.transformer as BackgroundTransformer)?.options?.blurRadius ||
BlurRadius.NONE
)
}

const toggleBlur = async (blurRadius: number) => {
if (!isCameraEnabled) await localParticipant.setCameraEnabled(true)
if (!localCameraTrack) return

setProcessorPending(true)

const processor = getProcessor()
const currentBlurRadius = getBlurRadius()

try {
if (blurRadius == currentBlurRadius && processor) {
await localCameraTrack.stopProcessor()
} else if (!processor) {
await localCameraTrack.setProcessor(BackgroundBlur(blurRadius))
} else {
await processor?.updateTransformerOptions({ blurRadius })
}
} catch (error) {
console.error('Error applying blur:', error)
} finally {
setProcessorPending(false)
}
}

useEffect(() => {
const videoElement = videoRef.current
if (!videoElement) return

const attachVideoTrack = async () => localCameraTrack?.attach(videoElement)
attachVideoTrack()

return () => {
Expand All @@ -27,8 +75,16 @@ export const Effects = () => {
}
}, [localCameraTrack, isCameraEnabled])

const isSelected = (blurRadius: BlurRadius) => {
return isCameraEnabled && getBlurRadius() == blurRadius
}

const tooltipLabel = (blurRadius: BlurRadius) => {
return t(`blur.${isSelected(blurRadius) ? 'clear' : 'apply'}`)
}

return (
<Div padding="0 1.5rem">
<VStack padding="0 1.5rem">
{localCameraTrack && isCameraEnabled ? (
<video
ref={videoRef}
Expand Down Expand Up @@ -63,6 +119,34 @@ export const Effects = () => {
</P>
</div>
)}
</Div>
{ProcessorWrapper.isSupported ? (
<HStack>
<ToggleButton
size={'sm'}
legacyStyle
aria-label={tooltipLabel(BlurRadius.LIGHT)}
tooltip={tooltipLabel(BlurRadius.LIGHT)}
isDisabled={processorPending}
onPress={async () => await toggleBlur(BlurRadius.LIGHT)}
isSelected={isSelected(BlurRadius.LIGHT)}
>
{t('blur.light')}
</ToggleButton>
<ToggleButton
size={'sm'}
legacyStyle
aria-label={tooltipLabel(BlurRadius.NORMAL)}
tooltip={tooltipLabel(BlurRadius.NORMAL)}
isDisabled={processorPending}
onPress={async () => await toggleBlur(BlurRadius.NORMAL)}
isSelected={isSelected(BlurRadius.NORMAL)}
>
{t('blur.normal')}
</ToggleButton>
</HStack>
) : (
<Text variant="sm">{t('notAvailable')}</Text>
)}
</VStack>
)
}
9 changes: 8 additions & 1 deletion src/frontend/src/locales/de/rooms.json
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,14 @@
}
},
"effects": {
"activateCamera": ""
"activateCamera": "",
"notAvailable": "",
"blur": {
"light": "",
"normal": "",
"apply": "",
"clear": ""
}
},
"sidePanel" : {
"heading": {
Expand Down
9 changes: 8 additions & 1 deletion src/frontend/src/locales/en/rooms.json
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,14 @@
}
},
"effects": {
"activateCamera": "Camera is disabled"
"activateCamera": "Your camera is disabled. Choose an option to enable it.",
"notAvailable": "The blur effect will be available soon on your browser. We're working on it! In the meantime, you can use Google Chrome :(",
"blur": {
"light": "Light blur",
"normal": "Blur",
"apply": "Enable blur",
"clear": "Disable blur"
}
},
"sidePanel" : {
"heading": {
Expand Down
9 changes: 8 additions & 1 deletion src/frontend/src/locales/fr/rooms.json
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,14 @@
}
},
"effects": {
"activateCamera": "Votre camera est désactivée"
"activateCamera": "Votre camera est désactivée. Choisissez une option pour l'activer.",
"notAvailable": "L'effet de flou sera bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome :(",
"blur": {
"light": "Léger flou",
"normal": "Flou",
"apply": "Appliquer le flou",
"clear": "Désactiver le flou"
}
},
"sidePanel" : {
"heading": {
Expand Down

0 comments on commit af5b0d1

Please sign in to comment.