Skip to content

Commit

Permalink
Reconnect to video streams on disconnect
Browse files Browse the repository at this point in the history
  • Loading branch information
andchiind committed Dec 2, 2024
1 parent 9c3730c commit f78597c
Show file tree
Hide file tree
Showing 4 changed files with 119 additions and 51 deletions.
7 changes: 7 additions & 0 deletions frontend/src/api/ApiCaller.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { MissionDefinition, PlantInfo } from 'models/MissionDefinition'
import { MissionDefinitionUpdateForm } from 'models/MissionDefinitionUpdateForm'
import { Deck } from 'models/Deck'
import { ApiError, isApiError } from './ApiError'
import { MediaStreamConfig } from 'models/VideoStream'

/** Implements the request sent to the backend api. */
export class BackendAPICaller {
Expand Down Expand Up @@ -140,6 +141,12 @@ export class BackendAPICaller {
return result.content
}

static async getRobotMediaConfig(robotId: string): Promise<MediaStreamConfig> {
const path: string = 'media-stream/' + robotId
const result = await this.GET<MediaStreamConfig>(path).catch(BackendAPICaller.handleError('GET', path))
return result.content
}

static async getMissionRuns(parameters: MissionRunQueryParameters): Promise<PaginatedResponse<Mission>> {
let path: string = 'missions/runs?'

Expand Down
148 changes: 99 additions & 49 deletions frontend/src/components/Contexts/MediaStreamContext.tsx
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
import { createContext, FC, useContext, useEffect, useState } from 'react'
import { SignalREventLabels, useSignalRContext } from './SignalRContext'
import { useRobotContext } from './RobotContext'
import {
ConnectionState,
RemoteParticipant,
RemoteTrack,
RemoteTrackPublication,
Room,
RoomEvent,
} from 'livekit-client'
import { ConnectionState, Room, RoomEvent } from 'livekit-client'
import { MediaConnectionType, MediaStreamConfig } from 'models/VideoStream'
import { BackendAPICaller } from 'api/ApiCaller'

type MediaStreamDictionaryType = {
[robotId: string]: MediaStreamConfig & { streams: MediaStreamTrack[] }
[robotId: string]: { isLoading: boolean } & MediaStreamConfig & { streams: MediaStreamTrack[] }
}

type MediaStreamConfigDictionaryType = {
[robotId: string]: MediaStreamConfig
}

interface IMediaStreamContext {
mediaStreams: MediaStreamDictionaryType
addMediaStreamConfigIfItDoesNotExist: (robotId: string) => void
}

interface Props {
Expand All @@ -25,6 +22,7 @@ interface Props {

const defaultMediaStreamInterface = {
mediaStreams: {},
addMediaStreamConfigIfItDoesNotExist: (robotId: string) => {},
}

const MediaStreamContext = createContext<IMediaStreamContext>(defaultMediaStreamInterface)
Expand All @@ -33,76 +31,128 @@ export const MediaStreamProvider: FC<Props> = ({ children }) => {
const [mediaStreams, setMediaStreams] = useState<MediaStreamDictionaryType>(
defaultMediaStreamInterface.mediaStreams
)
const { registerEvent, connectionReady } = useSignalRContext()
const { enabledRobots } = useRobotContext()
const [cachedConfigs] = useState<MediaStreamConfigDictionaryType>(
JSON.parse(window.localStorage.getItem('mediaConfigs') ?? '{}')
)

useEffect(() => {
// Here we maintain the localstorage with the connection details
let updatedConfigs: MediaStreamConfigDictionaryType = {}
Object.keys(mediaStreams).forEach((robotId) => {
const conf = mediaStreams[robotId]

if (conf.streams.length === 0 && !conf.isLoading) refreshRobotMediaConfig(robotId)
updatedConfigs[robotId] = {
url: conf.url,
token: conf.token,
mediaConnectionType: conf.mediaConnectionType,
robotId: conf.robotId,
}
})
window.localStorage.setItem('mediaConfigs', JSON.stringify(updatedConfigs))
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [mediaStreams])

const addTrackToConnection = (newTrack: MediaStreamTrack, robotId: string) => {
setMediaStreams((oldStreams) => {
if (!Object.keys(oldStreams).includes(robotId)) {
if (
!Object.keys(oldStreams).includes(robotId) ||
oldStreams[robotId].streams.find((s) => s.id === newTrack.id)
) {
return oldStreams
} else {
const newStreams = { ...oldStreams }
return {
...oldStreams,
[robotId]: { ...newStreams[robotId], streams: [...oldStreams[robotId].streams, newTrack] },
[robotId]: {
...oldStreams[robotId],
streams: [...oldStreams[robotId].streams, newTrack],
isLoading: false,
},
}
}
})
}

const createLiveKitConnection = async (config: MediaStreamConfig) => {
const createLiveKitConnection = async (config: MediaStreamConfig, cachedConfig: boolean = false) => {
const room = new Room()
room.on(RoomEvent.TrackSubscribed, handleTrackSubscribed)

function handleTrackSubscribed(
track: RemoteTrack,
publication: RemoteTrackPublication,
participant: RemoteParticipant
) {
addTrackToConnection(track.mediaStreamTrack, config.robotId)
}

window.addEventListener('unload', async () => room.disconnect())

room.on(RoomEvent.TrackSubscribed, (track) => addTrackToConnection(track.mediaStreamTrack, config.robotId))
room.on(RoomEvent.TrackUnpublished, (e) => {
setMediaStreams((oldStreams) => {
let streamsCopy = { ...oldStreams }
if (!Object.keys(streamsCopy).includes(config.robotId) || streamsCopy[config.robotId].isLoading)
return streamsCopy

let streamList = streamsCopy[config.robotId].streams
const streamIndex = streamList.findIndex((s) => s.id === e.trackSid)

if (streamIndex < 0) return streamsCopy

streamList.splice(streamIndex, 1)
streamsCopy[config.robotId].streams = streamList

if (streamList.length === 0) room.disconnect()

return streamsCopy
})
})

if (room.state === ConnectionState.Disconnected) {
room.connect(config.url, config.token)
.then(() => console.log(JSON.stringify(room.state)))
.catch((error) => console.warn('Error connecting to LiveKit Room, may already be connected:', error))
.then(() => console.log('LiveKit room status: ', JSON.stringify(room.state)))
.catch((error) => {
if (cachedConfig) refreshRobotMediaConfig(config.robotId)
else console.error('Failed to connect to LiveKit room: ', error)
})
}
}

const createMediaConnection = async (config: MediaStreamConfig) => {
const createMediaConnection = async (config: MediaStreamConfig, cachedConfig: boolean = false) => {
switch (config.mediaConnectionType) {
case MediaConnectionType.LiveKit:
return await createLiveKitConnection(config)
return await createLiveKitConnection(config, cachedConfig)
default:
console.error('Invalid media connection type received')
}
return undefined
}

// Register a signalR event handler that listens for new media stream connections
useEffect(() => {
if (connectionReady) {
registerEvent(SignalREventLabels.mediaStreamConfigReceived, (username: string, message: string) => {
const newMediaConfig: MediaStreamConfig = JSON.parse(message)
setMediaStreams((oldStreams) => {
if (Object.keys(oldStreams).includes(newMediaConfig.robotId)) {
return oldStreams
} else {
createMediaConnection(newMediaConfig)
return {
...oldStreams,
[newMediaConfig.robotId]: { ...newMediaConfig, streams: [] },
}
}
})
})
const addConfigToMediaStreams = (conf: MediaStreamConfig, cachedConfig: boolean = false) => {
setMediaStreams((oldStreams) => {
createMediaConnection(conf, cachedConfig)
return {
...oldStreams,
[conf.robotId]: { ...conf, streams: [], isLoading: true },
}
})
}

const addMediaStreamConfigIfItDoesNotExist = (robotId: string) => {
if (Object.keys(mediaStreams).includes(robotId)) {
const currentStream = mediaStreams[robotId]
if (currentStream.isLoading || currentStream.streams.find((stream) => stream.enabled)) return
} else if (Object.keys(cachedConfigs).includes(robotId)) {
const config = cachedConfigs[robotId]
addConfigToMediaStreams(config, true)
return
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [registerEvent, connectionReady, enabledRobots])

refreshRobotMediaConfig(robotId)
}

const refreshRobotMediaConfig = (robotId: string) => {
BackendAPICaller.getRobotMediaConfig(robotId)
.then((conf: MediaStreamConfig) => addConfigToMediaStreams(conf))
.catch((e) => console.error(e))
}

return (
<MediaStreamContext.Provider
value={{
mediaStreams,
addMediaStreamConfigIfItDoesNotExist,
}}
>
{children}
Expand Down
8 changes: 7 additions & 1 deletion frontend/src/components/Pages/MissionPage/MissionPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,13 @@ export const MissionPage = () => {
const [videoMediaStreams, setVideoMediaStreams] = useState<MediaStreamTrack[]>([])
const [selectedMission, setSelectedMission] = useState<Mission>()
const { registerEvent, connectionReady } = useSignalRContext()
const { mediaStreams } = useMediaStreamContext()
const { mediaStreams, addMediaStreamConfigIfItDoesNotExist } = useMediaStreamContext()

useEffect(() => {
if (selectedMission && !Object.keys(mediaStreams).includes(selectedMission?.robot.id))
addMediaStreamConfigIfItDoesNotExist(selectedMission?.robot.id)
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedMission])

useEffect(() => {
if (connectionReady) {
Expand Down
7 changes: 6 additions & 1 deletion frontend/src/components/Pages/RobotPage/RobotPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,15 @@ export const RobotPage = () => {
const { setAlert, setListAlert } = useAlertContext()
const { robotId } = useParams()
const { enabledRobots } = useRobotContext()
const { mediaStreams } = useMediaStreamContext()
const { mediaStreams, addMediaStreamConfigIfItDoesNotExist } = useMediaStreamContext()
const [videoMediaStreams, setVideoMediaStreams] = useState<MediaStreamTrack[]>([])
const { ongoingMissions } = useMissionsContext()

useEffect(() => {
if (robotId && !Object.keys(mediaStreams).includes(robotId)) addMediaStreamConfigIfItDoesNotExist(robotId)
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [robotId])

const selectedRobot = enabledRobots.find((robot) => robot.id === robotId)

const [isDialogOpen, setIsDialogOpen] = useState(false)
Expand Down

0 comments on commit f78597c

Please sign in to comment.