From 3723223d4fa49bb2af6d0ffadd3159aa9fa4ec76 Mon Sep 17 00:00:00 2001 From: andchiind Date: Mon, 25 Nov 2024 13:13:10 +0100 Subject: [PATCH] Reconnect to video streams on disconnect --- frontend/src/api/ApiCaller.tsx | 7 + .../Contexts/MediaStreamContext.tsx | 148 ++++++++++++------ .../Pages/MissionPage/MissionPage.tsx | 8 +- .../components/Pages/RobotPage/RobotPage.tsx | 7 +- 4 files changed, 119 insertions(+), 51 deletions(-) diff --git a/frontend/src/api/ApiCaller.tsx b/frontend/src/api/ApiCaller.tsx index 8fd03de92..cac0e2be9 100644 --- a/frontend/src/api/ApiCaller.tsx +++ b/frontend/src/api/ApiCaller.tsx @@ -13,6 +13,7 @@ import { MissionDefinition, PlantInfo } from 'models/MissionDefinition' import { MissionDefinitionUpdateForm } from 'models/MissionDefinitionUpdateForm' import { Deck } from 'models/Deck' import { ApiError, isApiError } from './ApiError' +import { MediaStreamConfig } from 'models/VideoStream' /** Implements the request sent to the backend api. */ export class BackendAPICaller { @@ -140,6 +141,12 @@ export class BackendAPICaller { return result.content } + static async getRobotMediaConfig(robotId: string): Promise { + const path: string = 'media-stream/' + robotId + const result = await this.GET(path).catch(BackendAPICaller.handleError('GET', path)) + return result.content + } + static async getMissionRuns(parameters: MissionRunQueryParameters): Promise> { let path: string = 'missions/runs?' diff --git a/frontend/src/components/Contexts/MediaStreamContext.tsx b/frontend/src/components/Contexts/MediaStreamContext.tsx index 118893e8a..babe587ab 100644 --- a/frontend/src/components/Contexts/MediaStreamContext.tsx +++ b/frontend/src/components/Contexts/MediaStreamContext.tsx @@ -1,22 +1,19 @@ import { createContext, FC, useContext, useEffect, useState } from 'react' -import { SignalREventLabels, useSignalRContext } from './SignalRContext' -import { useRobotContext } from './RobotContext' -import { - ConnectionState, - RemoteParticipant, - RemoteTrack, - RemoteTrackPublication, - Room, - RoomEvent, -} from 'livekit-client' +import { ConnectionState, Room, RoomEvent } from 'livekit-client' import { MediaConnectionType, MediaStreamConfig } from 'models/VideoStream' +import { BackendAPICaller } from 'api/ApiCaller' type MediaStreamDictionaryType = { - [robotId: string]: MediaStreamConfig & { streams: MediaStreamTrack[] } + [robotId: string]: { isLoading: boolean } & MediaStreamConfig & { streams: MediaStreamTrack[] } +} + +type MediaStreamConfigDictionaryType = { + [robotId: string]: MediaStreamConfig } interface IMediaStreamContext { mediaStreams: MediaStreamDictionaryType + addMediaStreamConfigIfItDoesNotExist: (robotId: string) => void } interface Props { @@ -25,6 +22,7 @@ interface Props { const defaultMediaStreamInterface = { mediaStreams: {}, + addMediaStreamConfigIfItDoesNotExist: (robotId: string) => {}, } const MediaStreamContext = createContext(defaultMediaStreamInterface) @@ -33,76 +31,128 @@ export const MediaStreamProvider: FC = ({ children }) => { const [mediaStreams, setMediaStreams] = useState( defaultMediaStreamInterface.mediaStreams ) - const { registerEvent, connectionReady } = useSignalRContext() - const { enabledRobots } = useRobotContext() + const [cachedConfigs, _] = useState( + JSON.parse(window.localStorage.getItem('mediaConfigs') ?? '{}') + ) + + useEffect(() => { + // Here we maintain the localstorage with the connection details + let updatedConfigs: MediaStreamConfigDictionaryType = {} + Object.keys(mediaStreams).forEach((robotId) => { + const conf = mediaStreams[robotId] + + if (conf.streams.length === 0 && !conf.isLoading) refreshRobotMediaConfig(robotId) + updatedConfigs[robotId] = { + url: conf.url, + token: conf.token, + mediaConnectionType: conf.mediaConnectionType, + robotId: conf.robotId, + } + }) + window.localStorage.setItem('mediaConfigs', JSON.stringify(updatedConfigs)) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [mediaStreams]) const addTrackToConnection = (newTrack: MediaStreamTrack, robotId: string) => { setMediaStreams((oldStreams) => { - if (!Object.keys(oldStreams).includes(robotId)) { + if ( + !Object.keys(oldStreams).includes(robotId) || + oldStreams[robotId].streams.find((s) => s.id === newTrack.id) + ) { return oldStreams } else { - const newStreams = { ...oldStreams } return { ...oldStreams, - [robotId]: { ...newStreams[robotId], streams: [...oldStreams[robotId].streams, newTrack] }, + [robotId]: { + ...oldStreams[robotId], + streams: [...oldStreams[robotId].streams, newTrack], + isLoading: false, + }, } } }) } - const createLiveKitConnection = async (config: MediaStreamConfig) => { + const createLiveKitConnection = async (config: MediaStreamConfig, cachedConfig: boolean = false) => { const room = new Room() - room.on(RoomEvent.TrackSubscribed, handleTrackSubscribed) - - function handleTrackSubscribed( - track: RemoteTrack, - publication: RemoteTrackPublication, - participant: RemoteParticipant - ) { - addTrackToConnection(track.mediaStreamTrack, config.robotId) - } + + window.addEventListener('unload', async () => room.disconnect()) + + room.on(RoomEvent.TrackSubscribed, (track) => addTrackToConnection(track.mediaStreamTrack, config.robotId)) + room.on(RoomEvent.TrackUnpublished, (e) => { + setMediaStreams((oldStreams) => { + let streamsCopy = { ...oldStreams } + if (!Object.keys(streamsCopy).includes(config.robotId) || streamsCopy[config.robotId].isLoading) + return streamsCopy + + let streamList = streamsCopy[config.robotId].streams + const streamIndex = streamList.findIndex((s) => s.id === e.trackSid) + + if (streamIndex < 0) return streamsCopy + + streamList.splice(streamIndex, 1) + streamsCopy[config.robotId].streams = streamList + + if (streamList.length === 0) room.disconnect() + + return streamsCopy + }) + }) + if (room.state === ConnectionState.Disconnected) { room.connect(config.url, config.token) - .then(() => console.log(JSON.stringify(room.state))) - .catch((error) => console.warn('Error connecting to LiveKit Room, may already be connected:', error)) + .then(() => console.log('LiveKit room status: ', JSON.stringify(room.state))) + .catch((error) => { + if (cachedConfig) refreshRobotMediaConfig(config.robotId) + else console.error('Failed to connect to LiveKit room: ', error) + }) } } - const createMediaConnection = async (config: MediaStreamConfig) => { + const createMediaConnection = async (config: MediaStreamConfig, cachedConfig: boolean = false) => { switch (config.mediaConnectionType) { case MediaConnectionType.LiveKit: - return await createLiveKitConnection(config) + return await createLiveKitConnection(config, cachedConfig) default: console.error('Invalid media connection type received') } return undefined } - // Register a signalR event handler that listens for new media stream connections - useEffect(() => { - if (connectionReady) { - registerEvent(SignalREventLabels.mediaStreamConfigReceived, (username: string, message: string) => { - const newMediaConfig: MediaStreamConfig = JSON.parse(message) - setMediaStreams((oldStreams) => { - if (Object.keys(oldStreams).includes(newMediaConfig.robotId)) { - return oldStreams - } else { - createMediaConnection(newMediaConfig) - return { - ...oldStreams, - [newMediaConfig.robotId]: { ...newMediaConfig, streams: [] }, - } - } - }) - }) + const addConfigToMediaStreams = (conf: MediaStreamConfig, cachedConfig: boolean = false) => { + setMediaStreams((oldStreams) => { + createMediaConnection(conf, cachedConfig) + return { + ...oldStreams, + [conf.robotId]: { ...conf, streams: [], isLoading: true }, + } + }) + } + + const addMediaStreamConfigIfItDoesNotExist = (robotId: string) => { + if (Object.keys(mediaStreams).includes(robotId)) { + const currentStream = mediaStreams[robotId] + if (currentStream.isLoading || currentStream.streams.find((stream) => stream.enabled)) return + } else if (Object.keys(cachedConfigs).includes(robotId)) { + const config = cachedConfigs[robotId] + addConfigToMediaStreams(config, true) + return } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [registerEvent, connectionReady, enabledRobots]) + + refreshRobotMediaConfig(robotId) + } + + const refreshRobotMediaConfig = (robotId: string) => { + BackendAPICaller.getRobotMediaConfig(robotId) + .then((conf: MediaStreamConfig) => addConfigToMediaStreams(conf)) + .catch((e) => console.error(e)) + } return ( {children} diff --git a/frontend/src/components/Pages/MissionPage/MissionPage.tsx b/frontend/src/components/Pages/MissionPage/MissionPage.tsx index f6fb417ef..660ac1866 100644 --- a/frontend/src/components/Pages/MissionPage/MissionPage.tsx +++ b/frontend/src/components/Pages/MissionPage/MissionPage.tsx @@ -41,7 +41,13 @@ export const MissionPage = () => { const [videoMediaStreams, setVideoMediaStreams] = useState([]) const [selectedMission, setSelectedMission] = useState() const { registerEvent, connectionReady } = useSignalRContext() - const { mediaStreams } = useMediaStreamContext() + const { mediaStreams, addMediaStreamConfigIfItDoesNotExist } = useMediaStreamContext() + + useEffect(() => { + if (selectedMission && !Object.keys(mediaStreams).includes(selectedMission?.robot.id)) + addMediaStreamConfigIfItDoesNotExist(selectedMission?.robot.id) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedMission]) useEffect(() => { if (connectionReady) { diff --git a/frontend/src/components/Pages/RobotPage/RobotPage.tsx b/frontend/src/components/Pages/RobotPage/RobotPage.tsx index 894c995ca..db502c535 100644 --- a/frontend/src/components/Pages/RobotPage/RobotPage.tsx +++ b/frontend/src/components/Pages/RobotPage/RobotPage.tsx @@ -64,11 +64,16 @@ export const RobotPage = () => { const { setAlert, setListAlert } = useAlertContext() const { robotId } = useParams() const { enabledRobots } = useRobotContext() - const { mediaStreams } = useMediaStreamContext() + const { mediaStreams, addMediaStreamConfigIfItDoesNotExist } = useMediaStreamContext() const [videoMediaStreams, setVideoMediaStreams] = useState([]) const selectedRobot = enabledRobots.find((robot) => robot.id === robotId) + useEffect(() => { + if (robotId && !Object.keys(mediaStreams).includes(robotId)) addMediaStreamConfigIfItDoesNotExist(robotId) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [robotId]) + const returnRobotToHome = () => { if (robotId) { BackendAPICaller.returnRobotToHome(robotId).catch((e) => {