Skip to content

Commit

Permalink
feat: Automatic Video Quality Adjustments [WPB-11479] (#18253)
Browse files Browse the repository at this point in the history
* feat: request the streams quality whenever the video display resolution changes

* feat: add track debugger and fix load resolution on max mode
  • Loading branch information
EnricoSchw authored Dec 4, 2024
1 parent 391f8ba commit ce47d8f
Show file tree
Hide file tree
Showing 11 changed files with 227 additions and 28 deletions.
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"@lexical/history": "0.20.2",
"@lexical/react": "0.20.2",
"@mediapipe/tasks-vision": "0.10.18",
"@wireapp/avs": "9.10.16",
"@wireapp/avs": "10.0.4",
"@wireapp/avs-debugger": "0.0.5",
"@wireapp/commons": "5.4.0",
"@wireapp/core": "46.11.4",
"@wireapp/react-ui-kit": "9.28.0",
Expand Down
2 changes: 1 addition & 1 deletion src/script/calling/Call.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ export class Call {
public blockMessages: boolean = false;
public currentPage: ko.Observable<number> = ko.observable(0);
public pages: ko.ObservableArray<Participant[]> = ko.observableArray();
readonly maximizedParticipant: ko.Observable<Participant | null>;
public readonly maximizedParticipant: ko.Observable<Participant | null>;
public readonly isActive: ko.PureComputed<boolean>;

private readonly audios: Record<string, {audioElement: HTMLAudioElement; stream: MediaStream}> = {};
Expand Down
13 changes: 11 additions & 2 deletions src/script/calling/CallState.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,9 @@ export class CallState {
public readonly activeCalls: ko.PureComputed<Call[]>;
public readonly joinedCall: ko.PureComputed<Call | undefined>;
public readonly activeCallViewTab = ko.observable(CallViewTab.ALL);
readonly hasAvailableScreensToShare: ko.PureComputed<boolean>;
readonly isSpeakersViewActive: ko.PureComputed<boolean>;
public readonly hasAvailableScreensToShare: ko.PureComputed<boolean>;
public readonly isSpeakersViewActive: ko.PureComputed<boolean>;
public readonly isMaximisedViewActive: ko.PureComputed<boolean>;
public readonly viewMode = ko.observable<CallingViewMode>(CallingViewMode.MINIMIZED);
public readonly detachedWindow = ko.observable<Window | null>(null);
public readonly isScreenSharingSourceFromDetachedWindow = ko.observable<boolean>(false);
Expand All @@ -96,6 +97,14 @@ export class CallState {
});
this.isSpeakersViewActive = ko.pureComputed(() => this.activeCallViewTab() === CallViewTab.SPEAKERS);

this.isMaximisedViewActive = ko.pureComputed(() => {
const call = this.joinedCall();
if (!call) {
return false;
}
return call.maximizedParticipant() !== null;
});

this.hasAvailableScreensToShare = ko.pureComputed(
() => this.selectableScreens().length > 0 || this.selectableWindows().length > 0,
);
Expand Down
81 changes: 75 additions & 6 deletions src/script/calling/CallingRepository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,15 @@ import {
LOG_LEVEL,
QUALITY,
REASON,
RESOLUTION,
STATE as CALL_STATE,
VIDEO_STATE,
VSTREAMS,
Wcall,
WcallClient,
WcallMember,
} from '@wireapp/avs';
import {AvsDebugger} from '@wireapp/avs-debugger';
import {Runtime} from '@wireapp/commons';
import {WebAppEvents} from '@wireapp/webapp-events';

Expand Down Expand Up @@ -256,14 +258,47 @@ export class CallingRepository {

this.onChooseScreen = (deviceId: string) => {};

// Request the video streams whenever the mode changes to active speaker
ko.computed(() => {
const call = this.callState.joinedCall();
if (!call) {
return;
}
const isSpeakersViewActive = this.callState.isSpeakersViewActive();
if (isSpeakersViewActive) {
this.requestVideoStreams(call.conversation.qualifiedId, call.activeSpeakers());
const videoQuality = call.activeSpeakers().length > 2 ? RESOLUTION.LOW : RESOLUTION.HIGH;

const speakes = call.activeSpeakers();
speakes.forEach(p => {
// This is a temporary solution. The SFT does not send a response when a track change has occurred.
// To prevent the wrong video from being briefly displayed, we introduce a timeout here.
p.isSwitchingVideoResolution(true);
window.setTimeout(() => {
p.isSwitchingVideoResolution(false);
}, 1000);
});

this.requestVideoStreams(call.conversation.qualifiedId, speakes, videoQuality);
}
});

// Request the video streams whenever toggle display maximised Participant.
ko.computed(() => {
const call = this.callState.joinedCall();
if (!call) {
return;
}
const maximizedParticipant = call.maximizedParticipant();
if (maximizedParticipant !== null) {
maximizedParticipant.isSwitchingVideoResolution(true);
// This is a temporary solution. The SFT does not send a response when a track change has occurred.
// To prevent the wrong video from being briefly displayed, we introduce a timeout here.
window.setTimeout(() => {
maximizedParticipant.isSwitchingVideoResolution(false);
}, 1000);
this.requestVideoStreams(call.conversation.qualifiedId, [maximizedParticipant], RESOLUTION.HIGH);
} else {
this.requestCurrentPageVideoStreams(call);
}
});
}
Expand Down Expand Up @@ -1204,6 +1239,7 @@ export class CallingRepository {
const conversationIdStr = this.serializeQualifiedId(conversationId);
this.wCall?.end(this.wUser, conversationIdStr);
callingSubscriptions.removeCall(conversationId);
AvsDebugger.reset();
};

private readonly leaveMLSConference = async (conversationId: QualifiedId) => {
Expand Down Expand Up @@ -1300,24 +1336,47 @@ export class CallingRepository {
this.wCall?.reject(this.wUser, this.serializeQualifiedId(conversationId));
}

/**
* This method monitors every change in the call and is therefore the main method for handling video requests.
* These changes include mute/unmute, screen sharing, or camera switching, joining or leaving of participants, or...
* @param call
* @param newPage
*/
changeCallPage(call: Call, newPage: number): void {
call.currentPage(newPage);
if (!this.callState.isSpeakersViewActive()) {
if (!this.callState.isSpeakersViewActive() && !this.callState.isMaximisedViewActive()) {
this.requestCurrentPageVideoStreams(call);
}
}

/**
* This method queries streams for the participants who are displayed on the active page! This can include up to nine
* participants and is used when flipping pages or starting a call.
* @param call
*/
requestCurrentPageVideoStreams(call: Call): void {
const currentPageParticipants = call.pages()[call.currentPage()];
this.requestVideoStreams(call.conversation.qualifiedId, currentPageParticipants);
const currentPageParticipants = call.pages()[call.currentPage()] ?? [];
const videoQuality: RESOLUTION = currentPageParticipants.length <= 2 ? RESOLUTION.HIGH : RESOLUTION.LOW;
this.requestVideoStreams(call.conversation.qualifiedId, currentPageParticipants, videoQuality);
}

requestVideoStreams(conversationId: QualifiedId, participants: Participant[]) {
requestVideoStreams(conversationId: QualifiedId, participants: Participant[], videoQuality: RESOLUTION) {
if (participants.length === 0) {
return;
}
// Filter myself out and do not request my own stream.
const requestParticipants = participants.filter(p => !this.isSelfUser(p));
if (requestParticipants.length === 0) {
return;
}

const convId = this.serializeQualifiedId(conversationId);

const payload = {
clients: participants.map(participant => ({
clients: requestParticipants.map(participant => ({
clientid: participant.clientId,
userid: this.serializeQualifiedId(participant.user.qualifiedId),
quality: videoQuality,
})),
convid: convId,
};
Expand Down Expand Up @@ -1355,6 +1414,7 @@ export class CallingRepository {
const conversationIdStr = this.serializeQualifiedId(conversationId);
delete this.poorCallQualityUsers[conversationIdStr];
this.wCall?.end(this.wUser, conversationIdStr);
AvsDebugger.reset();
};

muteCall(call: Call, shouldMute: boolean, reason?: MuteState): void {
Expand Down Expand Up @@ -2262,6 +2322,8 @@ export class CallingRepository {
this.callState
.calls()
.forEach((call: Call) => this.wCall?.end(this.wUser, this.serializeQualifiedId(call.conversation.qualifiedId)));

AvsDebugger.reset();
this.wCall?.destroy(this.wUser);
}

Expand Down Expand Up @@ -2312,6 +2374,13 @@ export class CallingRepository {
PrimaryModal.show(PrimaryModal.type.ACKNOWLEDGE, modalOptions);
}

private isSelfUser(participant: Participant): boolean {
if (this.selfUser == null || this.selfClientId == null) {
return false;
}
return participant.doesMatchIds(this.selfUser.qualifiedId, this.selfClientId);
}

//##############################################################################
// Logging
//##############################################################################
Expand Down
19 changes: 18 additions & 1 deletion src/script/calling/Participant.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@
*/

import {QualifiedId} from '@wireapp/api-client/lib/user';
import ko, {observable, pureComputed} from 'knockout';
import ko, {computed, observable, pureComputed} from 'knockout';

import {VIDEO_STATE} from '@wireapp/avs';
import {AvsDebugger} from '@wireapp/avs-debugger';

import {matchQualifiedIds} from 'Util/QualifiedId';

Expand All @@ -39,6 +40,7 @@ export class Participant {
public readonly hasPausedVideo: ko.PureComputed<boolean>;
public readonly sharesScreen: ko.PureComputed<boolean>;
public readonly sharesCamera: ko.PureComputed<boolean>;
public readonly isSwitchingVideoResolution = observable(false);
public readonly startedScreenSharingAt = observable<number>(0);
public readonly isActivelySpeaking = observable(false);
public readonly isSendingVideo: ko.PureComputed<boolean>;
Expand Down Expand Up @@ -67,6 +69,18 @@ export class Participant {
this.isSendingVideo = pureComputed(() => {
return this.videoState() !== VIDEO_STATE.STOPPED;
});
this.isSwitchingVideoResolution(false);

computed(() => {
const stream = this.videoStream();

if (stream && stream.getVideoTracks().length > 0) {
if (AvsDebugger.hasTrack(this.user.id)) {
AvsDebugger.removeTrack(this.user.id);
}
AvsDebugger.addTrack(this.user.id, this.user.name(), stream.getVideoTracks()[0]);
}
});
}

public releaseBlurredVideoStream(): void {
Expand Down Expand Up @@ -141,6 +155,9 @@ export class Participant {
track.stop();
}
mediaStream.removeTrack(track);
if (track.kind == 'video' && AvsDebugger.hasTrack(this.user.id)) {
AvsDebugger.removeTrack(this.user.id);
}
});
}
}
27 changes: 27 additions & 0 deletions src/script/components/ConfigToolbar/ConfigToolbar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ export function ConfigToolbar() {
const messageCountRef = useRef<number>(0); // For the message count
const [prefix, setPrefix] = useState('Message -'); // Prefix input
const wrapperRef = useRef(null);
const [avsDebuggerEnabled, setAvsDebuggerEnabled] = useState(!!window.wire?.app?.debug?.isEnabledAvsDebugger()); //

// Toggle config tool on 'cmd/ctrl + shift + 2'
useEffect(() => {
Expand Down Expand Up @@ -160,6 +161,25 @@ export function ConfigToolbar() {

useClickOutside(wrapperRef, () => setShowConfig(false));

const handleAvsEnable = (isChecked: boolean) => {
setAvsDebuggerEnabled(!!window.wire?.app?.debug?.enableAvsDebugger(isChecked));
};

const renderAvsSwitch = (value: boolean) => {
return (
<div style={{marginBottom: '10px'}}>
<label htmlFor="avs-debugger-checkbox" style={{display: 'block', fontWeight: 'bold'}}>
ENABLE AVS TRACK DEBUGGER
</label>
<Switch
id="avs-debugger-checkbox"
checked={avsDebuggerEnabled}
onToggle={isChecked => handleAvsEnable(isChecked)}
/>
</div>
);
};

if (!showConfig) {
return null;
}
Expand All @@ -173,11 +193,18 @@ export function ConfigToolbar() {
</h4>
<div>{renderConfig(configFeaturesState)}</div>

<hr />

<h3>Debug Functions</h3>

<Button onClick={() => window.wire?.app?.debug?.reconnectWebSocket()}>reconnectWebSocket</Button>
<Button onClick={() => window.wire?.app?.debug?.enablePushToTalk()}>enablePushToTalk</Button>
<Button onClick={() => window.wire?.app?.debug?.enablePushToTalk(null)}>disablePushToTalk</Button>

<div>{renderAvsSwitch(avsDebuggerEnabled)}</div>

<hr />

<h3>Message Automation</h3>
<Input
type="text"
Expand Down
30 changes: 19 additions & 11 deletions src/script/components/calling/GroupVideoGridTile.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,23 @@ const GroupVideoGridTile: React.FC<GroupVideoGridTileProps> = ({
isMaximized,
onTileDoubleClick,
}) => {
const {isMuted, videoState, videoStream, blurredVideoStream, isActivelySpeaking, isAudioEstablished} =
useKoSubscribableChildren(participant, [
'isMuted',
'videoStream',
'blurredVideoStream',
'isActivelySpeaking',
'videoState',
'isAudioEstablished',
]);
const {
isMuted,
videoState,
videoStream,
blurredVideoStream,
isActivelySpeaking,
isAudioEstablished,
isSwitchingVideoResolution,
} = useKoSubscribableChildren(participant, [
'isMuted',
'videoStream',
'blurredVideoStream',
'isActivelySpeaking',
'videoState',
'isAudioEstablished',
'isSwitchingVideoResolution',
]);
const {name} = useKoSubscribableChildren(participant?.user, ['name']);

const sharesScreen = videoState === VIDEO_STATE.SCREENSHARE;
Expand Down Expand Up @@ -220,7 +228,7 @@ const GroupVideoGridTile: React.FC<GroupVideoGridTileProps> = ({

{nameContainer}

{hasPausedVideo && (
{(hasPausedVideo || isSwitchingVideoResolution) && (
<div className="group-video-grid__pause-overlay">
<div className="background">
<div className="background-image"></div>
Expand All @@ -232,7 +240,7 @@ const GroupVideoGridTile: React.FC<GroupVideoGridTileProps> = ({
css={{fontsize: minimized ? '0.6875rem' : '0.875rem'}}
data-uie-name="status-video-paused"
>
{t('videoCallPaused')}
{hasPausedVideo ? t('videoCallPaused') : t('videoCallParticipantConnecting')}
</div>
{nameContainer}
</div>
Expand Down
1 change: 1 addition & 0 deletions src/script/team/TeamState.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ export class TeamState {
this.isConferenceCallingEnabled = ko.pureComputed(
() => this.teamFeatures()?.conferenceCalling?.status === FeatureStatus.ENABLED,
);

this.isGuestLinkEnabled = ko.pureComputed(
() => this.teamFeatures()?.conversationGuestLinks?.status === FeatureStatus.ENABLED,
);
Expand Down
Loading

0 comments on commit ce47d8f

Please sign in to comment.