+
+
+ video codec
+
+
+
+
+ audio codec
+
+
+
+
+
SVC Level
@@ -125,7 +165,7 @@
const DEVICE = 1;
const TRANSMITTING = 2;
const ERROR = 3;
-
+
let state = INITIALIZING;
let errorMessage = '';
@@ -134,16 +174,16 @@
for (const el of ['initializing', 'device', 'transmitting', 'error']) {
document.getElementById(el).style.display = 'none';
}
-
+
switch (state) {
case DEVICE:
document.getElementById('device').style.display = 'flex';
break;
-
+
case TRANSMITTING:
document.getElementById('transmitting').style.display = 'flex';
break;
-
+
case ERROR:
document.getElementById('error').style.display = 'flex';
document.getElementById('error-message').innerHTML = 'error: ' + errorMessage;
@@ -152,54 +192,54 @@
};
// populateDevices
const populateDevices = () => {
- return navigator.mediaDevices.enumerateDevices()
- .then((devices) => {
- for (const device of devices) {
- switch (device.kind) {
- case 'videoinput':
- {
- const opt = document.createElement('option');
- opt.value = device.deviceId;
- opt.text = device.label;
- document.getElementById('video_device').appendChild(opt);
- }
- break;
-
- case 'audioinput':
- {
- const opt = document.createElement('option');
- opt.value = device.deviceId;
- opt.text = device.label;
- document.getElementById('audio_device').appendChild(opt);
+ return navigator.mediaDevices.enumerateDevices()
+ .then((devices) => {
+ for (const device of devices) {
+ switch (device.kind) {
+ case 'videoinput':
+ {
+ const opt = document.createElement('option');
+ opt.value = device.deviceId;
+ opt.text = device.label;
+ document.getElementById('video_device').appendChild(opt);
+ }
+ break;
+
+ case 'audioinput':
+ {
+ const opt = document.createElement('option');
+ opt.value = device.deviceId;
+ opt.text = device.label;
+ document.getElementById('audio_device').appendChild(opt);
+ }
+ break;
}
- break;
}
- }
-
- if (navigator.mediaDevices.getDisplayMedia !== undefined) {
- const opt = document.createElement('option');
- opt.value = "screen";
- opt.text = "screen";
- document.getElementById('video_device').appendChild(opt);
- }
-
- if (document.getElementById('video_device').children.length !== 0) {
- document.getElementById('video_device').value = document.getElementById('video_device').children[1].value;
- }
-
- if (document.getElementById('audio_device').children.length !== 0) {
- document.getElementById('audio_device').value = document.getElementById('audio_device').children[1].value;
- }
- });
- };
+
+ if (navigator.mediaDevices.getDisplayMedia !== undefined) {
+ const opt = document.createElement('option');
+ opt.value = "screen";
+ opt.text = "screen";
+ document.getElementById('video_device').appendChild(opt);
+ }
+
+ if (document.getElementById('video_device').children.length !== 0) {
+ document.getElementById('video_device').value = document.getElementById('video_device').children[1].value;
+ }
+
+ if (document.getElementById('audio_device').children.length !== 0) {
+ document.getElementById('audio_device').value = document.getElementById('audio_device').children[1].value;
+ }
+ });
+ };
// onPublish
const onPublish = () => {
state = TRANSMITTING;
render();
-
+
const videoId = document.getElementById('video_device').value;
const audioId = document.getElementById('audio_device').value;
-
+
if (videoId !== 'screen') {
let video = false;
if (videoId !== 'none') {
@@ -207,24 +247,24 @@
deviceId: videoId,
};
}
-
+
let audio = false;
-
+
if (audioId !== 'none') {
audio = {
deviceId: audioId,
};
-
- // const voice = document.getElementById('audio_voice').checked;
- // if (!voice) {
- // audio.autoGainControl = false;
- // audio.echoCancellation = false;
- // audio.noiseSuppression = false;
- // }
+
+ const voice = document.getElementById('audio_voice').checked;
+ if (!voice) {
+ audio.autoGainControl = false;
+ audio.echoCancellation = false;
+ audio.noiseSuppression = false;
+ }
}
-
+
navigator.mediaDevices.getUserMedia({ video, audio })
- .then(onTransmit)
+ .then(onTransmitSVC)
.catch((err) => {
state = ERROR;
errorMessage = err.toString();
@@ -248,61 +288,64 @@
});
}
};
- // onTransmit
- const onTransmit = (stream) => {
+ // onTransmitSVC
+ const onTransmitSVC = (stream) => {
document.getElementById('video').srcObject = stream;
const resource = document.getElementById("resource").value;
//get svc_level
const rid_value = document.getElementById("svc_level").value;
var sendEncodings = [];
- if (rid_value == 'a') {
+ if (rid_value=="a") {
sendEncodings = [
- { rid: 'a', scaleResolutionDownBy: 2.0 },
- { rid: 'b', scaleResolutionDownBy: 1.0 },
- { rid: 'c' }
+ { rid:"a", scaleResolutionDownBy: 1.0 },
+ { rid:"b", scaleResolutionDownBy: 2.0 },
]
- }else if (rid_value == 'b') {
- sendEncodings = [
- { rid: 'b', scaleResolutionDownBy: 1.0 },
- { rid: 'c' }
- ]
- }else{
- sendEncodings = [
- { rid: 'c' }
+ }else {
+ sendEncodings = [
+ { rid:"b", scaleResolutionDownBy: 1.0 },
]
- }
+ }
+
const pc = new RTCPeerConnection();
pc.addTransceiver(stream.getVideoTracks()[0], {
direction: 'sendonly',
sendEncodings: sendEncodings
- // sendEncodings: [
- // { rid: 'a', scaleResolutionDownBy: 2.0 },
- // { rid: 'b', scaleResolutionDownBy: 1.0, },
- // { rid: 'c' }
- // ]
});
const whip = new WHIPClient();
const url = location.origin + "/whip/" + resource;
const token = document.getElementById("token").value;
- whip.publish(pc, url, token);
+ whip.publish(pc, url, token);
};
- // onTransmitSVC
- const onTransmitSVC = (stream) => {
- document.getElementById('video').srcObject = stream;
- const resource = document.getElementById("resource").value;
- const pc = new RTCPeerConnection();
- pc.addTransceiver(stream.getVideoTracks()[0], {
- direction: 'sendonly',
- sendEncodings: [
- { rid: 'a', scaleResolutionDownBy: 2.0 },
- { rid: 'b', scaleResolutionDownBy: 1.0, },
- { rid: 'c' }
- ]
- });
- const whip = new WHIPClient();
- const url = location.origin + "/whip/" + resource;
- const token = document.getElementById("token").value;
- whip.publish(pc, url, token);
+ // populateCodecs
+ const populateCodecs = () => {
+ const pc = new RTCPeerConnection({});
+ pc.addTransceiver("video", { direction: 'sendonly' });
+ pc.addTransceiver("audio", { direction: 'sendonly' });
+
+ return pc.createOffer()
+ .then((desc) => {
+ const sdp = desc.sdp.toLowerCase();
+
+ for (const codec of ['av1/90000', 'vp9/90000', 'vp8/90000', 'h264/90000']) {
+ if (sdp.includes(codec)) {
+ const opt = document.createElement('option');
+ opt.value = codec;
+ opt.text = codec.split('/')[0].toUpperCase();
+ document.getElementById('video_codec').appendChild(opt);
+ }
+ }
+
+ for (const codec of ['opus/48000', 'g722/8000', 'pcmu/8000', 'pcma/8000']) {
+ if (sdp.includes(codec)) {
+ const opt = document.createElement('option');
+ opt.value = codec;
+ opt.text = codec.split('/')[0].toUpperCase();
+ document.getElementById('audio_codec').appendChild(opt);
+ }
+ }
+
+ pc.close();
+ });
};
// initialize
const initialize = () => {
@@ -312,10 +355,11 @@
render();
return;
}
-
+
navigator.mediaDevices.getUserMedia({ video: true, audio: true })
.then(() => Promise.all([
populateDevices(),
+ populateCodecs(),
]))
.then(() => {
state = DEVICE;
@@ -338,7 +382,7 @@
onPublish();
}
-
+
async function startWhep() {
document.getElementById('video').style.display = 'none';
document.getElementById('remoteVideos').style.display = 'block';
@@ -360,13 +404,13 @@
el.style.height = "100%"
document.getElementById('remoteVideos').appendChild(el)
}
- // if (event.track.kind == "audio") {
- // var el = document.createElement(event.track.kind)
- // el.srcObject = event.streams[0]
- // el.autoplay = true
- // el.controls = true
- // document.getElementById('remoteVideos').appendChild(el)
- // }
+ if (event.track.kind == "audio") {
+ var el = document.createElement(event.track.kind)
+ el.srcObject = event.streams[0]
+ el.autoplay = true
+ el.controls = true
+ document.getElementById('remoteVideos').appendChild(el)
+ }
}
const whep = new WHEPClient();
const url = location.origin + "/whep/" + resource;
@@ -375,7 +419,8 @@
}
initialize();
-
+
+
\ No newline at end of file
diff --git a/assets/whip.js b/assets/whip.js
index 6ef9727f..c952b027 100644
--- a/assets/whip.js
+++ b/assets/whip.js
@@ -12,6 +12,112 @@ class WHIPClient {
}
async publish(pc, url, token) {
+ // edit answer_sdp
+ const onRemoteAnswer=(answer_sdp) =>{
+
+ const answer_modified = editAnswer(
+ answer_sdp,
+ document.getElementById('video_codec').value,
+ document.getElementById('audio_codec').value,
+ document.getElementById('video_bitrate').value,
+ document.getElementById('audio_bitrate').value,
+ document.getElementById('audio_voice').value,
+ );
+ return answer_modified;
+ }
+ const editAnswer = (answer_sdp, videoCodec, audioCodec, videoBitrate, audioBitrate, audioVoice) => {
+ const sections = answer_sdp.split('m=');
+ for (let i = 0; i < sections.length; i++) {
+ const section = sections[i];
+ if (section.startsWith('video')) {
+ sections[i] = setVideoBitrate(setCodec(section, videoCodec), videoBitrate);
+ } else if (section.startsWith('audio')) {
+ sections[i] = setAudioBitrate(setCodec(section, audioCodec), audioBitrate, audioVoice);
+ }
+ }
+
+ const answer_modified = sections.join('m=');
+ console.log(answer_modified);
+ return answer_modified;
+ };
+
+ const setVideoBitrate = (section, bitrate) => {
+ let lines = section.split('\r\n');
+
+ for (let i = 0; i < lines.length; i++) {
+ if (lines[i].startsWith('c=')) {
+ lines = [...lines.slice(0, i+1), 'b=TIAS:' + (parseInt(bitrate) * 1024).toString(), ...lines.slice(i+1)];
+ break
+ }
+ }
+
+ return lines.join('\r\n');
+ };
+
+ const setAudioBitrate = (section, bitrate, voice) => {
+ let opusPayloadFormat = '';
+ let lines = section.split('\r\n');
+
+ for (let i = 0; i < lines.length; i++) {
+ if (lines[i].startsWith('a=rtpmap:') && lines[i].toLowerCase().includes('opus/')) {
+ opusPayloadFormat = lines[i].slice('a=rtpmap:'.length).split(' ')[0];
+ break;
+ }
+ }
+
+ if (opusPayloadFormat === '') {
+ return section;
+ }
+
+ for (let i = 0; i < lines.length; i++) {
+ if (lines[i].startsWith('a=fmtp:' + opusPayloadFormat + ' ')) {
+ if (voice) {
+ lines[i] = 'a=fmtp:' + opusPayloadFormat + ' minptime=10;useinbandfec=1;maxaveragebitrate='
+ + (parseInt(bitrate) * 1024).toString();
+ } else {
+ lines[i] = 'a=fmtp:' + opusPayloadFormat + ' maxplaybackrate=48000;stereo=1;sprop-stereo=1;maxaveragebitrate'
+ + (parseInt(bitrate) * 1024).toString();
+ }
+ }
+ }
+
+ return lines.join('\r\n');
+ };
+
+ const setCodec = (section, codec) => {
+ const lines = section.split('\r\n');
+ const lines2 = [];
+ const payloadFormats = [];
+
+ for (const line of lines) {
+ if (!line.startsWith('a=rtpmap:')) {
+ lines2.push(line);
+ } else {
+ if (line.toLowerCase().includes(codec)) {
+ payloadFormats.push(line.slice('a=rtpmap:'.length).split(' ')[0]);
+ lines2.push(line);
+ }
+ }
+ }
+
+ const lines3 = [];
+
+ for (const line of lines2) {
+ if (line.startsWith('a=fmtp:')) {
+ if (payloadFormats.includes(line.slice('a=fmtp:'.length).split(' ')[0])) {
+ lines3.push(line);
+ }
+ } else if (line.startsWith('a=rtcp-fb:')) {
+ if (payloadFormats.includes(line.slice('a=rtcp-fb:'.length).split(' ')[0])) {
+ lines3.push(line);
+ }
+ } else {
+ lines3.push(line);
+ }
+ }
+
+ return lines3.join('\r\n');
+ };
//If already publishing
if (this.pc)
throw new Error("Already publishing")
@@ -187,9 +293,16 @@ class WHIPClient {
this.iceUsername = offer.sdp.match(/a=ice-ufrag:(.*)\r\n/)[1];
this.icePassword = offer.sdp.match(/a=ice-pwd:(.*)\r\n/)[1];
//}
-
+
//And set remote description
- await pc.setRemoteDescription({type: "answer", sdp: answer});
+ const answer_sdp = onRemoteAnswer(answer);
+ console.log(answer_sdp);
+ await pc.setRemoteDescription({type: "answer", sdp: answer_sdp});
+
+
+
+
+
}
restart() {
@@ -372,4 +485,6 @@ class WHIPClient {
headers
});
}
+
+
};
\ No newline at end of file
From ae6c806e2d0a892a6606728e78fba6fd7a11797e Mon Sep 17 00:00:00 2001
From: fastfeee <3307603357@qq.com>
Date: Thu, 26 Oct 2023 10:52:01 +0800
Subject: [PATCH 2/7] feat(assets): add multi video codec and bitrate
But the svc is still :construction: .
---
assets/index.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/assets/index.html b/assets/index.html
index ad0b324b..a035b0e2 100644
--- a/assets/index.html
+++ b/assets/index.html
@@ -302,7 +302,7 @@
]
}else {
sendEncodings = [
- { rid:"b", scaleResolutionDownBy: 1.0 },
+ { rid:"b", scaleResolutionDownBy: 2.0 },
]
}
From b10d83f9a62912670bb85cecd968406dc692458d Mon Sep 17 00:00:00 2001
From: fastfeee <3307603357@qq.com>
Date: Mon, 30 Oct 2023 10:32:08 +0800
Subject: [PATCH 3/7] fix(index.html): fix the audio input and output
separation
---
assets/index.html | 175 +++++++++++++++++++++++++---------------------
1 file changed, 96 insertions(+), 79 deletions(-)
diff --git a/assets/index.html b/assets/index.html
index a035b0e2..7b61f9ae 100644
--- a/assets/index.html
+++ b/assets/index.html
@@ -77,15 +77,19 @@