From 616a9463912f425d3663cad80cf290fef30626db Mon Sep 17 00:00:00 2001 From: Alex Shortt Date: Tue, 6 Sep 2022 17:07:03 -0700 Subject: [PATCH 1/4] Network Audio Fixes (pt 2) (#112) * replace voiceStreams with mediaConnections, close connections when disabling voice, refactor updating logic * remove listeners on enable change * clean up some code - remove usecallbacks * attempt a mic fix --- .../Multiplayer/ideas/PingPongMulti.tsx | 15 +++ examples/worlds/Multiplayer/index.tsx | 2 + .../Network/ideas/NetworkedEntities/index.tsx | 42 ++++--- .../ideas/NetworkedEntities/logic/entity.ts | 82 ++++++------- src/layers/Network/logic/connection.ts | 14 ++- src/layers/Network/logic/mic.ts | 54 +++++++++ src/layers/Network/logic/voice.ts | 112 +++++++++--------- 7 files changed, 195 insertions(+), 126 deletions(-) create mode 100644 examples/worlds/Multiplayer/ideas/PingPongMulti.tsx create mode 100644 src/layers/Network/logic/mic.ts diff --git a/examples/worlds/Multiplayer/ideas/PingPongMulti.tsx b/examples/worlds/Multiplayer/ideas/PingPongMulti.tsx new file mode 100644 index 00000000..6695b284 --- /dev/null +++ b/examples/worlds/Multiplayer/ideas/PingPongMulti.tsx @@ -0,0 +1,15 @@ +import { useNetwork } from "spacesvr"; +import { useEffect } from "react"; + +export default function PingPongMulti() { + const { voice, setVoice } = useNetwork(); + + useEffect(() => { + setTimeout(() => { + console.log("setting to ", !voice); + setVoice(!voice); + }, 5000); + }, [setVoice, voice]); + + return null; +} diff --git a/examples/worlds/Multiplayer/index.tsx b/examples/worlds/Multiplayer/index.tsx index 5ff96a2f..9186df60 100644 --- a/examples/worlds/Multiplayer/index.tsx +++ b/examples/worlds/Multiplayer/index.tsx @@ -1,5 +1,6 @@ import { StandardReality, Background, Model } from "spacesvr"; import LightSwitch from "./ideas/LightSwitch"; +import PingPongMulti from "./ideas/PingPongMulti"; export default function Multiplayer() { return ( @@ -7,6 +8,7 @@ export default function Multiplayer() { playerProps={{ pos: [5, 1, 0], rot: Math.PI }} networkProps={{ autoconnect: true, voice: true }} > + {/**/} diff --git a/src/layers/Network/ideas/NetworkedEntities/index.tsx b/src/layers/Network/ideas/NetworkedEntities/index.tsx index b501e486..a2bfdf72 100644 --- a/src/layers/Network/ideas/NetworkedEntities/index.tsx +++ b/src/layers/Network/ideas/NetworkedEntities/index.tsx @@ -49,13 +49,11 @@ export default function NetworkedEntities() { }, })); - const snapshot: Snapshot = { + SI.vault.add({ id: Math.random().toString(), time: new Date().getTime(), state, - }; - - SI.vault.add(snapshot); + }); }); // send own player data @@ -77,23 +75,27 @@ export default function NetworkedEntities() { let i = 0; for (const entityState of snapshot.state) { const { x, y, z, q } = entityState; - obj.position.x = x as number; - obj.position.y = y as number; - obj.position.z = z as number; + obj.position.set(x as number, y as number, z as number); obj.position.y -= 0.2; // they were floating before, idk where the constant comes from really const quat = q as Quat; - obj.quaternion.x = quat.x; - obj.quaternion.y = quat.y; - obj.quaternion.z = quat.z; - obj.quaternion.w = quat.w; + obj.quaternion.set( + quat.x as number, + quat.y as number, + quat.z as number, + quat.w as number + ); obj.updateMatrix(); mesh.current.setMatrixAt(i, obj.matrix); - const audio = entities[i]?.posAudio; - if (audio) { - obj.matrix.decompose(audio.position, audio.quaternion, audio.scale); - audio.updateMatrix(); - audio.rotateY(Math.PI); // for some reason it's flipped + const posAudio = entities[i]?.posAudio; + if (posAudio) { + obj.matrix.decompose( + posAudio.position, + posAudio.quaternion, + posAudio.scale + ); + posAudio.rotation.y += Math.PI; // for some reason it's flipped + posAudio.updateMatrix(); } i++; @@ -107,11 +109,15 @@ export default function NetworkedEntities() { } return ( - + {entities.map( (entity) => entity.posAudio && ( - + ) )} { - const { connections, connected, voiceStreams } = useNetwork(); + const { connections, connected, mediaConnections } = useNetwork(); const { paused } = useEnvironment(); const listener = useListener(); @@ -23,61 +23,54 @@ export const useEntities = (): Entity[] => { const entities = useMemo(() => [], []); - const sameIds = (ids1: string[], ids2: string[]) => - ids1.sort().join(",") === ids2.sort().join(","); + const needsAudio = (e: Entity) => mediaConnections.has(e.id) && !e.posAudio; // check for a change in player list, re-render if there is a change - const connectionIds = useRef([]); - const voiceIds = useRef([]); - useLimitedFrame(6, () => { + useLimitedFrame(5, () => { if (!connected) return; - // check for changes in connections - if (!sameIds(connectionIds.current, Array.from(connections.keys()))) { - connectionIds.current = Array.from(connections.keys()); + // changed flag to trigger re-render at the end + let changed = false; - // remove entities that are no longer connected - entities.map((e) => { - if (!connectionIds.current.includes(e.id)) { - entities.splice(entities.indexOf(e), 1); - } - }); - - // add in new entities - for (const id of connectionIds.current) { - if (!entities.some((e) => e.id === id)) { - entities.push({ id, posAudio: undefined }); + // remove old entities + entities.map((e) => { + if (!connections.has(e.id)) { + if (e.posAudio) { + e.posAudio.remove(); + e.posAudio = undefined; } + entities.splice(entities.indexOf(e), 1); + changed = true; } + }); - rerender(); + // add in new entities + for (const id of Array.from(connections.keys())) { + if (!entities.some((e) => e.id === id)) { + entities.push({ id, posAudio: undefined }); + changed = true; + } } // dont run until first time unpaused to make sure audio context is running from first press - if ( - !firstPaused && - !sameIds(voiceIds.current, Array.from(voiceStreams.keys())) - ) { - voiceIds.current = Array.from(voiceStreams.keys()); - - // remove voice streams that are no longer connected + if (!firstPaused) { + // remove media connections streams that are no longer connected entities.map((e) => { - if (!voiceIds.current.includes(e.id)) { + if (!mediaConnections.has(e.id)) { e.posAudio?.remove(); e.posAudio = undefined; + changed = true; } }); - // add in new voice streams - for (const id of voiceIds.current) { - const entity = entities.find((e) => e.id === id); - if (!entity) continue; - - const stream = voiceStreams.get(id)!; - if (!stream) continue; + entities.filter(needsAudio).map((e) => { + // add in new media connections if the stream is active + const mediaConn = mediaConnections.get(e.id); + if (!mediaConn) return; + if (!mediaConn.remoteStream) return; const audioElem = document.createElement("audio"); - audioElem.srcObject = stream; + audioElem.srcObject = mediaConn.remoteStream; // remote is incoming, local is own voice audioElem.muted = true; audioElem.autoplay = true; audioElem.loop = true; @@ -85,18 +78,19 @@ export const useEntities = (): Entity[] => { audioElem.playsInline = true; const posAudio = new PositionalAudio(listener); - posAudio.userData.peerId = id; - posAudio.setMediaStreamSource(stream); + posAudio.userData.peerId = e.id; + posAudio.setMediaStreamSource(audioElem.srcObject); posAudio.setRefDistance(2); - posAudio.setDirectionalCone(200, 290, 0.2); - posAudio.setVolume(0.6); + posAudio.setDirectionalCone(200, 290, 0.35); // posAudio.add(new PositionalAudioHelper(posAudio, 1)); - entity.posAudio = posAudio; - } + e.posAudio = posAudio; - rerender(); + changed = true; + }); } + + if (changed) rerender(); }); return entities; diff --git a/src/layers/Network/logic/connection.ts b/src/layers/Network/logic/connection.ts index f3c8a2bb..ac352455 100644 --- a/src/layers/Network/logic/connection.ts +++ b/src/layers/Network/logic/connection.ts @@ -1,19 +1,19 @@ import { useEffect, useMemo, useState } from "react"; -import { DataConnection, Peer } from "peerjs"; +import { DataConnection, MediaConnection, Peer } from "peerjs"; import { isLocalNetwork } from "./local"; import { LocalSignaller } from "./signallers/LocalSignaller"; import { MuseSignaller } from "./signallers/MuseSignaller"; import { useWaving } from "./wave"; import { Signaller, SignallerConfig } from "./signallers"; import { Channels, useChannels } from "./channels"; -import { useVoice } from "./voice"; +import { useVoiceConnections } from "./voice"; import { getMuseIceServers } from "./ice"; export type ConnectionState = { connected: boolean; connect: (config?: ConnectionConfig) => Promise; connections: Map; - voiceStreams: Map; + mediaConnections: Map; disconnect: () => void; voice: boolean; setVoice: (v: boolean) => void; @@ -43,6 +43,10 @@ export const useConnection = ( console.log("connection closed with peer"); connections.delete(conn.peer); }); + conn.on("error", () => { + console.log("connection closed with peer"); + connections.delete(conn.peer); + }); channels.greet(conn); connections.set(conn.peer, conn); }); @@ -125,16 +129,16 @@ export const useConnection = ( const [voice, setVoice] = useState(!!externalConfig.voice); useEffect(() => setVoice(!!externalConfig.voice), [externalConfig.voice]); - const voiceStreams = useVoice(voice, peer, connections); + const mediaConnections = useVoiceConnections(voice, peer, connections); return { connected, connect, disconnect, connections, - voiceStreams, useChannel: channels.useChannel, voice, setVoice, + mediaConnections, }; }; diff --git a/src/layers/Network/logic/mic.ts b/src/layers/Network/logic/mic.ts new file mode 100644 index 00000000..356ce937 --- /dev/null +++ b/src/layers/Network/logic/mic.ts @@ -0,0 +1,54 @@ +import { useEffect, useState } from "react"; +import { useEnvironment } from "../../Environment"; + +/** + * WHen enabled, will ask user for mic permissions and return the local microphone stream + * @param enabled + */ +export const useMicrophone = (enabled = true): MediaStream | undefined => { + const { paused } = useEnvironment(); + const [firstPaused, setFirstPaused] = useState(true); + useEffect(() => setFirstPaused(paused && firstPaused), [paused, firstPaused]); + + function iOS() { + return ( + [ + "iPad Simulator", + "iPhone Simulator", + "iPod Simulator", + "iPad", + "iPhone", + "iPod", + ].includes(navigator.platform) || + // iPad on iOS 13 detection + (navigator.userAgent.includes("Mac") && "ontouchend" in document) + ); + } + + const [localStream, setLocalStream] = useState(); + + // attempt to request permission for microphone, only try once + const [attempted, setAttempted] = useState(false); + useEffect(() => { + // https://bugs.webkit.org/show_bug.cgi?id=230902#c47 + if (!enabled || attempted || (iOS() && firstPaused)) return; + + setAttempted(true); + + navigator.getUserMedia = + navigator.getUserMedia || + navigator.webkitGetUserMedia || + navigator.mozGetUserMedia || + navigator.msGetUserMedia; + + navigator.getUserMedia( + { audio: true }, + (str) => setLocalStream(str), + (err) => { + console.error(err); + } + ); + }, [attempted, enabled, firstPaused]); + + return localStream; +}; diff --git a/src/layers/Network/logic/voice.ts b/src/layers/Network/logic/voice.ts index cc99285a..1327a702 100644 --- a/src/layers/Network/logic/voice.ts +++ b/src/layers/Network/logic/voice.ts @@ -1,5 +1,7 @@ import { DataConnection, Peer, MediaConnection } from "peerjs"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; +import { useMicrophone } from "./mic"; +import { useEnvironment } from "../../Environment"; type GetUserMedia = ( options: { video?: boolean; audio?: boolean }, @@ -16,86 +18,78 @@ declare global { } } -export const useVoice = ( +/** + * When enabled, is responsible for requesting mic permissions, calling and answering peers to create media connections, + * and closing media connections on disable + * + * @param enabled + * @param peer + * @param connections + */ +export const useVoiceConnections = ( enabled: boolean, peer: Peer | undefined, connections: Map -): Map => { - const [stream, setStream] = useState(); - const voiceStreams = useMemo>( - () => new Map(), - [] - ); +): Map => { + const mediaConns = useMemo>(() => new Map(), []); - // attempt to request permission for microphone, only try once - const [attempted, setAttempted] = useState(false); - useEffect(() => { - if (!enabled || attempted) return; - - setAttempted(true); + const localStream = useMicrophone(enabled); - navigator.getUserMedia = - navigator.getUserMedia || - navigator.webkitGetUserMedia || - navigator.mozGetUserMedia || - navigator.msGetUserMedia; + // handle calling and answering peers + useEffect(() => { + if (!peer || !localStream) return; - navigator.getUserMedia( - { audio: true }, - (str) => setStream(str), - (err) => { - console.error(err); - } - ); - }, [attempted, enabled, peer]); + const call = (conn: DataConnection) => { + console.log("calling peer with id", conn.peer); + handleMediaConn(peer.call(conn.peer, localStream)); + conn.on("close", () => { + console.log("closing voice stream with peer", conn.peer); + mediaConns.delete(conn.peer); + }); + }; - const handleMediaConn = useCallback( - (mediaConn: MediaConnection) => { + // handle a new media connection (incoming or created + const handleMediaConn = (mediaConn: MediaConnection) => { console.log("media connection opened with peer", mediaConn.peer); - mediaConn.answer(stream); - - mediaConn.on("stream", (str: MediaStream) => { - voiceStreams.set(mediaConn.peer, str); - }); + mediaConn.answer(localStream); + mediaConns.set(mediaConn.peer, mediaConn); mediaConn.on("close", () => { console.log("closing voice stream with peer", mediaConn.peer); - voiceStreams.delete(mediaConn.peer); + mediaConns.delete(mediaConn.peer); }); mediaConn.on("error", (err: any) => { console.error("error with voice stream with peer", mediaConn.peer, err); - voiceStreams.delete(mediaConn.peer); + mediaConns.delete(mediaConn.peer); }); - }, - [stream, voiceStreams] - ); - - const callPeer = useCallback( - (conn: DataConnection, peer: Peer, stream: MediaStream) => { - console.log("calling peer with id", conn.peer); - handleMediaConn(peer.call(conn.peer, stream)); - conn.on("close", () => { - console.log("closing voice stream with peer", conn.peer); - voiceStreams.delete(conn.peer); - }); - }, - [handleMediaConn, voiceStreams] - ); - - useEffect(() => { - if (!peer || !stream) return; + }; // set up incoming and outgoing calls for any future connections peer.on("call", handleMediaConn); - peer.on("connection", (conn) => callPeer(conn, peer, stream)); + peer.on("connection", call); // call any already connected peers for (const [peerId, conn] of connections) { - if (voiceStreams.has(peerId)) return; - callPeer(conn, peer, stream); + if (mediaConns.has(peerId)) return; + call(conn); + } + + return () => { + peer.removeListener("call", handleMediaConn); + peer.removeListener("connection", call); + }; + }, [connections, peer, localStream, mediaConns, enabled]); + + // close all media connections with peers on disable + useEffect(() => { + if (!enabled) { + mediaConns.forEach((conn) => { + conn.close(); + mediaConns.delete(conn.peer); + }); } - }, [callPeer, connections, handleMediaConn, peer, stream, voiceStreams]); + }, [enabled, mediaConns]); - return voiceStreams; + return mediaConns; }; From ab3b2d5adb880bb65e9da2516c95a4567c59576c Mon Sep 17 00:00:00 2001 From: Alex Shortt Date: Wed, 7 Sep 2022 02:08:42 +0200 Subject: [PATCH 2/4] update readme to mach past changes --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index ab359630..db4463d0 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,9 @@ type NetworkState = { connect: (config?: ConnectionConfig) => Promise; // when autoconnect is off, use this to manually connect connections: Map; // reference to active peer connections disconnect: () => void; + voice: boolean; // whether voice is enabled + setVoice: (v: boolean) => void; // enable/disable voice + mediaConnections: Map; // reference to active media connections useChannel: ( id: string, type: ChannelType, @@ -382,6 +385,17 @@ Adds an infinite plane to walk on (added by default with the Environment Layer) /> ``` +#### Model + +Quickly add a GLTF/GLB model to your scene. Will handle Suspense, KTX2, Draco, Meshopt. + +```tsx + +``` + #### Video Add a video file to your space with positional audio. Handles media playback rules for Safari, iOS, etc. From 1fe18ef0dfe38f50efe65f8f00386e8f2ef835ff Mon Sep 17 00:00:00 2001 From: Alex Shortt Date: Wed, 7 Sep 2022 02:12:44 +0200 Subject: [PATCH 3/4] smol fixes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index db4463d0..c249ce25 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ spacesvr
- A standardized reality for future of the 3D Web. + A standardized reality for the future of the 3D Web.
@@ -37,7 +37,7 @@ The mission of spacesvr is to organize and implement the standards for experiencing 3D content on the web in the same way that there exists standards for experiencing 2D content with HTML/CSS/JS. -spacesvr is designed to empower the artist. Instead of worrying about file structures or basic functionality like cross-device compatability, artists should spend their time telling their story. As such, consumption is optimized for simplicity, and the organization provides a framework to mediate along. +spacesvr is designed to empower the artist. Instead of worrying about file structures or basic functionality like cross-device compatability, artists should spend their time telling their story. As such, consumption is optimized for simplicity, and the organization provides a framework to tell stories. spacesvr is actively maintained by [Muse](https://www.muse.place?utm_source=npmjs&utm_campaign=learn_more), a YC-backed startup that provides tooling for visually building worlds. Muse's mission is to accelerate the adoption of 3D websites by increasing their accessibility, both for the end user and for the creator. Muse is completely built on spacesvr. From e95b0c79ec44e473dfe5a0908e5a454dc3988305 Mon Sep 17 00:00:00 2001 From: Alex Shortt Date: Wed, 7 Sep 2022 02:24:16 +0200 Subject: [PATCH 4/4] v2.3.2 --- package.json | 2 +- src/layers/Environment/ui/PauseMenu/index.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 33dba6ef..a093fb91 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "spacesvr", - "version": "2.3.1", + "version": "2.3.2", "private": true, "description": "A standardized reality for future of the 3D Web", "keywords": [ diff --git a/src/layers/Environment/ui/PauseMenu/index.tsx b/src/layers/Environment/ui/PauseMenu/index.tsx index cfeddcfe..9ae68190 100644 --- a/src/layers/Environment/ui/PauseMenu/index.tsx +++ b/src/layers/Environment/ui/PauseMenu/index.tsx @@ -48,7 +48,7 @@ export default function PauseMenu(props: PauseMenuProps) { const PAUSE_ITEMS: PauseItem[] = [ ...pauseMenuItems, { - text: "v2.3.1", + text: "v2.3.2", link: "https://www.npmjs.com/package/spacesvr", }, ...menuItems,