diff --git a/package.json b/package.json index 14cccf47..e972ee06 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,9 @@ "@babel/plugin-syntax-flow": "7.18.6", "@babel/plugin-transform-react-jsx": "7.19.0", "@craco/craco": "6.4.5", + "@mediapipe/camera_utils": "^0.3.1640029074", + "@mediapipe/drawing_utils": "^0.3.1620248257", + "@mediapipe/pose": "^0.5.1635988162", "@reduxjs/toolkit": "1.8.6", "@testing-library/react": "13.4.0", "@types/jest": "29.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 418959bc..e60f4529 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ specifiers: '@babel/preset-env': 7.19.4 '@babel/preset-typescript': 7.18.6 '@craco/craco': 6.4.5 + '@mediapipe/camera_utils': ^0.3.1640029074 + '@mediapipe/drawing_utils': ^0.3.1620248257 + '@mediapipe/pose': ^0.5.1635988162 '@reduxjs/toolkit': 1.8.6 '@testing-library/jest-dom': 5.16.5 '@testing-library/react': 13.4.0 @@ -68,6 +71,9 @@ dependencies: '@babel/plugin-syntax-flow': 7.18.6_@babel+core@7.19.3 '@babel/plugin-transform-react-jsx': 7.19.0_@babel+core@7.19.3 '@craco/craco': 6.4.5_koip3jzghppdm6uujk6cx352di + '@mediapipe/camera_utils': 0.3.1640029074 + '@mediapipe/drawing_utils': 0.3.1620248257 + '@mediapipe/pose': 0.5.1635988162 '@reduxjs/toolkit': 1.8.6_kuo2ie247izvzll3jejufdtq3q '@testing-library/react': 13.4.0_biqbaboplfbrettd7655fr4n2y '@types/jest': 29.2.0 @@ -92,6 +98,7 @@ dependencies: prop-types: 15.8.1 rc-menu: 9.6.4_biqbaboplfbrettd7655fr4n2y rc-table: 7.27.2_biqbaboplfbrettd7655fr4n2y + rc-tween-one: 3.0.6_biqbaboplfbrettd7655fr4n2y react: 18.2.0 react-apple-emojis: 2.1.1_v2m5e27vhdewzwhryxwfaorcca react-beautiful-dnd: 13.1.1_biqbaboplfbrettd7655fr4n2y @@ -104,7 +111,7 @@ dependencies: react-redux: 8.0.4_yfr4m6wk75wdra335rld4tkrbu react-router: 6.4.2_react@18.2.0 react-router-dom: 6.4.2_biqbaboplfbrettd7655fr4n2y - react-scripts: 5.0.1_xgafidmxutkymyk6hrj4mgb47m + react-scripts: 5.0.1_q5exndy6ksylmkqoqxdzmpht4y react-webcam: 7.0.1_biqbaboplfbrettd7655fr4n2y redux: 4.2.0 redux-persist: 6.0.0_react@18.2.0+redux@4.2.0 @@ -1669,7 +1676,7 @@ packages: dependencies: cross-spawn: 7.0.3 lodash: 4.17.21 - react-scripts: 5.0.1_xgafidmxutkymyk6hrj4mgb47m + react-scripts: 5.0.1_q5exndy6ksylmkqoqxdzmpht4y webpack-merge: 4.2.2 dev: false @@ -1684,7 +1691,7 @@ packages: cosmiconfig-typescript-loader: 1.0.9_t6iukbiom2jlsbp4uv3roalcz4 cross-spawn: 7.0.3 lodash: 4.17.21 - react-scripts: 5.0.1_xgafidmxutkymyk6hrj4mgb47m + react-scripts: 5.0.1_q5exndy6ksylmkqoqxdzmpht4y semver: 7.3.7 webpack-merge: 4.2.2 transitivePeerDependencies: @@ -2405,6 +2412,18 @@ packages: - encoding - supports-color + /@mediapipe/camera_utils/0.3.1640029074: + resolution: {integrity: sha512-jRV/Mp2lgqNYT68TeVRu/Xq+ptZO9F9vCjxzQsA3VM2r7oXg0TrnfO9De2KKOUTpt0p28dKHs625J8GdWjha8A==} + dev: false + + /@mediapipe/drawing_utils/0.3.1620248257: + resolution: {integrity: sha512-s598oo1K6C62mX3rWXJ7n1RJFdXjyQn3f6CeI+lU6kD69MVyBcV3hdmO5LnEZlCw5NRDANtaM8WAOuqZHaldLg==} + dev: false + + /@mediapipe/pose/0.5.1635988162: + resolution: {integrity: sha512-t0dpl+iG/MTPtPPxEYHyVWo+X7G+qgYUYaB4y9pxavQRNzuQQeHeSmUnT+A8qOKnuB0ccxgyrrFmGUaaL0F3wQ==} + dev: false + /@nodelib/fs.scandir/2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -3554,10 +3573,8 @@ packages: ajv: 6.12.6 dev: false - /ajv-formats/2.1.1_ajv@8.11.0: + /ajv-formats/2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} - peerDependencies: - ajv: ^8.0.0 peerDependenciesMeta: ajv: optional: true @@ -11455,12 +11472,6 @@ packages: /react-dev-utils/12.0.1_iphotpp42ipt4ovquw5uqijhcq: resolution: {integrity: sha512-84Ivxmr17KjUupyqzFode6xKhjwuEJDROWKJy/BthkL7Wn6NJ8h4WE6k/exAv6ImS+0oZLRRW5j/aINMHyeGeQ==} engines: {node: '>=14'} - peerDependencies: - typescript: '>=2.7' - webpack: '>=4' - peerDependenciesMeta: - typescript: - optional: true dependencies: '@babel/code-frame': 7.18.6 address: 1.2.0 @@ -11491,7 +11502,9 @@ packages: transitivePeerDependencies: - eslint - supports-color + - typescript - vue-template-compiler + - webpack dev: false /react-dom/18.2.0_react@18.2.0: @@ -11650,12 +11663,11 @@ packages: react: 18.2.0 dev: false - /react-scripts/5.0.1_xgafidmxutkymyk6hrj4mgb47m: + /react-scripts/5.0.1_q5exndy6ksylmkqoqxdzmpht4y: resolution: {integrity: sha512-8VAmEm/ZAwQzJ+GOMLbBsTdDKOpuZh7RPs0UymvBR2vRk4iZWCskjbFnxqjrzoIvlNNRZ3QJFx6/qDSi6zSnaQ==} engines: {node: '>=14.0.0'} hasBin: true peerDependencies: - eslint: '*' react: '>= 16' typescript: ^3.2.1 || ^4 peerDependenciesMeta: @@ -12221,7 +12233,7 @@ packages: dependencies: '@types/json-schema': 7.0.11 ajv: 8.11.0 - ajv-formats: 2.1.1_ajv@8.11.0 + ajv-formats: 2.1.1 ajv-keywords: 5.1.0_ajv@8.11.0 dev: false diff --git a/src/hooks/api.tsx b/src/hooks/api.tsx index 0c44b570..fef72745 100644 --- a/src/hooks/api.tsx +++ b/src/hooks/api.tsx @@ -199,7 +199,7 @@ const useApi = () => { * Uses the configured {@link config.websocketUrl} to connect to the websocket. * @returns {ApiSocketConnection} the created {@link ApiSocketConnection} */ - const openSocket = async (): Promise => { + const openSocket = (): ApiSocketConnection => { return new ApiSocketConnection(token, window._env_.WEBSOCKET_URL); }; diff --git a/src/pages/train/components/webcamStreamCapture.tsx b/src/pages/train/components/webcamStreamCapture.tsx index 1c507f00..cab7cccf 100644 --- a/src/pages/train/components/webcamStreamCapture.tsx +++ b/src/pages/train/components/webcamStreamCapture.tsx @@ -5,11 +5,13 @@ import React, { useRef, useState, } from "react"; -import Webcam from "react-webcam"; import useWindowDimensions from "@hooks/windowDimension"; import { ApiSocketConnection } from "@hooks/api"; import { PlayCircleOutlined } from "@ant-design/icons"; import { Button } from "antd"; +import { Camera } from "@mediapipe/camera_utils"; +import { Pose, POSE_CONNECTIONS, Results } from "@mediapipe/pose"; +import { drawConnectors, drawLandmarks } from "@mediapipe/drawing_utils"; interface Props { webSocketRef: RefObject; @@ -29,86 +31,128 @@ const WebcamStreamCapture: React.FC = ({ active, cameraShown, }: Props): JSX.Element => { - const webcamRef = useRef(null); - const mediaRecorderRef = useRef(null); + const videoRef = useRef(null); + const canvasRef = useRef(null); + const [capturing, setCapturing] = useState(false); const [countdown, setCountdown] = useState(-1); - const [webcamReady, setWebcamReady] = useState(false); - - useEffect(() => { - if (webcamRef.current) { - webcamRef.current.video?.addEventListener("canplay", () => { - setWebcamReady(true); - }); - } - }, [webcamRef]); - - const sendChunks = useCallback( - (data: Blob): void => { - if (!active) return; - webSocketRef.current?.send(data); - }, - [active, webSocketRef] - ); - const handleDataAvailable = useCallback( - ({ data }: { data: Blob }) => { - if (data.size > 0) { - sendChunks(data); - } + const sendImage = useCallback( + async (video: HTMLVideoElement) => { + if (!capturing || !active) return; + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + const context = canvas.getContext("2d"); + if (!context) return; + context.drawImage(video, 0, 0); + const image = canvas.toDataURL("image/jpeg"); + const utf8Encode = new TextEncoder(); + webSocketRef.current?.send(utf8Encode.encode(image)); }, - [sendChunks] + [capturing, webSocketRef, active] ); - const handlerRef = useRef<({ data }: { data: Blob }) => void>(); - - useEffect(() => { - if (handlerRef.current) - mediaRecorderRef.current?.removeEventListener( - "dataavailable", - handlerRef.current - ); - mediaRecorderRef.current?.addEventListener( - "dataavailable", - handleDataAvailable - ); - handlerRef.current = handleDataAvailable; - }, [handleDataAvailable]); - - const handleStartCaptureClick = useCallback(() => { - if (!webcamRef.current?.stream?.active || !webSocketRef.current) return; + const startCountdown = useCallback(async () => { + for (let i = 3; i > 0; i--) { + setCountdown(i); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + setCountdown(0); setCapturing(true); - webSocketRef.current?.send( JSON.stringify({ message_type: "start_set", data: { user_token: "", exercise_id: 1 }, }) ); + }, [webSocketRef]); - mediaRecorderRef.current = new MediaRecorder( - webcamRef.current.stream as MediaStream, - { - mimeType: "video/webm", - } - ); - mediaRecorderRef.current.addEventListener( - "dataavailable", - handleDataAvailable + // ------------------------------- + // MEDIAPIPE POSE ESTIMATION START + // ------------------------------- + const [pose, setPose] = useState(); + + const onResults = useCallback((results: Results) => { + if (!results.poseLandmarks || !canvasRef.current) return; + const canvasCtx = canvasRef.current.getContext("2d"); + if (!canvasCtx) return; + canvasCtx.clearRect( + 0, + 0, + canvasRef.current.width, + canvasRef.current.height ); - // data available every 100 milliseconds - mediaRecorderRef.current.start(100); - }, [webSocketRef, handleDataAvailable]); + drawConnectors(canvasCtx, results.poseLandmarks, POSE_CONNECTIONS, { + color: "#00FF00", + lineWidth: 4, + }); + drawLandmarks(canvasCtx, results.poseLandmarks, { + color: "#FF0000", + lineWidth: 2, + }); + }, []); - const startCountdown = useCallback(async () => { - for (let i = 3; i > 0; i--) { - setCountdown(i); - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - setCountdown(0); - handleStartCaptureClick(); - }, [handleStartCaptureClick]); + useEffect(() => { + // if (!webSocketRef.current) return; + const pose = new Pose({ + locateFile: (file) => { + return `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`; + }, + }); + pose.setOptions({ + modelComplexity: 0, + smoothLandmarks: true, + enableSegmentation: true, + smoothSegmentation: true, + minDetectionConfidence: 0.5, + minTrackingConfidence: 0.5, + }); + pose.onResults(onResults); + setPose(pose); + return () => { + console.log("Cleaning up"); + pose.close(); + }; + }, [onResults, webSocketRef]); + + useEffect(() => { + if (!videoRef.current) return; + const camera = new Camera(videoRef?.current, { + onFrame: async () => { + if (!videoRef.current) return; + sendImage(videoRef.current); + await pose?.send({ image: videoRef.current }); + }, + }); + camera.start(); + return () => { + console.log("Stopping camera"); + camera.stop(); + }; + }, [videoRef, pose, sendImage]); + + // set the correct size attribute for the canvas + // even though it is possisioned with css, + // mediapipe needs the width and height attributes for some unholy reason + useEffect(() => { + if (!videoRef.current || !canvasRef.current) return; + const obs = new ResizeObserver(() => { + canvasRef.current?.setAttribute( + "width", + `${videoRef.current?.clientWidth}` + ); + canvasRef.current?.setAttribute( + "height", + `${videoRef.current?.clientHeight}` + ); + }); + obs.observe(videoRef.current as Element); + }, [videoRef, canvasRef]); + // ----------------------------- + // MEDIAPIPE POSE ESTIMATION END + // ----------------------------- const { height } = useWindowDimensions(); @@ -122,16 +166,15 @@ const WebcamStreamCapture: React.FC = ({ margin: "10px", }} > -
= ({ WebkitBackdropFilter: !cameraShown || (capturing && !active) ? "blur(50px)" : "none", borderRadius: "30px", + }} + >
+ +
= ({ }} icon={} className="no-font-fix-button-weirdness" - disabled={!webcamReady} /> )} {countdown > 0 && ( diff --git a/src/pages/train/training/index.tsx b/src/pages/train/training/index.tsx index d667ca22..80a5ed46 100644 --- a/src/pages/train/training/index.tsx +++ b/src/pages/train/training/index.tsx @@ -288,28 +288,25 @@ const Training: React.FC = ({ > {feedback.totalPoints} - - {active && ( -
setCameraShown(!cameraShown)} - style={{ - marginRight: "-5px", - marginLeft: "auto", - width: "50px", - height: "50px", - borderRadius: "50%", - backgroundColor: "white", - padding: "5px", - cursor: "pointer", - }} - > - {cameraShown ? ( - - ) : ( - - )} -
- )} +
setCameraShown(!cameraShown)} + style={{ + marginRight: "-5px", + marginLeft: "auto", + width: "50px", + height: "50px", + borderRadius: "50%", + backgroundColor: "white", + padding: "5px", + cursor: "pointer", + }} + > + {cameraShown ? ( + + ) : ( + + )} +