From 3f66117ea81708ceb69883a7d3b5f3cc86580f6e Mon Sep 17 00:00:00 2001 From: kroy94 Date: Fri, 18 Mar 2022 17:21:34 -0400 Subject: [PATCH 1/4] updated model URL --- client/components/Home.js | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/client/components/Home.js b/client/components/Home.js index f9137e9..82c47a7 100644 --- a/client/components/Home.js +++ b/client/components/Home.js @@ -15,7 +15,7 @@ export const Home = (props) => { const canvasRef = useRef(null); const [translation, setTranslation] = useState(null); - const URL = "https://teachablemachine.withgoogle.com/models/HBBfwFtF-/"; + const URL = "https://teachablemachine.withgoogle.com/models/SdeOHBnL5/"; let model, webcam, labelContainer, maxPredictions; @@ -23,8 +23,7 @@ export const Home = (props) => { const metadataURL = URL + "metadata.json"; const loadModel = async () => { const model = await tmImage.load(checkpointURL, metadataURL); - console.log("model-----hi", model); - console.log("hi"); + console.log("Model Loaded", model); const net = await handpose.load(); console.log("net", net); setInterval(() => { @@ -108,12 +107,6 @@ export const Home = (props) => { } else { return; } - /* - - - - */ - //------- // for (let i = 0; i < maxPredictions; i++) { // const classPrediction = From 37ff4a41ff4c0a8b19d9394403704c262bf18985 Mon Sep 17 00:00:00 2001 From: kroy94 Date: Fri, 18 Mar 2022 17:48:51 -0400 Subject: [PATCH 2/4] added some functionality about handDetection, that's commented out for now --- client/components/Home.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/client/components/Home.js b/client/components/Home.js index 82c47a7..8284498 100644 --- a/client/components/Home.js +++ b/client/components/Home.js @@ -14,6 +14,11 @@ export const Home = (props) => { const webcamRef = useRef(null); const canvasRef = useRef(null); const [translation, setTranslation] = useState(null); + /* +THIS WAS AN ATTEMPT TO UPDATE +TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME + const [handDetection, setHandDetection] = useState(false); + */ const URL = "https://teachablemachine.withgoogle.com/models/SdeOHBnL5/"; @@ -59,6 +64,12 @@ export const Home = (props) => { //make detections for hands and finger gestures if (hand.length > 0) { + /* + THIS WAS AN ATTEMPT TO UPDATE + TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME + setHandDetection(true); + */ + console.log('HAND DETECTION -->', handDetection); const gestureEstimator = new fp.GestureEstimator([ // fp.Gestures.VictoryGesture, fp.Gestures.ThumbsUpGesture, @@ -78,6 +89,12 @@ export const Home = (props) => { } } else { return; + /* + THIS WAS AN ATTEMPT TO UPDATE TRANSLATION + TO NULL WHEN HAND IS NOT IN FRAME + setHandDetection(false); + console.log('HAND DETECTION -->', handDetection); + */ } // if (hand.length === 0) { // setTranslation[null]; @@ -119,6 +136,16 @@ export const Home = (props) => { } } useEffect(() => { loadModel() }, []) + /* + THIS WAS AN ATTEMPT TO UPDATE TRANSLATION + TO NULL WHEN HAND IS NOT IN FRAME + useEffect(() => { + if (handDetection === false) { + setTranslation(null) + } + }, [handDetection]); + */ + //only happens when you load the model //state/ //use effect will update everytime that state changes From 401fca33d4dcdee5ccbb78deb79243fe8ef68ddd Mon Sep 17 00:00:00 2001 From: Anh Duong Date: Fri, 18 Mar 2022 18:38:34 -0400 Subject: [PATCH 3/4] fixed bug: only shows translation when detecting hands --- client/components/Home.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/client/components/Home.js b/client/components/Home.js index 8284498..0d11e70 100644 --- a/client/components/Home.js +++ b/client/components/Home.js @@ -15,11 +15,12 @@ export const Home = (props) => { const canvasRef = useRef(null); const [translation, setTranslation] = useState(null); /* -THIS WAS AN ATTEMPT TO UPDATE +THIS WAS AN ATTEMPT TO UPDATE TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME const [handDetection, setHandDetection] = useState(false); */ + //const URL = "https://teachablemachine.withgoogle.com/models/SdeOHBnL5/"; const URL = "https://teachablemachine.withgoogle.com/models/SdeOHBnL5/"; let model, webcam, labelContainer, maxPredictions; @@ -33,7 +34,7 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME console.log("net", net); setInterval(() => { detect(model, net); - }, 1000); + }, 500); }; //Loop and detect hands @@ -65,11 +66,11 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME //make detections for hands and finger gestures if (hand.length > 0) { /* - THIS WAS AN ATTEMPT TO UPDATE + THIS WAS AN ATTEMPT TO UPDATE TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME setHandDetection(true); */ - console.log('HAND DETECTION -->', handDetection); + // console.log('HAND DETECTION -->', handDetection); const gestureEstimator = new fp.GestureEstimator([ // fp.Gestures.VictoryGesture, fp.Gestures.ThumbsUpGesture, @@ -88,9 +89,11 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME // console.log("EMOJI", emoji); } } else { + // Kaia just added the line below + setTranslation(null); return; /* - THIS WAS AN ATTEMPT TO UPDATE TRANSLATION + THIS WAS AN ATTEMPT TO UPDATE TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME setHandDetection(false); console.log('HAND DETECTION -->', handDetection); @@ -110,7 +113,8 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME console.log(probability); const maxPro = probability.indexOf(Math.max.apply(null, probability)); - if (prediction[maxPro].probability > 0.9) { + //// Kaia just added the line below (hand.length > 0) + if (prediction[maxPro].probability > 0.9 && hand.length > 0) { setTranslation(prediction[maxPro].className); } else { setTranslation(null); @@ -135,9 +139,11 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME drawHand(hand, ctx); } } - useEffect(() => { loadModel() }, []) + useEffect(() => { + loadModel(); + }, []); /* - THIS WAS AN ATTEMPT TO UPDATE TRANSLATION + THIS WAS AN ATTEMPT TO UPDATE TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME useEffect(() => { if (handDetection === false) { From 5ae89617ad54754322f43773e5620fe5f93cfdde Mon Sep 17 00:00:00 2001 From: kroy94 Date: Mon, 21 Mar 2022 11:13:55 -0400 Subject: [PATCH 4/4] removing some comments to clean it up --- client/components/Home.js | 68 +++------------------------------------ 1 file changed, 5 insertions(+), 63 deletions(-) diff --git a/client/components/Home.js b/client/components/Home.js index 0d11e70..1ea5be6 100644 --- a/client/components/Home.js +++ b/client/components/Home.js @@ -14,13 +14,7 @@ export const Home = (props) => { const webcamRef = useRef(null); const canvasRef = useRef(null); const [translation, setTranslation] = useState(null); - /* -THIS WAS AN ATTEMPT TO UPDATE -TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME - const [handDetection, setHandDetection] = useState(false); - */ - //const URL = "https://teachablemachine.withgoogle.com/models/SdeOHBnL5/"; const URL = "https://teachablemachine.withgoogle.com/models/SdeOHBnL5/"; let model, webcam, labelContainer, maxPredictions; @@ -40,8 +34,6 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME //Loop and detect hands async function detect(model, net) { - // predict can take in an image, video or canvas html element - // let prediction; what is this? if ( typeof webcamRef.current !== "undefined" && webcamRef.current !== null && @@ -65,14 +57,8 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME //make detections for hands and finger gestures if (hand.length > 0) { - /* - THIS WAS AN ATTEMPT TO UPDATE - TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME - setHandDetection(true); - */ - // console.log('HAND DETECTION -->', handDetection); const gestureEstimator = new fp.GestureEstimator([ - // fp.Gestures.VictoryGesture, + //do we need this thumgs up gesture estimator? fp.Gestures.ThumbsUpGesture, ]); @@ -84,28 +70,15 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME const maxScore = score.indexOf(Math.max.apply(null, score)); console.log("gestures name is -", gesture.gestures[maxScore].name); - // setEmoji(gesture.gestures[maxScore].name); - - // console.log("EMOJI", emoji); } } else { // Kaia just added the line below setTranslation(null); return; - /* - THIS WAS AN ATTEMPT TO UPDATE TRANSLATION - TO NULL WHEN HAND IS NOT IN FRAME - setHandDetection(false); - console.log('HAND DETECTION -->', handDetection); - */ } - // if (hand.length === 0) { - // setTranslation[null]; - let prediction = await model.predict(video); console.log("PREDICTION-----", prediction); - //------- if (prediction && prediction.length > 0) { const probability = prediction.map( (prediction) => prediction.probability @@ -113,28 +86,16 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME console.log(probability); const maxPro = probability.indexOf(Math.max.apply(null, probability)); - //// Kaia just added the line below (hand.length > 0) + // Kaia just added the line below (hand.length > 0) if (prediction[maxPro].probability > 0.9 && hand.length > 0) { setTranslation(prediction[maxPro].className); } else { setTranslation(null); } - // console.log("gestures name is -", prediction[maxPro].name); - - // console.log("TRANSLATION---", translation); - // setEmoji(gesture.gestures[maxScore].name); - - // console.log("EMOJI", emoji); } else { return; } - //------- - // for (let i = 0; i < maxPredictions; i++) { - // const classPrediction = - // prediction[i].className + ": " + prediction[i].probability.toFixed(2); - // // labelContainer.childNodes[i].innerHTML = classPrediction; - // } - //draw mesh + const ctx = canvasRef.current.getContext("2d"); drawHand(hand, ctx); } @@ -142,27 +103,12 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME useEffect(() => { loadModel(); }, []); - /* - THIS WAS AN ATTEMPT TO UPDATE TRANSLATION - TO NULL WHEN HAND IS NOT IN FRAME - useEffect(() => { - if (handDetection === false) { - setTranslation(null) - } - }, [handDetection]); - */ - - //only happens when you load the model - //state/ - //use effect will update everytime that state changes - // const { username } = props; return (
@@ -201,9 +144,8 @@ TRANSLATION TO NULL WHEN HAND IS NOT IN FRAME
); }; -/** - * CONTAINER - */ + + const mapState = (state) => { return { username: state.auth.username,