Merge pull request #2 from 1KrishnaSingh346/main

Updated Files

SignLanguage/Pipfile

@@ -0,0 +1,11 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+
+[requires]
+python_version = "3.8"

SignLanguage/README copy.md

@@ -0,0 +1,25 @@
+# DeepASL
+
+![deep_asl_ex_vid_AdobeCreativeCloudExpress](https://user-images.githubusercontent.com/89669770/160533990-ae71afe4-67f4-4d21-93b4-8bf65fba739f.gif)
+
+**Overview**
+
+DeepASL utilizes webcam video feed and some Python code to interpret American Sign Language hand gestures in real time! My goal with this project was to learn the very basics of what makes up [Convolutional Neural Networks](https://cs231n.github.io/convolutional-networks/) and how [Computer Vision](http://vision.stanford.edu/teaching/cs131_fall2122/index.html) can make them interactively useful in the real world.
+
+**Installation and Usage**
+
+1. Clone this repo: ```git clone https://github.com/cesarealmendarez/DeepASL.git```
+2. Navigate to project: ```cd DeepASL```
+3. Install required packages: ```pip3 install opencv-python mediapipe numpy```
+4. Run DeepASL: ```python3 app.py```
+
+**What's on my Screen?**
+
+Once you run DeepASL, two windows will appear, the Analytics window displays the raw video along with the extracted data points used in interpreting hand landmarks/steadiness, depth perception, output confidence, and finally triggering the snapshot. The Hand Segmentation window shows what the network will break down into a pattern of 1's and 0's prompting it's best attempt to guess what ASL letter you're showing!
+
+**Resources Used**
+
+1. [MediaPipe](https://github.com/google/mediapipe): Used to perceive the shape of the hand and create a skeleton-like outline, enabling segmentation of useful classification features.
+2. [MNIST Handwritten Digits Classification using a Convolutional Neural Network (CNN)](https://towardsdatascience.com/mnist-handwritten-digits-classification-using-a-convolutional-neural-network-cnn-af5fafbc35e9) 
+3. [A Comprehensive Guide to Convolutional Neural Networks — the ELI5 way](https://towardsdatascience.com/a-comprehensive-guide-to-convolutional-neural-networks-the-eli5-way-3bd2b1164a53) 
+4. [Simple Introduction to Convolutional Neural Networks](https://towardsdatascience.com/simple-introduction-to-convolutional-neural-networks-cdf8d3077bac)

SignLanguage/README.md

@@ -0,0 +1 @@
+# SignLanguage

SignLanguage/l_convolution.py

@@ -0,0 +1,41 @@
+import numpy as np
+
+class Convolution:
+    def __init__(self, num_filters, nominal_filters):
+        self.num_filters = num_filters
+
+        if isinstance(nominal_filters, str) == True:
+            self.filters = np.random.randn(num_filters, 3, 3) / 9
+
+        else:
+            self.filters = nominal_filters
+
+    def iterate_image(self, image):
+        h, w = image.shape
+
+        for i in range(h - 2):
+            for j in range(w - 2):
+                image_region = image[i : (i + 3), j : (j + 3)]
+
+                yield image_region, i, j
+
+    def forward_propagation(self, input):
+        self.last_input = input
+        h, w = input.shape
+        output = np.zeros((h - 2, w - 2, self.num_filters))
+
+        for image_region, i, j, in self.iterate_image(input):
+            output[i, j] = np.sum(image_region * self.filters, axis = (1, 2))
+
+        return output
+
+    def back_propagation(self, d_L_d_out, learn_rate):
+        d_L_d_filters = np.zeros(self.filters.shape)
+
+        for image_region, i, j in self.iterate_image(self.last_input):
+            for f in range(self.num_filters):
+                d_L_d_filters[f] += d_L_d_out[i, j, f] * image_region
+
+        self.filters -= learn_rate * d_L_d_filters
+
+        return None

SignLanguage/l_maxpool.py

@@ -0,0 +1,39 @@
+import numpy as np
+
+class MaxPool:
+    def iterate_image_3x3(self, image):
+        h, w, _ = image.shape
+        new_h = h // 2
+        new_w = w // 2
+
+        for i in range(new_h):
+            for j in range(new_w):
+                image_region_3x3 = image[(i * 2) : (i * 2 + 2), (j * 2) : (j * 2 + 2)]
+
+                yield image_region_3x3, i, j
+
+    def back_propagation(self, d_L_d_out):
+        d_L_d_input = np.zeros(self.last_input.shape)
+
+        for image_region_3x3, i, j in self.iterate_image_3x3(self.last_input):
+            h, w, f = image_region_3x3.shape
+            amax = np.amax(image_region_3x3, axis = (0, 1))
+
+            for i2 in range(h):
+                for j2 in range(w):
+                    for f2 in range(f):
+                        if image_region_3x3[i2, j2, f2] == amax[f2]:
+                            d_L_d_input[i * 2 + i2, j * 2 + j2, f2] = d_L_d_out[i, j, f2]
+
+        return d_L_d_input
+
+    def forward_propagation(self, input):
+        self.last_input = input
+
+        h, w, num_filters = input.shape
+        output = np.zeros((h // 2, w // 2, num_filters))
+
+        for img_region, i, j in self.iterate_image_3x3(input):
+            output[i, j] = np.amax(img_region, axis = (0, 1))
+
+        return output

SignLanguage/l_softmax.py

@@ -0,0 +1,51 @@
+import numpy as np
+
+class Softmax:
+    def __init__(self, input_length, nodes, nominal_weights, nominal_biases):
+        if isinstance(nominal_weights, str) == True:
+            self.weights = np.random.randn(input_length, nodes) / input_length
+            self.biases = np.zeros(nodes)
+
+        else:
+            self.weights = nominal_weights
+            self.biases = nominal_biases
+
+    def forward_propagation(self, input):
+        self.last_input_shape = input.shape
+
+        input = input.flatten()
+        self.last_input = input
+
+        input_length, nodes = self.weights.shape
+
+        totals = np.dot(input, self.weights) + self.biases
+        self.last_totals = totals
+        exp = np.exp(totals)
+
+        return exp / np.sum(exp, axis = 0)
+
+    def back_propagation(self, d_L_d_out, learn_rate):
+        for i, gradient in enumerate(d_L_d_out):
+            if gradient == 0:
+                continue
+
+            t_exp = np.exp(self.last_totals)
+            S = np.sum(t_exp)
+
+            d_out_d_t = -t_exp[i] * t_exp / (S ** 2)
+            d_out_d_t[i] = t_exp[i] * (S - t_exp[i]) / (S ** 2)
+
+            d_t_d_w = self.last_input
+            d_t_d_b = 1
+            d_t_d_inputs = self.weights
+
+            d_L_d_t = gradient * d_out_d_t
+
+            d_L_d_w = d_t_d_w[np.newaxis].T @ d_L_d_t[np.newaxis]
+            d_L_d_b = d_L_d_t * d_t_d_b
+            d_L_d_inputs = d_t_d_inputs @ d_L_d_t
+
+            self.weights -= learn_rate * d_L_d_w
+            self.biases -= learn_rate * d_L_d_b
+
+            return d_L_d_inputs.reshape(self.last_input_shape)

SignLanguage/net_init.py

@@ -0,0 +1,33 @@
+from l_convolution import Convolution
+from l_maxpool import MaxPool
+from l_softmax import Softmax
+import pickle
+
+nominal_filter_file_dir = "NominalFilters.txt"
+nominal_weights_file_dir = "NominalWeights.txt"
+nominal_biases_file_dir = "NominalBiases.txt"
+
+nominal_filters_test = []
+nominal_biases_test = []
+nominal_weights_test = [] 
+
+with open(nominal_filter_file_dir, 'rb') as f:
+    nominal_filters_test = pickle.load(f)
+
+with open(nominal_weights_file_dir, 'rb') as d:
+    nominal_weights_test = pickle.load(d)
+
+with open(nominal_biases_file_dir, 'rb') as e:
+    nominal_biases_test = pickle.load(e) 
+
+def trained_forward_propagation(image):
+    convolution = Convolution(8, nominal_filters_test)
+    maxpool = MaxPool()
+    softmax = Softmax(13 * 13 * 8, 25, nominal_weights_test, nominal_biases_test)
+
+    convolution_output = convolution.forward_propagation((image / 255) - 0.5)
+    maxpool_output = maxpool.forward_propagation(convolution_output)
+    softmax_output = softmax.forward_propagation(maxpool_output)
+
+    return softmax_output    
+

SignLanguage/requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+opencv-python==4.5.5.62
+mediapipe
+numpy
+scikit-learn
+statistics # This may not be needed as it is part of the Python Standard Library

SignLanguage/runner.py

@@ -0,0 +1,9 @@
+import streamlit as st
+
+# Load the model
+# Assuming app.py contains the necessary functions for processing
+from app import start_video_capture
+
+st.title("Sign Language Recognition")
+# Display the video capturing component
+start_video_capture()

VideoStream.py

@@ -0,0 +1,39 @@
+from flask import Flask, render_template, Response
+import cv2
+import mediapipe as mp
+import numpy as np
+
+app = Flask(__name__)
+
+video_capture = cv2.VideoCapture(0)
+mediapipe_hands = mp.solutions.hands.Hands(static_image_mode=False, max_num_hands=1, min_detection_confidence=0.5, min_tracking_confidence=0.5)
+mp_drawing = mp.solutions.drawing_utils
+
+def generate_frames():
+    while True:
+        success, frame = video_capture.read()
+        if not success:
+            break
+        else:
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = mediapipe_hands.process(frame_rgb)
+            if results.multi_hand_landmarks:
+                for hand_landmarks in results.multi_hand_landmarks:
+                    mp_drawing.draw_landmarks(frame, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS)
+
+            ret, buffer = cv2.imencode('.jpg', frame)
+            frame = buffer.tobytes()
+
+            yield (b'--frame\r\n'
+                   b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/video_feed')
+def video_feed():
+    return Response(generate_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')
+
+if __name__ == '__main__':
+    app.run(debug=True)

dist/gglsignin.js

@@ -0,0 +1,51 @@
+import { initializeApp } from "https://www.gstatic.com/firebasejs/10.13.0/firebase-app.js";
+
+import { getAuth, signInWithPopup, GoogleAuthProvider } from "https://www.gstatic.com/firebasejs/10.13/firebase-auth.js";
+
+const firebaseConfig = {
+  apiKey: "AIzaSyAefVT_gZdsg2W64Iq7qZz2fg7WNIFZ4p4",
+  authDomain: "login-check-8dddf.firebaseapp.com",
+  projectId: "login-check-8dddf",
+  storageBucket: "login-check-8dddf.appspot.com",
+  messagingSenderId: "357442081317",
+  appId: "1:357442081317:web:70e9bb535fe12a192525cc"
+};
+
+// provider.addScope('https://www.googleapis.com/auth/contacts.readonly');
+
+// import { getAuth } from "https://www.gstatic.com/firebasejs/10.13/firebase-auth.js";
+
+const app=initializeApp(firebaseConfig);
+const auth = getAuth(app);
+auth.languageCode = 'en';
+const provider = new GoogleAuthProvider();
+// To apply the default browser preference instead of explicitly setting it.
+// auth.useDeviceLanguage();
+
+// provider.setCustomParameters({
+//   'login_hint': '[email protected]'
+// });
+
+const gglsignin = document.getElementById("gglsignin");
+
+gglsignin.addEventListener("click", function(event)
+{
+  event.preventDefault()
+
+  signInWithPopup(auth, provider)
+  .then((result) => {
+    const credential = GoogleAuthProvider.credentialFromResult(result);
+    const token = credential.accessToken;
+    const user = result.user;
+    alert("User logged-in successfully. Welcome, " + user.displayName);
+    window.location.href="main.html";
+
+
+  }).catch((error) => {
+    const errorCode = error.code;
+    const errorMessage = error.message;
+    alert("Data not found. Please sign-up.")
+  });
+})
+
+

dist/gglsignup.js

@@ -0,0 +1,50 @@
+import { initializeApp } from "https://www.gstatic.com/firebasejs/10.13.0/firebase-app.js";
+
+import { getAuth, signInWithPopup, GoogleAuthProvider } from "https://www.gstatic.com/firebasejs/10.13/firebase-auth.js";
+
+const firebaseConfig = {
+  apiKey: "AIzaSyAefVT_gZdsg2W64Iq7qZz2fg7WNIFZ4p4",
+  authDomain: "login-check-8dddf.firebaseapp.com",
+  projectId: "login-check-8dddf",
+  storageBucket: "login-check-8dddf.appspot.com",
+  messagingSenderId: "357442081317",
+  appId: "1:357442081317:web:70e9bb535fe12a192525cc"
+};
+
+// provider.addScope('https://www.googleapis.com/auth/contacts.readonly');
+
+// import { getAuth } from "https://www.gstatic.com/firebasejs/10.13/firebase-auth.js";
+
+const app=initializeApp(firebaseConfig);
+const auth = getAuth(app);
+auth.languageCode = 'en';
+const provider = new GoogleAuthProvider();
+// To apply the default browser preference instead of explicitly setting it.
+// auth.useDeviceLanguage();
+
+// provider.setCustomParameters({
+//   'login_hint': '[email protected]'
+// });
+
+const gglsignup = document.getElementById("gglsignup");
+
+gglsignup.addEventListener("click", function(event)
+{
+  event.preventDefault()
+
+  signInWithPopup(auth, provider)
+  .then((result) => {
+    const credential = GoogleAuthProvider.credentialFromResult(result);
+    const token = credential.accessToken;
+    const user = result.user;
+    console.log(user);
+    window.location.href="main.html";
+
+
+  }).catch((error) => {
+    const errorCode = error.code;
+    const errorMessage = error.message;
+  });
+})
+
+