diff --git a/client/src/components/Negotiate/Chat.jsx b/client/src/components/Negotiate/Chat.jsx index 5671e76..3b0620e 100644 --- a/client/src/components/Negotiate/Chat.jsx +++ b/client/src/components/Negotiate/Chat.jsx @@ -7,6 +7,7 @@ import { AuthContext } from '../context/Authcontext'; import { useNavigate } from 'react-router-dom'; import { LuUserCircle2 } from "react-icons/lu"; import { IoArrowBackOutline } from "react-icons/io5"; +import { AudioRecorder } from 'react-audio-voice-recorder'; import { MdCall } from "react-icons/md"; import { IoVideocam } from "react-icons/io5"; import { IoMdMore } from "react-icons/io"; @@ -30,6 +31,7 @@ export default function Chat() { const navigate = useNavigate(); const [currperson, setperson] = useState(); const [loading, setLoading] = useState(true); + const [lang, setLang] = useState("kannada"); const chatDisplayRef = useRef(null); // Ref to chat display container const bottomRef = useRef(null); // Ref to the last message to scroll into view @@ -226,7 +228,28 @@ export default function Chat() { value={message} onChange={handleChange} /> - +
+ handleAudioUpload(blob, lang)} //sending POST request + audioTrackConstraints={{ + noiseSuppression: true, + echoCancellation: true, + // autoGainControl, + // channelCount, + // deviceId, + // groupId, + // sampleRate, + // sampleSize, + }} + onNotAllowedOrFound={(err) => console.table(err)} + downloadOnSavePress={false} + downloadFileExtension="webm" + mediaRecorderOptions={{ + audioBitsPerSecond: 128000, + }} + showVisualizer={true} + /> +
@@ -294,4 +317,25 @@ const loadMessage = async (currentRoomID) => { console.error("Error fetching messages: ", error); } +} + +const handleAudioUpload = async (blob, lang) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = async () => { + const base64data = reader.result.split(',')[1]; // Extract Base64 part + + const response = await fetch(`http://localhost:${SERVER_PORT}/chat`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', // Send JSON data + }, + body: JSON.stringify({ + audio: base64data, + mimeType: blob.type, + lang: lang, + }), + }); + console.log(response.json().then((result => { console.log(result.transcription) }))); + } } \ No newline at end of file diff --git a/server/audioHandler.js b/server/audioHandler.js new file mode 100644 index 0000000..2a80635 --- /dev/null +++ b/server/audioHandler.js @@ -0,0 +1,53 @@ +const { GoogleGenerativeAI } = require("@google/generative-ai"); +const mime = require('mime-types'); + +const genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY); +const model = genAI.getGenerativeModel({ model: "gemini-1.5-flash-latest" }); + +const transcribeAudio = async (audioBuffer, mimeType, lang) => { + console.log("transcribing audio"); + const audio = { + inlineData: { + data: Buffer.from(audioBuffer).toString("base64"), + mimeType: mimeType || 'audio/wav', + }, + }; + + const prompt = `transcribe the audio and give the transcription in ${lang} without any timestamps and new line characters`; + + const result = await model.generateContent([audio, prompt]); + return result.response.text(); // Return the transcription result +}; + + +const handleAudio = async (req, res) => { + console.log("Received POST request"); + try { + const { audio, mimeType, lang } = req.body; + // console.log("audio ", audio); + // console.log("mimeType ", mimeType); + // console.log("lang ", lang); + if (!audio) { + return res.status(400).json({ message: "No audio data received" }); + } + if (!lang) { + return res.status(400).json({ message: "No language data received" }); + } + + // Decode Base64 audio data + const audioBuffer = Buffer.from(audio, 'base64'); + const mimeTypeFromHeader = mimeType || 'audio/wav'; + + // Perform transcription + const transcription = await transcribeAudio(audioBuffer, mimeTypeFromHeader, lang); + console.log(transcription); + res.status(200).json({ message: "Transcription completed", transcription }); + + } catch (error) { + console.error("Error processing audio: ", error); + res.status(500).json({ message: "Internal server error", error: error.message }); + } +}; + + +module.exports = {handleAudio}; \ No newline at end of file diff --git a/server/package-lock.json b/server/package-lock.json index 1e67da7..473503a 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@google/generative-ai": "^0.19.0", "body-parser": "^1.20.2", "cors": "^2.8.5", "dotenv": "^16.4.5", @@ -210,6 +211,14 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/@google/generative-ai": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.19.0.tgz", + "integrity": "sha512-iGf/62v3sTwtEJGJY6S5m7PfkglU8hi1URaxqIjiRg1OItV27xyc4aVeR0og8cDkZFkUlGZKv+23bJtT1QWFzQ==", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.11.1.tgz", diff --git a/server/package.json b/server/package.json index dc251fe..0c8491b 100644 --- a/server/package.json +++ b/server/package.json @@ -11,6 +11,7 @@ "license": "ISC", "description": "", "dependencies": { + "@google/generative-ai": "^0.19.0", "body-parser": "^1.20.2", "cors": "^2.8.5", "dotenv": "^16.4.5", diff --git a/server/server.js b/server/server.js index f541157..258b151 100644 --- a/server/server.js +++ b/server/server.js @@ -5,6 +5,7 @@ const http = require('http'); const cors = require('cors'); const app = express(); const {Server} = require('socket.io'); +const {handleAudio} = require("./audioHandler"); const SERVER_PORT = process.env.SERVER_PORT; const CLIENT_PORT = process.env.CLIENT_PORT; @@ -14,6 +15,9 @@ const server = http.createServer(app); //middleware app.use(cors()); +app.use(express.json()); +app.use(express.urlencoded({ extended: true })); +// app.use(express.raw({ type: 'audio/wav', limit: '10mb' })); //routings app.get('/states', (req, res) => { @@ -40,6 +44,7 @@ app.get('/areas/:state/:district/:subdistrict', (req, res) => { const areas = states[state][district][subdistrict] || []; res.json(areas); }); +app.post('/chat', handleAudio); //socket.io connections const io = new Server(server, { @@ -49,6 +54,8 @@ const io = new Server(server, { } }) + + io.on("connection", (socket)=>{ console.log(socket.id); @@ -72,3 +79,4 @@ io.on("connection", (socket)=>{ server.listen(SERVER_PORT, () => { console.log(`Server is running on http://localhost:${SERVER_PORT}`); }); +