diff --git a/client/src/components/Negotiate/Chat.jsx b/client/src/components/Negotiate/Chat.jsx
index 5671e76..3b0620e 100644
--- a/client/src/components/Negotiate/Chat.jsx
+++ b/client/src/components/Negotiate/Chat.jsx
@@ -7,6 +7,7 @@ import { AuthContext } from '../context/Authcontext';
import { useNavigate } from 'react-router-dom';
import { LuUserCircle2 } from "react-icons/lu";
import { IoArrowBackOutline } from "react-icons/io5";
+import { AudioRecorder } from 'react-audio-voice-recorder';
import { MdCall } from "react-icons/md";
import { IoVideocam } from "react-icons/io5";
import { IoMdMore } from "react-icons/io";
@@ -30,6 +31,7 @@ export default function Chat() {
const navigate = useNavigate();
const [currperson, setperson] = useState();
const [loading, setLoading] = useState(true);
+ const [lang, setLang] = useState("kannada");
const chatDisplayRef = useRef(null); // Ref to chat display container
const bottomRef = useRef(null); // Ref to the last message to scroll into view
@@ -226,7 +228,28 @@ export default function Chat() {
value={message}
onChange={handleChange}
/>
-
+
+
handleAudioUpload(blob, lang)} //sending POST request
+ audioTrackConstraints={{
+ noiseSuppression: true,
+ echoCancellation: true,
+ // autoGainControl,
+ // channelCount,
+ // deviceId,
+ // groupId,
+ // sampleRate,
+ // sampleSize,
+ }}
+ onNotAllowedOrFound={(err) => console.table(err)}
+ downloadOnSavePress={false}
+ downloadFileExtension="webm"
+ mediaRecorderOptions={{
+ audioBitsPerSecond: 128000,
+ }}
+ showVisualizer={true}
+ />
+
@@ -294,4 +317,25 @@ const loadMessage = async (currentRoomID) => {
console.error("Error fetching messages: ", error);
}
+}
+
+const handleAudioUpload = async (blob, lang) => {
+ const reader = new FileReader();
+ reader.readAsDataURL(blob);
+ reader.onloadend = async () => {
+ const base64data = reader.result.split(',')[1]; // Extract Base64 part
+
+ const response = await fetch(`http://localhost:${SERVER_PORT}/chat`, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json', // Send JSON data
+ },
+ body: JSON.stringify({
+ audio: base64data,
+ mimeType: blob.type,
+ lang: lang,
+ }),
+ });
+ console.log(response.json().then((result => { console.log(result.transcription) })));
+ }
}
\ No newline at end of file
diff --git a/server/audioHandler.js b/server/audioHandler.js
new file mode 100644
index 0000000..2a80635
--- /dev/null
+++ b/server/audioHandler.js
@@ -0,0 +1,53 @@
+const { GoogleGenerativeAI } = require("@google/generative-ai");
+const mime = require('mime-types');
+
+const genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY);
+const model = genAI.getGenerativeModel({ model: "gemini-1.5-flash-latest" });
+
+const transcribeAudio = async (audioBuffer, mimeType, lang) => {
+ console.log("transcribing audio");
+ const audio = {
+ inlineData: {
+ data: Buffer.from(audioBuffer).toString("base64"),
+ mimeType: mimeType || 'audio/wav',
+ },
+ };
+
+ const prompt = `transcribe the audio and give the transcription in ${lang} without any timestamps and new line characters`;
+
+ const result = await model.generateContent([audio, prompt]);
+ return result.response.text(); // Return the transcription result
+};
+
+
+const handleAudio = async (req, res) => {
+ console.log("Received POST request");
+ try {
+ const { audio, mimeType, lang } = req.body;
+ // console.log("audio ", audio);
+ // console.log("mimeType ", mimeType);
+ // console.log("lang ", lang);
+ if (!audio) {
+ return res.status(400).json({ message: "No audio data received" });
+ }
+ if (!lang) {
+ return res.status(400).json({ message: "No language data received" });
+ }
+
+ // Decode Base64 audio data
+ const audioBuffer = Buffer.from(audio, 'base64');
+ const mimeTypeFromHeader = mimeType || 'audio/wav';
+
+ // Perform transcription
+ const transcription = await transcribeAudio(audioBuffer, mimeTypeFromHeader, lang);
+ console.log(transcription);
+ res.status(200).json({ message: "Transcription completed", transcription });
+
+ } catch (error) {
+ console.error("Error processing audio: ", error);
+ res.status(500).json({ message: "Internal server error", error: error.message });
+ }
+};
+
+
+module.exports = {handleAudio};
\ No newline at end of file
diff --git a/server/package-lock.json b/server/package-lock.json
index 1e67da7..473503a 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -9,6 +9,7 @@
"version": "1.0.0",
"license": "ISC",
"dependencies": {
+ "@google/generative-ai": "^0.19.0",
"body-parser": "^1.20.2",
"cors": "^2.8.5",
"dotenv": "^16.4.5",
@@ -210,6 +211,14 @@
"uuid": "dist/bin/uuid"
}
},
+ "node_modules/@google/generative-ai": {
+ "version": "0.19.0",
+ "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.19.0.tgz",
+ "integrity": "sha512-iGf/62v3sTwtEJGJY6S5m7PfkglU8hi1URaxqIjiRg1OItV27xyc4aVeR0og8cDkZFkUlGZKv+23bJtT1QWFzQ==",
+ "engines": {
+ "node": ">=18.0.0"
+ }
+ },
"node_modules/@grpc/grpc-js": {
"version": "1.11.1",
"resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.11.1.tgz",
diff --git a/server/package.json b/server/package.json
index dc251fe..0c8491b 100644
--- a/server/package.json
+++ b/server/package.json
@@ -11,6 +11,7 @@
"license": "ISC",
"description": "",
"dependencies": {
+ "@google/generative-ai": "^0.19.0",
"body-parser": "^1.20.2",
"cors": "^2.8.5",
"dotenv": "^16.4.5",
diff --git a/server/server.js b/server/server.js
index f541157..258b151 100644
--- a/server/server.js
+++ b/server/server.js
@@ -5,6 +5,7 @@ const http = require('http');
const cors = require('cors');
const app = express();
const {Server} = require('socket.io');
+const {handleAudio} = require("./audioHandler");
const SERVER_PORT = process.env.SERVER_PORT;
const CLIENT_PORT = process.env.CLIENT_PORT;
@@ -14,6 +15,9 @@ const server = http.createServer(app);
//middleware
app.use(cors());
+app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
+// app.use(express.raw({ type: 'audio/wav', limit: '10mb' }));
//routings
app.get('/states', (req, res) => {
@@ -40,6 +44,7 @@ app.get('/areas/:state/:district/:subdistrict', (req, res) => {
const areas = states[state][district][subdistrict] || [];
res.json(areas);
});
+app.post('/chat', handleAudio);
//socket.io connections
const io = new Server(server, {
@@ -49,6 +54,8 @@ const io = new Server(server, {
}
})
+
+
io.on("connection", (socket)=>{
console.log(socket.id);
@@ -72,3 +79,4 @@ io.on("connection", (socket)=>{
server.listen(SERVER_PORT, () => {
console.log(`Server is running on http://localhost:${SERVER_PORT}`);
});
+