shinyonaika-test

Gamifying Cognitive Behavioural Therapy using neural networks and game technology

The AI model used are Whisper for converting audio to text and Cohere:classify for understanding the user intend.

#Installation of whisper and cohere library are:

!pip install git+https://github.com/openai/whisper.git !sudo apt update && sudo apt install ffmpeg !pip install cohere

**#used Pyaudio for recording live audio **#pyaudio installation

!pip install pyaudio

After that include all the package that are need and record the live voice of the user as the input usig Pyaudio.

from IPython.display import HTML, Audio from google.colab.output import eval_js from base64 import b64decode import numpy as np from scipy.io.wavfile import read as wav_read import io import ffmpeg import wave import whisper import cohere from cohere.classify import Example AUDIO_HTML = """

<script> var my_div = document.createElement("DIV"); var my_p = document.createElement("P"); var my_btn = document.createElement("BUTTON"); var t = document.createTextNode("Press to start recording"); my_btn.appendChild(t); //my_p.appendChild(my_btn); my_div.appendChild(my_btn); document.body.appendChild(my_div); var base64data = 0; var reader; var recorder, gumStream; var recordButton = my_btn; var handleSuccess = function(stream) { gumStream = stream; var options = { //bitsPerSecond: 8000, //chrome seems to ignore, always 48k mimeType : 'audio/webm;codecs=opus' //mimeType : 'audio/webm;codecs=pcm' }; //recorder = new MediaRecorder(stream, options); recorder = new MediaRecorder(stream); recorder.ondataavailable = function(e) { var url = URL.createObjectURL(e.data); var preview = document.createElement('audio'); preview.controls = true; preview.src = url; document.body.appendChild(preview); reader = new FileReader(); reader.readAsDataURL(e.data); reader.onloadend = function() { base64data = reader.result; //console.log("Inside FileReader:" + base64data); } }; recorder.start(); }; recordButton.innerText = "Recording... press to stop"; navigator.mediaDevices.getUserMedia({audio: true}).then(handleSuccess); function toggleRecording() { if (recorder && recorder.state == "recording") { recorder.stop(); gumStream.getAudioTracks()[0].stop(); recordButton.innerText = "Saving the recording... pls wait!" } } // https://stackoverflow.com/a/951057 function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } var data = new Promise(resolve=>{ //recordButton.addEventListener("click", toggleRecording); recordButton.onclick = ()=>{ toggleRecording() sleep(2000).then(() => { // wait 2000ms for the data to be available... // ideally this should use something like await... //console.log("Inside data:" + base64data) resolve(base64data.toString()) }); } }); </script>

"""

def get_audio(): display(HTML(AUDIO_HTML)) data = eval_js("data") binary = b64decode(data.split(',')[1])

process = (ffmpeg .input('pipe:0') .output('pipe:1', format='wav') .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True, quiet=True, overwrite_output=True) ) output, err = process.communicate(input=binary)

riff_chunk_size = len(output) - 8

Break up the chunk size into four bytes, held in b.

q = riff_chunk_size b = [] for i in range(4): q, r = divmod(q, 256) b.append(r)

Replace bytes 4:8 in proc.stdout with the actual size of the RIFF chunk.

riff = output[:4] + bytes(b) + output[8:]

sr, audio = wav_read(io.BytesIO(riff))

return audio, sr

file_name = 'audio.wav' nchannels = 1 # mono audio sample_width = 2 # 16-bit audio framerate = 44100 # CD-quality audio nframes = 0 # to be determined later comptype = 'NONE' compname = 'not compressed'

Open the WAV file for writing

with wave.open(file_name, 'wb') as f:

# Set the WAV file parameters
audio, sr= get_audio()
f.setparams((nchannels, sample_width, framerate, nframes, comptype, compname))

# Write the audio data to the file
audio_data = bytes([0x00, 0x7f, 0x00, 0x7f])  # Example audio data
f.writeframes(audio)

# Update the number of frames in the WAV file header
nframes = len(audio_data) // sample_width

And using Whisper AI model the audio recorded is converted into text file and store id the sane directory.

!whisper "audio.wav" --model medium.en

Using file operation open the text file and read the content from the and check with the classify with predefined Example defined .

file=open('audio.txt','r') inputs=file.readlines() co= cohere.Client('wDK8enezNw0SdTyDpqEBx4nTrVDyoeaVh6C1QydU')

examples = [ Example('Socializing','Lonliness'), Example('Low self-esteem','Lonliness'), Example('Understood','Lonliness'), Example('Self-criticism','Lonliness'), Example('betrayal','Lonliness'), Example('Misfit','Lonliness'), Example('belonging','Lonliness'), Example('Community','Lonliness'), Example('Tribe','Lonliness'), Example('Support','Lonliness'), Example('Trust issues','Lonliness'), Example('Introverted ness','Lonliness'), Example('Social anxiety','Lonliness'), Example('postponing','procastination'), Example('delay','procastination') ] response = co.classify( model='small', inputs=inputs, examples=examples )

print(response.classifications)